diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index e5becd67e..ac8c01001 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -1,4 +1,5 @@ import io +import asyncio from http import client import pytest @@ -11,6 +12,7 @@ from waterbutler.providers.nextcloud.metadata import (NextcloudFileMetadata, NextcloudFileRevisionMetadata) +from unittest import mock from tests import utils from tests.providers.nextcloud.fixtures import ( provider, @@ -599,3 +601,138 @@ def test_shares_storage_root(self, provider, provider_different_credentials): def test_can_duplicate_names(self, provider): assert provider.can_duplicate_names() + + +class FilePathFactory: + def __init__(self, _href): + self._href = _href + self.is_file = True + + +class TestMetadataFolder: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_folder_path_is_dir(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_metadata_2, file_checksum, file_checksum_2, file_checksum_3): + + # Define shared values + checksum_api_path = 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + revision_path = '&revision=1591864889' + + # Setup path for the _metadata_folder method + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url and call revision + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + + # Setup url and call checksum api for file_checksum + checksum_url = provider._ocs_url + checksum_api_path + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + + # Setup url and call checksum api for file_checksum_2 + checksum_url = provider._ocs_url + checksum_api_path + revision_path + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + + # Call checksum api for file_checksum_3 + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + # Setup return value for mock + future = asyncio.Future() + future.set_result([FilePathFactory('/my_folder/dissertation.aux')]) + + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): + result = await provider._metadata_folder(path) + + # Assert result + assert isinstance(result, list) + assert len(result) > 0 + + +class TestRevisionsNextCloud: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions_nextcloud(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_checksum, file_checksum_2, file_checksum_3): + # Setup path for the _metadata_folder method + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url for call revision + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + + # Call checksum api for file_checksum_1 + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + + # Call checksum api for file_checksum_2 + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + + # Call checksum api for file_checksum_3 + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + result = await provider.revisions(path) + + assert isinstance(result, list) + assert len(result) == 3 + assert isinstance(result[0], NextcloudFileRevisionMetadata) + assert isinstance(result[1], NextcloudFileRevisionMetadata) + assert isinstance(result[2], NextcloudFileRevisionMetadata) + + assert result[0].modified == 'Sun, 10 Jul 2016 23:28:31 GMT' + assert result[0].version == 'a3c411808d58977a9ecd7485b5b7958e' + assert result[0].version_identifier == 'revision' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions_nextcloud_when_have_more_than_one_version(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_checksum, file_checksum_2, file_checksum_3): + # Define shared values + checksum_api_path = 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + revision_path = '&revision=1591864889' + + # Setup path for the _metadata_folder method + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url for call revision + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + + # Setup url and call checksum api for file_checksum + checksum_url = provider._ocs_url + checksum_api_path + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + + # Setup url and call checksum api for file_checksum_2 + checksum_url = provider._ocs_url + checksum_api_path + revision_path + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + + # Call checksum api for file_checksum_3 + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + # Setup return value for mock + future = asyncio.Future() + future.set_result([FilePathFactory('/my_folder/dissertation.aux'), FilePathFactory('/my_folder/source.aux')]) + + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): + result = await provider._metadata_revision(path) + + # Assert result + assert isinstance(result, list) + assert len(result) == 2 + diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index 29261b50d..80ce15da1 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) _THROTTLES = weakref.WeakKeyDictionary() # type: weakref.WeakKeyDictionary -NO_URL_ENCODED_PROVIDERS = ['nextcloud', 'owncloud', 'nextcloudinstitutions'] +NO_URL_ENCODED_PROVIDERS = ['nextcloud', 'nextcloudinstitutions', 'owncloud'] def throttle(concurrency=10, interval=1): @@ -292,8 +292,10 @@ async def make_request(self, method, url, *args, **kwargs): while retry >= 0: # Don't overwrite the callable ``url`` so that signed URLs are refreshed for every retry non_callable_url = url() if callable(url) else url + # some providers need encode URL before passing it to aiohttp request if self.NAME not in NO_URL_ENCODED_PROVIDERS: - # Fix storage 'nextcloud', 'owncloud', 'nextcloudinstitutions' return HTTP 400 bad request + # use `encoded=True` parameter prevents URL auto-encoding, + # the user is responsible about URL correctness. non_callable_url = URL(non_callable_url, encoded=True) try: self.provider_metrics.incr('requests.count') diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 074a9fb44..86f849429 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -280,28 +280,29 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): items = await utils.parse_dav_response(self.NAME, content, self.folder, skip_first) await response.release() - for i in items: - if i.is_file and self.NAME == 'nextcloudinstitutions': - params = { - 'path': i._href, - 'hash': 'md5,sha256,sha512' - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) - - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - i.extra = extra - await response.release() + if not path.is_dir: + for i in items: + if i.is_file: + params = { + 'path': i._href, + 'hash': 'md5,sha256,sha512' + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) + + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + i.extra = extra + await response.release() return items @@ -326,27 +327,26 @@ async def _metadata_revision(self, path): if len(items) != 1: return items - if self.NAME == 'nextcloudinstitutions': - params = { - 'path': path.full_path, - 'hash': 'md5,sha256,sha512' - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) + params = { + 'path': path.full_path, + 'hash': 'md5,sha256,sha512' + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - items[0].extra = extra - await response.release() + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + items[0].extra = extra + await response.release() fileid = items[0].fileid @@ -364,29 +364,28 @@ async def _metadata_revision(self, path): revision_items = await utils.parse_dav_response(self.NAME, content, self.folder, True) await response.release() - if self.NAME == 'nextcloudinstitutions': - for rev in revision_items: - params = { - 'path': path.full_path, - 'hash': 'md5,sha256,sha512', - 'revision': str(rev.etag) - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) - - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - rev.extra = extra - await response.release() + for rev in revision_items: + params = { + 'path': path.full_path, + 'hash': 'md5,sha256,sha512', + 'revision': str(rev.etag) + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) + + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + rev.extra = extra + await response.release() items.extend(revision_items)