From 2fc977bada0ed64a8a3d4cfc331e11c1776f8641 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Thu, 5 Jan 2023 11:47:25 +0700 Subject: [PATCH 01/13] C3 & D3 - Add code --- waterbutler/providers/nextcloud/provider.py | 63 ++++++++++++--------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 074a9fb44..198fa8755 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -1,5 +1,6 @@ import logging import aiohttp +import time from waterbutler.core import streams from waterbutler.core import provider @@ -280,28 +281,32 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): items = await utils.parse_dav_response(self.NAME, content, self.folder, skip_first) await response.release() - for i in items: - if i.is_file and self.NAME == 'nextcloudinstitutions': - params = { - 'path': i._href, - 'hash': 'md5,sha256,sha512' - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) - - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - i.extra = extra - await response.release() + if not path.is_dir: + for i in items: + if i.is_file: + start_time = time.time() + params = { + 'path': i._href, + 'hash': 'md5,sha256' + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) + + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + i.extra = extra + await response.release() + res = time.time() - start_time + logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_folder: {res * 1000}') return items @@ -326,10 +331,11 @@ async def _metadata_revision(self, path): if len(items) != 1: return items - if self.NAME == 'nextcloudinstitutions': + if self.NAME: + start_time = time.time() params = { 'path': path.full_path, - 'hash': 'md5,sha256,sha512' + 'hash': 'md5,sha256' } response = await self.make_request('GET', self._ocs_url + 'apps/checksum_api/api/checksum', @@ -347,6 +353,8 @@ async def _metadata_revision(self, path): extra['hashes'] = await utils.parse_checksum_response(content) items[0].extra = extra await response.release() + res = time.time() - start_time + logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_revision: {res * 1000}') fileid = items[0].fileid @@ -364,11 +372,12 @@ async def _metadata_revision(self, path): revision_items = await utils.parse_dav_response(self.NAME, content, self.folder, True) await response.release() - if self.NAME == 'nextcloudinstitutions': + if self.NAME: for rev in revision_items: + start_time = time.time() params = { 'path': path.full_path, - 'hash': 'md5,sha256,sha512', + 'hash': 'md5,sha256', 'revision': str(rev.etag) } response = await self.make_request('GET', @@ -387,6 +396,8 @@ async def _metadata_revision(self, path): extra['hashes'] = await utils.parse_checksum_response(content) rev.extra = extra await response.release() + res = time.time() - start_time + logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_revision: {res * 1000}') items.extend(revision_items) From 01829a5bab2c82cd856c1f7ba562dbbfab2abc0d Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Thu, 12 Jan 2023 10:08:37 +0700 Subject: [PATCH 02/13] C3 & D3 - Update code --- waterbutler/providers/nextcloud/provider.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 198fa8755..7faf09c67 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -1,6 +1,5 @@ import logging import aiohttp -import time from waterbutler.core import streams from waterbutler.core import provider @@ -284,10 +283,9 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): if not path.is_dir: for i in items: if i.is_file: - start_time = time.time() params = { 'path': i._href, - 'hash': 'md5,sha256' + 'hash': 'md5,sha256,sha512' } response = await self.make_request('GET', self._ocs_url + 'apps/checksum_api/api/checksum', @@ -305,8 +303,6 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): extra['hashes'] = await utils.parse_checksum_response(content) i.extra = extra await response.release() - res = time.time() - start_time - logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_folder: {res * 1000}') return items @@ -331,11 +327,10 @@ async def _metadata_revision(self, path): if len(items) != 1: return items - if self.NAME: - start_time = time.time() + if self.NAME == 'nextcloudinstitutions': params = { 'path': path.full_path, - 'hash': 'md5,sha256' + 'hash': 'md5,sha256,sha512' } response = await self.make_request('GET', self._ocs_url + 'apps/checksum_api/api/checksum', @@ -353,8 +348,6 @@ async def _metadata_revision(self, path): extra['hashes'] = await utils.parse_checksum_response(content) items[0].extra = extra await response.release() - res = time.time() - start_time - logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_revision: {res * 1000}') fileid = items[0].fileid @@ -372,12 +365,11 @@ async def _metadata_revision(self, path): revision_items = await utils.parse_dav_response(self.NAME, content, self.folder, True) await response.release() - if self.NAME: + if self.NAME == 'nextcloudinstitutions': for rev in revision_items: - start_time = time.time() params = { 'path': path.full_path, - 'hash': 'md5,sha256', + 'hash': 'md5,sha256,sha512', 'revision': str(rev.etag) } response = await self.make_request('GET', @@ -396,8 +388,6 @@ async def _metadata_revision(self, path): extra['hashes'] = await utils.parse_checksum_response(content) rev.extra = extra await response.release() - res = time.time() - start_time - logger.info(f'Time execute apps/checksum_api/api/checksum in _metadata_revision: {res * 1000}') items.extend(revision_items) From 0824006e314136fdbb089ef6d46e451c8213513a Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Tue, 17 Jan 2023 14:33:39 +0700 Subject: [PATCH 03/13] C3 & D3 - UT code --- tests/providers/nextcloud/test_provider.py | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index e5becd67e..8621c7ff5 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -1,4 +1,5 @@ import io +import asyncio from http import client import pytest @@ -11,6 +12,7 @@ from waterbutler.providers.nextcloud.metadata import (NextcloudFileMetadata, NextcloudFileRevisionMetadata) +from unittest import mock from tests import utils from tests.providers.nextcloud.fixtures import ( provider, @@ -599,3 +601,35 @@ def test_shares_storage_root(self, provider, provider_different_credentials): def test_can_duplicate_names(self, provider): assert provider.can_duplicate_names() + + + +class FilePathFactory: + def __init__(self, _href): + self._href = _href + self.is_file = True + + +class TestMetadataFolder: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_metadata_folder_path_is_dir(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_metadata_2, file_checksum, file_checksum_2, file_checksum_3): + + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + future = asyncio.Future() + future.set_result([FilePathFactory('/my_folder/dissertation.aux')]) + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): + result = await provider._metadata_folder(path) + assert isinstance(result, list) From a35b60d4cf4d21a53cabce5e3352f6d2e5e81057 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Tue, 17 Jan 2023 14:55:56 +0700 Subject: [PATCH 04/13] C3 & D3 - udapte UT code --- tests/providers/nextcloud/test_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index 8621c7ff5..951bd8b01 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -603,7 +603,6 @@ def test_can_duplicate_names(self, provider): assert provider.can_duplicate_names() - class FilePathFactory: def __init__(self, _href): self._href = _href @@ -633,3 +632,4 @@ async def test_metadata_folder_path_is_dir(self, provider, file_metadata, file_r with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): result = await provider._metadata_folder(path) assert isinstance(result, list) + assert len(result) > 0 From 1e5f330ad6e2ce0583f8193859591f33e7575f85 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Tue, 7 Feb 2023 16:01:58 +0700 Subject: [PATCH 05/13] C2 - Add code --- waterbutler/providers/nextcloud/provider.py | 84 ++++++++++----------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 7faf09c67..86f849429 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -327,27 +327,26 @@ async def _metadata_revision(self, path): if len(items) != 1: return items - if self.NAME == 'nextcloudinstitutions': - params = { - 'path': path.full_path, - 'hash': 'md5,sha256,sha512' - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) + params = { + 'path': path.full_path, + 'hash': 'md5,sha256,sha512' + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - items[0].extra = extra - await response.release() + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + items[0].extra = extra + await response.release() fileid = items[0].fileid @@ -365,29 +364,28 @@ async def _metadata_revision(self, path): revision_items = await utils.parse_dav_response(self.NAME, content, self.folder, True) await response.release() - if self.NAME == 'nextcloudinstitutions': - for rev in revision_items: - params = { - 'path': path.full_path, - 'hash': 'md5,sha256,sha512', - 'revision': str(rev.etag) - } - response = await self.make_request('GET', - self._ocs_url + 'apps/checksum_api/api/checksum', - params=params, - expects=(200, 404), - throws=exceptions.MetadataError, - auth=self._auth, - connector=self.connector(), - headers={'OCS-APIRequest': 'true'} - ) - - if response.status == 200: - content = await response.content.read() - extra = {} - extra['hashes'] = await utils.parse_checksum_response(content) - rev.extra = extra - await response.release() + for rev in revision_items: + params = { + 'path': path.full_path, + 'hash': 'md5,sha256,sha512', + 'revision': str(rev.etag) + } + response = await self.make_request('GET', + self._ocs_url + 'apps/checksum_api/api/checksum', + params=params, + expects=(200, 404), + throws=exceptions.MetadataError, + auth=self._auth, + connector=self.connector(), + headers={'OCS-APIRequest': 'true'} + ) + + if response.status == 200: + content = await response.content.read() + extra = {} + extra['hashes'] = await utils.parse_checksum_response(content) + rev.extra = extra + await response.release() items.extend(revision_items) From a6b6d60374012a4b5d1d3240ca7217fa9cfc9f3f Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Fri, 10 Feb 2023 11:50:08 +0700 Subject: [PATCH 06/13] =?UTF-8?q?refs=20(C).2.5=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20Add=20UT=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/providers/nextcloud/test_provider.py | 55 ++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index 951bd8b01..6a40197be 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -633,3 +633,58 @@ async def test_metadata_folder_path_is_dir(self, provider, file_metadata, file_r result = await provider._metadata_folder(path) assert isinstance(result, list) assert len(result) > 0 + + +class TestRevisionsNextCloud: + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions_nextcloud(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_checksum, file_checksum_2, file_checksum_3): + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + result = await provider.revisions(path) + + assert isinstance(result, list) + assert len(result) == 3 + assert isinstance(result[0], NextcloudFileRevisionMetadata) + assert isinstance(result[1], NextcloudFileRevisionMetadata) + assert isinstance(result[2], NextcloudFileRevisionMetadata) + + assert result[0].modified == 'Sun, 10 Jul 2016 23:28:31 GMT' + assert result[0].version == 'a3c411808d58977a9ecd7485b5b7958e' + assert result[0].version_identifier == 'revision' + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_revisions_nextcloud_with_len_1(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_checksum, file_checksum_2, file_checksum_3): + path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + url = provider._webdav_url_ + path.full_path + aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid + aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + future = asyncio.Future() + future.set_result([FilePathFactory('/my_folder/dissertation.aux'), FilePathFactory('/my_folder/source.aux')]) + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): + result = await provider._metadata_revision(path) + + assert isinstance(result, list) + assert len(result) == 2 From 5ade4f3282454d027a70fe40ce3309e8dad3198f Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Fri, 10 Feb 2023 11:56:11 +0700 Subject: [PATCH 07/13] =?UTF-8?q?refs=20(C).3.5=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=83=A6=E3=83=BC=E3=82=B9=E3=82=B1=E3=83=BC?= =?UTF-8?q?=E3=82=B9:=20Update=20UT=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/providers/nextcloud/test_provider.py | 25 +++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index 6a40197be..30c2cb44a 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -616,21 +616,40 @@ class TestMetadataFolder: async def test_metadata_folder_path_is_dir(self, provider, file_metadata, file_revision_metadata, file_metadata_object, file_metadata_2, file_checksum, file_checksum_2, file_checksum_3): + # Define shared values + checksum_api_path = 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + revision_path = '&revision=1591864889' + + # Setup path for the _metadata_folder method path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata url = provider._webdav_url_ + path.full_path aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url and call revision url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + + # Setup url and call checksum api for file_checksum + checksum_url = provider._ocs_url + checksum_api_path aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + + # Setup url and call checksum api for file_checksum_2 + checksum_url = provider._ocs_url + checksum_api_path + revision_path aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + + # Call checksum api for file_checksum_3 aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + + # Setup return value for mock future = asyncio.Future() future.set_result([FilePathFactory('/my_folder/dissertation.aux')]) + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): result = await provider._metadata_folder(path) + + # Assert result assert isinstance(result, list) assert len(result) > 0 From b9741f91336353b2fb76430d2ce2b6f1ce8ec8b9 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Mon, 13 Feb 2023 17:21:04 +0700 Subject: [PATCH 08/13] =?UTF-8?q?refs=20(C).2.5=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20Update=20UT=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/providers/nextcloud/test_provider.py | 41 ++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/tests/providers/nextcloud/test_provider.py b/tests/providers/nextcloud/test_provider.py index 30c2cb44a..ac8c01001 100644 --- a/tests/providers/nextcloud/test_provider.py +++ b/tests/providers/nextcloud/test_provider.py @@ -659,16 +659,27 @@ class TestRevisionsNextCloud: @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_revisions_nextcloud(self, provider, file_metadata, file_revision_metadata, file_metadata_object, - file_checksum, file_checksum_2, file_checksum_3): + file_checksum, file_checksum_2, file_checksum_3): + # Setup path for the _metadata_folder method path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata url = provider._webdav_url_ + path.full_path aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url for call revision url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) + + # Call checksum api for file_checksum_1 checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) + + # Call checksum api for file_checksum_2 checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) + + # Call checksum api for file_checksum_3 checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) @@ -686,24 +697,42 @@ async def test_revisions_nextcloud(self, provider, file_metadata, file_revision_ @pytest.mark.asyncio @pytest.mark.aiohttpretty - async def test_revisions_nextcloud_with_len_1(self, provider, file_metadata, file_revision_metadata, file_metadata_object, - file_checksum, file_checksum_2, file_checksum_3): + async def test_revisions_nextcloud_when_have_more_than_one_version(self, provider, file_metadata, file_revision_metadata, file_metadata_object, + file_checksum, file_checksum_2, file_checksum_3): + # Define shared values + checksum_api_path = 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + revision_path = '&revision=1591864889' + + # Setup path for the _metadata_folder method path = WaterButlerPath('/dissertation.aux', prepend=provider.folder) + + # Setup url for call metadata url = provider._webdav_url_ + path.full_path aiohttpretty.register_uri('PROPFIND', url, body=file_metadata, auto_length=True, status=207) + + # Setup url for call revision url = provider._dav_url_ + 'versions/' + provider.credentials['username'] + '/versions/' + file_metadata_object.fileid aiohttpretty.register_uri('PROPFIND', url, body=file_revision_metadata, auto_length=True, status=207) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512' + + # Setup url and call checksum api for file_checksum + checksum_url = provider._ocs_url + checksum_api_path aiohttpretty.register_uri('GET', checksum_url, body=file_checksum, auto_length=True, status=200) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591876099' + + # Setup url and call checksum api for file_checksum_2 + checksum_url = provider._ocs_url + checksum_api_path + revision_path aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_2, auto_length=True, status=200) - checksum_url = provider._ocs_url + 'apps/checksum_api/api/checksum?path=/my_folder/dissertation.aux&hash=md5,sha256,sha512&revision=1591864889' + + # Call checksum api for file_checksum_3 aiohttpretty.register_uri('GET', checksum_url, body=file_checksum_3, auto_length=True, status=200) + # Setup return value for mock future = asyncio.Future() future.set_result([FilePathFactory('/my_folder/dissertation.aux'), FilePathFactory('/my_folder/source.aux')]) + with mock.patch('waterbutler.providers.nextcloud.utils.parse_dav_response', return_value=future): result = await provider._metadata_revision(path) + # Assert result assert isinstance(result, list) assert len(result) == 2 + From 2d2fbeb16a75e7aa47ab48256d12d110a22b67fc Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Wed, 8 Mar 2023 09:41:59 +0700 Subject: [PATCH 09/13] =?UTF-8?q?refs=20(C).2.6=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20Fix=20bug=20IT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- waterbutler/providers/nextcloud/provider.py | 10 +++++----- waterbutler/server/api/v1/provider/__init__.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 86f849429..246d26ab0 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -177,7 +177,7 @@ async def download(self, path, accept_url=False, range=None, **kwargs): if revision is None: download_resp = await self.make_request( 'GET', - self._webdav_url_ + path.full_path, + self._webdav_url_ + path.full_path.replace(' ', '%20'), range=range, expects=(200, 206,), throws=exceptions.DownloadError, @@ -214,7 +214,7 @@ async def upload(self, stream, path, conflict='replace', **kwargs): response = await self.make_request( 'PUT', - self._webdav_url_ + path.full_path, + self._webdav_url_ + path.full_path.replace(' ', '%20'), data=stream, headers={'Content-Length': str(stream.size)}, expects=(201, 204,), @@ -234,7 +234,7 @@ async def delete(self, path, **kwargs): """ delete_resp = await self.make_request( 'DELETE', - self._webdav_url_ + path.full_path, + self._webdav_url_ + path.full_path.replace(' ', '%20'), expects=(204,), throws=exceptions.DeleteError, auth=self._auth, @@ -267,7 +267,7 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): * 207: Multipart response """ response = await self.make_request('PROPFIND', - self._webdav_url_ + path.full_path, + self._webdav_url_ + path.full_path.replace(' ', '%20'), expects=(204, 207), throws=exceptions.MetadataError, auth=self._auth, @@ -310,7 +310,7 @@ async def _metadata_revision(self, path): query = ' ' response = await self.make_request('PROPFIND', - self._webdav_url_ + path.full_path, + self._webdav_url_ + path.full_path.replace(' ', '%20'), data=query, expects=(204, 207), throws=exceptions.MetadataError, diff --git a/waterbutler/server/api/v1/provider/__init__.py b/waterbutler/server/api/v1/provider/__init__.py index 98c4201a3..ebabd7252 100644 --- a/waterbutler/server/api/v1/provider/__init__.py +++ b/waterbutler/server/api/v1/provider/__init__.py @@ -42,7 +42,7 @@ async def prepare(self, *args, **kwargs): method = self.request.method.lower() # TODO Find a nicer way to handle this - if method == 'options': + if method == 'options' or self.path_kwargs['path'].endswith('.map.css'): return self.arguments = { From ba49a6227466d420a182171f53f64f5d84418227 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Wed, 8 Mar 2023 13:51:42 +0700 Subject: [PATCH 10/13] =?UTF-8?q?refs=20(C).2.6=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20Fix=20bug=20IT=2002?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- waterbutler/server/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/waterbutler/server/utils.py b/waterbutler/server/utils.py index f89f8b752..46782219b 100644 --- a/waterbutler/server/utils.py +++ b/waterbutler/server/utils.py @@ -130,6 +130,7 @@ async def write_stream(self, stream): # Temp fix, write does not accept bytearrays currently if isinstance(chunk, bytearray): chunk = bytes(chunk) + self.set_header('Content-Length', len(chunk)) self.write(chunk) self.bytes_downloaded += len(chunk) del chunk From 88186f47d61d4bb1307937e97d7d2ff1be82ad2d Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Thu, 9 Mar 2023 18:18:54 +0700 Subject: [PATCH 11/13] =?UTF-8?q?refs=20(C).2.6=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20Fix=20bug=20IT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- waterbutler/providers/nextcloud/provider.py | 6 +++--- waterbutler/server/api/v1/provider/__init__.py | 2 +- waterbutler/server/utils.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 246d26ab0..9655ad330 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -109,7 +109,7 @@ async def validate_v1_path(self, path, **kwargs): full_path = WaterButlerPath(path, prepend=self.folder) response = await self.make_request('PROPFIND', - self._webdav_url_ + full_path.full_path, + self._webdav_url_ + full_path.full_path.replace(' ', '%20'), expects=(200, 207, 404), throws=exceptions.MetadataError, auth=self._auth, @@ -445,12 +445,12 @@ async def _do_dav_move_copy(self, src_path, dest_path, operation): resp = await self.make_request( operation, - self._webdav_url_ + src_path.full_path, + self._webdav_url_ + src_path.full_path.replace(' ', '%20'), expects=(201, 204), # WebDAV MOVE/COPY: 201 = Created, 204 = Updated existing throws=exceptions.IntraCopyError, auth=self._auth, connector=self.connector(), - headers={'Destination': '/remote.php/webdav' + dest_path.full_path} + headers={'Destination': '/remote.php/webdav' + dest_path.full_path.replace(' ', '%20')} ) await resp.release() diff --git a/waterbutler/server/api/v1/provider/__init__.py b/waterbutler/server/api/v1/provider/__init__.py index ebabd7252..63719ce61 100644 --- a/waterbutler/server/api/v1/provider/__init__.py +++ b/waterbutler/server/api/v1/provider/__init__.py @@ -42,7 +42,7 @@ async def prepare(self, *args, **kwargs): method = self.request.method.lower() # TODO Find a nicer way to handle this - if method == 'options' or self.path_kwargs['path'].endswith('.map.css'): + if method == 'options' or self.path_kwargs['path'].endswith(('.css', '.map')): return self.arguments = { diff --git a/waterbutler/server/utils.py b/waterbutler/server/utils.py index 46782219b..a730bf0c6 100644 --- a/waterbutler/server/utils.py +++ b/waterbutler/server/utils.py @@ -130,7 +130,10 @@ async def write_stream(self, stream): # Temp fix, write does not accept bytearrays currently if isinstance(chunk, bytearray): chunk = bytes(chunk) - self.set_header('Content-Length', len(chunk)) + self.set_header('Content-Length', str(len(chunk))) + if stream.size: + if stream.size > len(chunk): + self.set_header('Content-Length', str(stream.size)) self.write(chunk) self.bytes_downloaded += len(chunk) del chunk From 00214bfdeeaaf6dbaf3e91f5fbae0827ff13cf62 Mon Sep 17 00:00:00 2001 From: tma-ndhuy Date: Thu, 23 Mar 2023 15:29:33 +0700 Subject: [PATCH 12/13] Fix CI error "ZipStreamReader object has no attribute size" in the TestZipHandler.test_download_stream test case --- waterbutler/server/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waterbutler/server/utils.py b/waterbutler/server/utils.py index a730bf0c6..ebc4fec76 100644 --- a/waterbutler/server/utils.py +++ b/waterbutler/server/utils.py @@ -131,7 +131,7 @@ async def write_stream(self, stream): if isinstance(chunk, bytearray): chunk = bytes(chunk) self.set_header('Content-Length', str(len(chunk))) - if stream.size: + if hasattr(stream, 'size'): if stream.size > len(chunk): self.set_header('Content-Length', str(stream.size)) self.write(chunk) From 30b35db1a6aee92530c9665f58bc2d2d02af4131 Mon Sep 17 00:00:00 2001 From: Phat Nguyen Date: Thu, 7 Dec 2023 11:25:56 +0700 Subject: [PATCH 13/13] =?UTF-8?q?refs=20(C).2.6=20[=E6=A9=9F=E9=96=A2?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8]GakuNin=20RDM=20?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E6=A9=9F=E8=83=BD=E3=81=AE?= =?UTF-8?q?=E6=97=A2=E5=AD=98=E3=81=AE=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?:=20revert=20redundant=20changes=20in=20server=20package=20and?= =?UTF-8?q?=20correct=20URL=20encoding=20from=20nextcloud=20and=20nextclou?= =?UTF-8?q?d=20for=20institutions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- waterbutler/core/provider.py | 7 ++++++- waterbutler/providers/nextcloud/provider.py | 16 ++++++++-------- waterbutler/server/api/v1/provider/__init__.py | 2 +- waterbutler/server/utils.py | 4 ---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/waterbutler/core/provider.py b/waterbutler/core/provider.py index fe2fe5e61..5a9359e3d 100644 --- a/waterbutler/core/provider.py +++ b/waterbutler/core/provider.py @@ -25,6 +25,7 @@ logger = logging.getLogger(__name__) _THROTTLES = weakref.WeakKeyDictionary() # type: weakref.WeakKeyDictionary +NO_URL_ENCODED_PROVIDERS = ['nextcloud', 'nextcloudinstitutions'] def throttle(concurrency=10, interval=1): @@ -287,7 +288,11 @@ async def make_request(self, method, url, *args, **kwargs): while retry >= 0: # Don't overwrite the callable ``url`` so that signed URLs are refreshed for every retry non_callable_url = url() if callable(url) else url - non_callable_url = URL(non_callable_url, encoded=True) + # some provider need encode URL before pass to aiohttp request + if self.NAME not in NO_URL_ENCODED_PROVIDERS: + # use encoded=True parameter prevents URL auto-encoding, + # user is responsible about URL correctness. + non_callable_url = URL(non_callable_url, encoded=True) try: self.provider_metrics.incr('requests.count') # TODO: use a `dict` to select methods with either `lambda` or `functools.partial` diff --git a/waterbutler/providers/nextcloud/provider.py b/waterbutler/providers/nextcloud/provider.py index 9655ad330..86f849429 100644 --- a/waterbutler/providers/nextcloud/provider.py +++ b/waterbutler/providers/nextcloud/provider.py @@ -109,7 +109,7 @@ async def validate_v1_path(self, path, **kwargs): full_path = WaterButlerPath(path, prepend=self.folder) response = await self.make_request('PROPFIND', - self._webdav_url_ + full_path.full_path.replace(' ', '%20'), + self._webdav_url_ + full_path.full_path, expects=(200, 207, 404), throws=exceptions.MetadataError, auth=self._auth, @@ -177,7 +177,7 @@ async def download(self, path, accept_url=False, range=None, **kwargs): if revision is None: download_resp = await self.make_request( 'GET', - self._webdav_url_ + path.full_path.replace(' ', '%20'), + self._webdav_url_ + path.full_path, range=range, expects=(200, 206,), throws=exceptions.DownloadError, @@ -214,7 +214,7 @@ async def upload(self, stream, path, conflict='replace', **kwargs): response = await self.make_request( 'PUT', - self._webdav_url_ + path.full_path.replace(' ', '%20'), + self._webdav_url_ + path.full_path, data=stream, headers={'Content-Length': str(stream.size)}, expects=(201, 204,), @@ -234,7 +234,7 @@ async def delete(self, path, **kwargs): """ delete_resp = await self.make_request( 'DELETE', - self._webdav_url_ + path.full_path.replace(' ', '%20'), + self._webdav_url_ + path.full_path, expects=(204,), throws=exceptions.DeleteError, auth=self._auth, @@ -267,7 +267,7 @@ async def _metadata_folder(self, path, skip_first=True, **kwargs): * 207: Multipart response """ response = await self.make_request('PROPFIND', - self._webdav_url_ + path.full_path.replace(' ', '%20'), + self._webdav_url_ + path.full_path, expects=(204, 207), throws=exceptions.MetadataError, auth=self._auth, @@ -310,7 +310,7 @@ async def _metadata_revision(self, path): query = ' ' response = await self.make_request('PROPFIND', - self._webdav_url_ + path.full_path.replace(' ', '%20'), + self._webdav_url_ + path.full_path, data=query, expects=(204, 207), throws=exceptions.MetadataError, @@ -445,12 +445,12 @@ async def _do_dav_move_copy(self, src_path, dest_path, operation): resp = await self.make_request( operation, - self._webdav_url_ + src_path.full_path.replace(' ', '%20'), + self._webdav_url_ + src_path.full_path, expects=(201, 204), # WebDAV MOVE/COPY: 201 = Created, 204 = Updated existing throws=exceptions.IntraCopyError, auth=self._auth, connector=self.connector(), - headers={'Destination': '/remote.php/webdav' + dest_path.full_path.replace(' ', '%20')} + headers={'Destination': '/remote.php/webdav' + dest_path.full_path} ) await resp.release() diff --git a/waterbutler/server/api/v1/provider/__init__.py b/waterbutler/server/api/v1/provider/__init__.py index 63719ce61..98c4201a3 100644 --- a/waterbutler/server/api/v1/provider/__init__.py +++ b/waterbutler/server/api/v1/provider/__init__.py @@ -42,7 +42,7 @@ async def prepare(self, *args, **kwargs): method = self.request.method.lower() # TODO Find a nicer way to handle this - if method == 'options' or self.path_kwargs['path'].endswith(('.css', '.map')): + if method == 'options': return self.arguments = { diff --git a/waterbutler/server/utils.py b/waterbutler/server/utils.py index ebc4fec76..f89f8b752 100644 --- a/waterbutler/server/utils.py +++ b/waterbutler/server/utils.py @@ -130,10 +130,6 @@ async def write_stream(self, stream): # Temp fix, write does not accept bytearrays currently if isinstance(chunk, bytearray): chunk = bytes(chunk) - self.set_header('Content-Length', str(len(chunk))) - if hasattr(stream, 'size'): - if stream.size > len(chunk): - self.set_header('Content-Length', str(stream.size)) self.write(chunk) self.bytes_downloaded += len(chunk) del chunk