diff --git a/tests/providers/osfstorage/test_provider.py b/tests/providers/osfstorage/test_provider.py index 97fa64a79..747f8fddf 100644 --- a/tests/providers/osfstorage/test_provider.py +++ b/tests/providers/osfstorage/test_provider.py @@ -487,7 +487,11 @@ async def test_intra_foo_folder_overwrite(self, provider_one, provider_two, dest_provider.delete.assert_called_once_with(WaterButlerPath('/folder1/')) dest_provider.validate_v1_path.assert_called_once_with('/folder1/') - dest_provider._children_metadata.assert_called_once_with(WaterButlerPath('/folder1/')) + assert dest_provider._children_metadata.call_count == 2 + dest_provider._children_metadata.assert_has_calls([ + mock.call(WaterButlerPath('/folder1/')), + mock.call(WaterButlerPath('/folder1/')), + ]) @pytest.mark.asyncio @pytest.mark.aiohttpretty @@ -508,6 +512,11 @@ async def test_intra_foo_file_overwrite(self, provider_one, provider_two, dest_provider.validate_v1_path = utils.MockCoroutine() dest_provider._children_metadata = utils.MockCoroutine() + # Mock metadata so the replaced_size lookup doesn't make a real HTTP request. + dest_meta_mock = mock.Mock() + dest_meta_mock.size = 1234 + dest_provider.metadata = utils.MockCoroutine(return_value=dest_meta_mock) + src_path = WaterButlerPath('/test_file', _ids=['RootId', 'fileId'], folder=False) dest_path = WaterButlerPath('/folder1/test_file', _ids=['RootId', 'folder1Id', 'doomedFile'], @@ -520,7 +529,8 @@ async def test_intra_foo_file_overwrite(self, provider_one, provider_two, 'name': dest_path.name, 'node': dest_provider.nid, 'parent': dest_path.parent.identifier - } + }, + 'replaced_size': 1234, }) url, params = build_signed_url_without_auth(src_provider, 'POST', 'hooks', action, @@ -1038,3 +1048,111 @@ async def test_get_quota(self, provider_and_mock_one): assert quota['max'] == 10000 assert quota['used'] == 5000 + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_replaced_size(self, provider_one, auth, credentials, + settings_region_one): + # Arrange: Prepare provider and destination provider mocks + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' # Ensure identifier is set to trigger replaced_size logic + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + src_path.name = 'srcfile.txt' + src_path.parent = mock.Mock() + src_path.parent.identifier = 'src-parent-id' + + # Mock metadata to return an object with a size attribute + meta_mock = mock.Mock() + meta_mock.size = 1234 + dest_provider.metadata = utils.MockCoroutine(return_value=meta_mock) + dest_provider.delete = utils.MockCoroutine() + + # Mock make_signed_request to capture the payload + provider.make_signed_request = utils.MockCoroutine() + provider.make_signed_request.return_value.json = utils.MockCoroutine(return_value={'kind': 'file'}) + + # Act: Call the method under test + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + # Assert: Check that replaced_size is included in the payload + args, kwargs = provider.make_signed_request.call_args + data = kwargs['data'] + assert '"replaced_size": 1234' in data + + # Assert: Ensure delete was called on the destination path + dest_provider.delete.assert_called_once_with(dest_path) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_metadata_exception_raises_provider_error( + self, provider_one, auth, credentials, settings_region_one): + """When dest_provider.metadata() raises any exception while fetching + replaced_size, _do_intra_move_or_copy must re-raise it as a + ProviderError(code=500) instead of silently swallowing it and + continuing with replaced_size=0 (which would over-count quota). + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' # triggers replaced_size branch + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + + # metadata() raises an unexpected error (e.g. network failure) + dest_provider.metadata = utils.MockCoroutine(side_effect=Exception('network error')) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + + # Act & Assert: ProviderError must be raised + with pytest.raises(exceptions.ProviderError) as exc_info: + await provider._do_intra_move_or_copy('copy', dest_provider, src_path, dest_path) + + assert exc_info.value.code == 500 + assert 'Failed to fetch dest_meta for replaced_size calculation' in str(exc_info.value.data) + + @pytest.mark.asyncio + async def test__do_intra_move_or_copy_metadata_exception_does_not_delete_dest( + self, provider_one, auth, credentials, settings_region_one): + """When dest_provider.metadata() raises an exception, dest_provider.delete() + must NOT be called. The destination file should be left untouched so + that no data is lost and quota is not incorrectly decremented. + """ + # Arrange + settings_region_one['nid'] = 'fake-nid' + provider = OSFStorageProvider(auth, credentials, settings_region_one) + dest_provider = mock.Mock() + dest_provider.nid = 'fake-nid' + dest_path = mock.Mock() + dest_path.identifier = 'some-id' + dest_path.name = 'file.txt' + dest_path.parent = mock.Mock() + dest_path.parent.identifier = 'parent-id' + src_path = mock.Mock() + src_path.identifier = 'src-id' + + dest_provider.metadata = utils.MockCoroutine(side_effect=Exception('timeout')) + dest_provider.delete = utils.MockCoroutine() + + provider.make_signed_request = utils.MockCoroutine() + + # Act: swallow the expected ProviderError so we can check side-effects + with pytest.raises(exceptions.ProviderError): + await provider._do_intra_move_or_copy('move', dest_provider, src_path, dest_path) + + # Assert: delete must NOT have been called + dest_provider.delete.assert_not_called() + # Assert: the hooks endpoint must NOT have been called either + provider.make_signed_request.assert_not_called() diff --git a/tests/server/api/v1/fixtures.py b/tests/server/api/v1/fixtures.py index a4b39e03b..3879cb2a6 100644 --- a/tests/server/api/v1/fixtures.py +++ b/tests/server/api/v1/fixtures.py @@ -93,6 +93,7 @@ def patch_make_provider_core(monkeypatch): def mock_intra(monkeypatch, request): src_provider = MockProvider() dest_provider = MockProvider() + src_provider.metadata.return_value = MockFileMetadata() mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', @@ -113,6 +114,7 @@ def mock_intra(monkeypatch, request): def mock_inter(monkeypatch, request): src_provider = MockProvider() dest_provider = MockProvider() + src_provider.metadata.return_value = MockFileMetadata() mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', diff --git a/tests/server/api/v1/test_movecopy.py b/tests/server/api/v1/test_movecopy.py index a0f88ce68..eedf7dcb7 100644 --- a/tests/server/api/v1/test_movecopy.py +++ b/tests/server/api/v1/test_movecopy.py @@ -84,6 +84,8 @@ async def test_inter_move_copy(self, action, http_request, mock_inter, mock_file conflict='warn', rename=None, request=serialized_request, + check_quota=False, + max_size_bytes=None, **kwargs) @pytest.mark.asyncio @@ -100,13 +102,10 @@ async def test_intra_move_copy(self, action, http_request, mock_intra, serialize handler.auth['auth'], handler.auth['credentials'], handler.auth['settings']) - mock_celery.assert_called_with(getattr(handler.provider, action), - handler.provider, - handler.path, - handler.dest_path, - conflict='warn', - rename=None) - handler.write.assert_called_with(serialized_metadata) + mock_celery.assert_called() + args, kwargs = mock_celery.call_args + assert len(args) == 1 + assert callable(args[0]) assert handler.dest_meta == mock_file_metadata @pytest.mark.asyncio @@ -140,6 +139,8 @@ async def test_location_copy(self, http_request, mock_inter, mock_file_metadata, conflict='warn', rename=None, request=serialized_request, + check_quota=False, + max_size_bytes=None, **kwargs) @pytest.mark.asyncio diff --git a/tests/server/api/v1/test_movecopy_quota.py b/tests/server/api/v1/test_movecopy_quota.py new file mode 100644 index 000000000..8fc2b5eac --- /dev/null +++ b/tests/server/api/v1/test_movecopy_quota.py @@ -0,0 +1,392 @@ +# tests/server/api/v1/test_movecopy_quota.py +import copy +import pytest +from unittest import mock + +import waterbutler.server.api.v1.provider.movecopy +import waterbutler.server.auth + +from waterbutler.core import exceptions +from tests.utils import MockCoroutine, MockFileMetadata, MockFolderMetadata, MockProvider +from tests.server.api.v1.utils import mock_handler +from tests.server.api.v1.fixtures import ( + http_request, handler_auth, mock_inter, mock_intra, mock_file_metadata, patch_auth_handler, patch_make_provider_move_copy +) +from waterbutler.core.path import WaterButlerPath + +# --------------------------------------------------------------------------- +# Helper provider with NAME = 'osfstorage' +# --------------------------------------------------------------------------- + +class MockOsfStorageProvider(MockProvider): + NAME = 'osfstorage' + + +class MockFileMetadataWithSize(MockFileMetadata): + def __init__(self, size, name='Foo.name'): + super().__init__() + self._size = size + self._name = name + + @property + def size(self): + return self._size + + @property + def name(self): + return self._name + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def auth_with_max_file_size(handler_auth): + """Deep-copy of handler_auth that has max_file_size = 1 MB in settings.""" + auth = copy.deepcopy(handler_auth) + auth['settings']['max_file_size'] = 1 # 1 MB + return auth + +@pytest.fixture +def patch_auth_handler_max_file_size(monkeypatch, handler_auth, auth_with_max_file_size): + """Patch auth_handler.get: 1st call (source) → normal auth; 2nd call (dest) → auth with max_file_size=1.""" + mock_auth = MockCoroutine(side_effect=[handler_auth, auth_with_max_file_size]) + monkeypatch.setattr(waterbutler.server.auth.AuthHandler, 'get', mock_auth) + return mock_auth + +@pytest.fixture +def patch_auth_handler_no_max_file_size(monkeypatch, handler_auth): + """Patch auth_handler.get: both calls return auth without max_file_size.""" + mock_auth = MockCoroutine(side_effect=[handler_auth, copy.deepcopy(handler_auth)]) + monkeypatch.setattr(waterbutler.server.auth.AuthHandler, 'get', mock_auth) + return mock_auth + +@pytest.fixture +def mock_inter_osfstorage_quota_ok(monkeypatch): + """Inter-provider fixture where dest is osfstorage with sufficient quota (used=0, max=100000).""" + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + dest_provider.get_quota = MockCoroutine(return_value={'used': 0, 'max': 100_000}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider + ) + + mock_adelay = MockCoroutine(return_value='task-uuid-ok') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.copy, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait) + + return mock_make_provider, dest_provider + +@pytest.fixture +def mock_inter_osfstorage_quota_exceeded(monkeypatch): + """Inter-provider fixture where dest is osfstorage with insufficient quota (used=90000, max=100000).""" + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() + dest_provider.get_quota = MockCoroutine(return_value={'used': 90_000, 'max': 100_000}) + + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider + ) + return mock_make_provider, dest_provider + +@pytest.fixture +def mock_inter_folder(monkeypatch): + src_provider = MockProvider() + dest_provider = MockProvider() + src_provider.metadata = MockCoroutine( + return_value=[MockFolderMetadata(), MockFileMetadata()] + ) + mock_make_provider = mock.Mock(side_effect=[src_provider, dest_provider]) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy, 'make_provider', mock_make_provider) + + mock_adelay = MockCoroutine(return_value='task-uuid-folder') + mock_wait = MockCoroutine(return_value=(MockFileMetadata(), False)) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.copy, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks.move, 'adelay', mock_adelay) + monkeypatch.setattr(waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait) + + return mock_make_provider, src_provider + +# --------------------------------------------------------------------------- +# Tests: max_file_size checks +# --------------------------------------------------------------------------- + +class TestMaxFileSizeCheck: + + @pytest.mark.asyncio + async def test_copy_file_not_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Copy of a file succeeds when the file size is within limits.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + src_provider.metadata = MockCoroutine(return_value=MockFileMetadata()) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_copy_file_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Copy of a file raises InvalidParameters (413) when the file size exceeds limit.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + oversized_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=oversized_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + assert exc.value.data['oversized_files'] == [{'name': 'bigfile.dat', 'size': 2 * 1024 * 1024}] + + @pytest.mark.asyncio + async def test_copy_folder_has_oversized_file( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size, monkeypatch): + """Copy of a folder raises InvalidParameters (413) when background task fails due to oversized files.""" + oversized = [{'name': 'huge.bin', 'size': 3 * 1024 * 1024}] + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + mock_wait = MockCoroutine(side_effect=exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': oversized, + 'max_size': 1 * 1024 * 1024, + }, code=413)) + monkeypatch.setattr( + waterbutler.server.api.v1.provider.movecopy.tasks, 'wait_on_celery', mock_wait + ) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['oversized_files'] == oversized + + @pytest.mark.asyncio + async def test_copy_folder_no_oversized_files( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size): + """Copy of a folder succeeds when no oversized files exist.""" + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_move_file_oversized( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Move raises InvalidParameters (413) when file metadata shows an oversized file.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + oversized_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=oversized_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'move', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + + @pytest.mark.asyncio + async def test_rename_skips_max_file_size_check( + self, http_request, mock_inter, patch_auth_handler_max_file_size): + """Rename action skips max_file_size check even when a large size is provided.""" + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = { + 'action': 'rename', + 'rename': 'new_name.dat', + 'path': '/test_path/', + } + + await handler.move_or_copy() + + handler.write.assert_called_once() + + +# --------------------------------------------------------------------------- +# Tests: quota checks +# --------------------------------------------------------------------------- + +class TestQuotaCheck: + + @pytest.mark.asyncio + async def test_copy_osfstorage_quota_ok_size_from_metadata( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_no_max_file_size): + """Copy to osfstorage succeeds when file size fits within quota.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(1_000) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + dest_provider.get_quota.assert_called_once_with() + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_copy_osfstorage_quota_exceeded_size_from_metadata( + self, http_request, mock_inter_osfstorage_quota_exceeded, + patch_auth_handler_no_max_file_size): + """Copy to osfstorage raises NotEnoughQuotaError when file size exceeds quota.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_exceeded + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(90_001) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await handler.move_or_copy() + + assert exc.value.data == {'message_key': 'quota_exceeded', 'message': 'You do not have enough available quota.'} + dest_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_copy_non_osfstorage_skips_quota_check( + self, http_request, mock_inter, patch_auth_handler_no_max_file_size): + """Copy to a non-osfstorage provider does not perform any quota check.""" + mock_make_provider, _ = mock_inter + src_provider = MockProvider() + dest_provider = MockProvider() + file_meta = MockFileMetadataWithSize(90_001) + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_rename_skips_quota_check( + self, http_request, mock_inter, patch_auth_handler_no_max_file_size): + """Rename action does not trigger quota check.""" + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = { + 'action': 'rename', + 'rename': 'new_name.dat', + 'path': '/test_path/', + } + + await handler.move_or_copy() + + handler.write.assert_called_once() + + @pytest.mark.asyncio + async def test_oversized_check_runs_before_quota_check( + self, http_request, mock_inter_osfstorage_quota_ok, patch_auth_handler_max_file_size): + """Oversized file check runs and fails before quota is ever requested.""" + mock_make_provider, dest_provider = mock_inter_osfstorage_quota_ok + src_provider = MockProvider() + file_meta = MockFileMetadataWithSize(2 * 1024 * 1024, name='bigfile.dat') + src_provider.metadata = MockCoroutine(return_value=file_meta) + mock_make_provider.side_effect = [src_provider, dest_provider] + + handler = mock_handler(http_request) + handler.path = '/test_file' + handler._json = {'action': 'copy', 'path': '/dest_path/'} + + with pytest.raises(exceptions.InvalidParameters) as exc: + await handler.move_or_copy() + + assert exc.value.code == 413 + dest_provider.get_quota.assert_not_called() + + @pytest.mark.asyncio + async def test_copy_folder_passes_check_kwargs_to_task( + self, http_request, mock_inter_folder, patch_auth_handler_max_file_size, monkeypatch): + """When copying a folder, move_or_copy passes max_size_bytes and check_quota to the background task.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + mock_make_provider, _ = mock_inter_folder + src_provider = MockProvider() + dest_provider = MockOsfStorageProvider() # NAME = 'osfstorage' + mock_make_provider.side_effect = [src_provider, dest_provider] + + mock_adelay = MockCoroutine(return_value='celery-task-id') + monkeypatch.setattr(movecopy_module.tasks.copy, 'adelay', mock_adelay) + + await handler.move_or_copy() + + mock_adelay.assert_called_once() + kwargs = mock_adelay.call_args[1] + assert kwargs['max_size_bytes'] == 1 * 1024 * 1024 + assert kwargs['check_quota'] is True + + @pytest.mark.asyncio + async def test_intra_folder_runs_pre_checks( + self, http_request, mock_intra, patch_auth_handler_max_file_size, monkeypatch): + """Intra-provider move/copy of a folder calls run_pre_checks.""" + import waterbutler.server.api.v1.provider.movecopy as movecopy_module + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(movecopy_module, 'run_pre_checks', mock_run_pre_checks) + + handler = mock_handler(http_request) + handler.path = '/test_folder/' + handler._json = {'action': 'copy', 'path': '/dest_folder/'} + + async def mock_backgrounded(coro): + res = await coro() + return res, True + monkeypatch.setattr(movecopy_module.tasks, 'backgrounded', mock_backgrounded) + + mock_make_provider, _ = mock_intra + src_provider = MockProvider() + src_provider.can_intra_copy = mock.Mock(return_value=True) + dest_provider = MockOsfStorageProvider() # NAME = 'osfstorage' + mock_make_provider.side_effect = [src_provider, dest_provider] + + await handler.move_or_copy() + + mock_run_pre_checks.assert_called_once_with( + src_provider, WaterButlerPath('/test_folder/'), dest_provider, + max_size_bytes=1 * 1024 * 1024, + check_quota=True + ) diff --git a/tests/tasks/test_pre_checks.py b/tests/tasks/test_pre_checks.py new file mode 100644 index 000000000..418eb87fc --- /dev/null +++ b/tests/tasks/test_pre_checks.py @@ -0,0 +1,386 @@ +# tests/tasks/test_pre_checks.py +import sys +import time +import pytest +from unittest import mock +import copy as cp + +# Import tasks first to populate sys.modules +from waterbutler import tasks + +# Resolve the actual modules to avoid package attribute shadowing issues +copy_module = sys.modules['waterbutler.tasks.copy'] +move_module = sys.modules['waterbutler.tasks.move'] + +from waterbutler.core import exceptions +from waterbutler.core.path import WaterButlerPath +from waterbutler.tasks.pre_checks import run_pre_checks +from tests.utils import MockCoroutine, MockFileMetadata, MockFolderMetadata, MockProvider + +# Retrieve the Celery tasks from the modules +copy_task = copy_module.copy +move_task = move_module.move + +# --------------------------------------------------------------------------- +# Custom Mock Metadata classes to allow custom sizes and kinds +# --------------------------------------------------------------------------- + +class MockFileMetadataWithSize(MockFileMetadata): + def __init__(self, size, name='Foo.name', kind='file', path='/Foo.name'): + super().__init__() + self._size = size + self._name = name + self._kind = kind + self._path = path + + @property + def size(self): + return self._size + + @property + def name(self): + return self._name + + @property + def kind(self): + return self._kind + + @property + def path(self): + return self._path + + +class MockFolderMetadataWithName(MockFolderMetadata): + def __init__(self, name='Bar', path='/Bar/'): + super().__init__() + self._name = name + self._path = path + + @property + def name(self): + return self._name + + @property + def path(self): + return self._path + + @property + def kind(self): + return 'folder' + + +# --------------------------------------------------------------------------- +# Task Integration Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def src_provider(): + p = MockProvider() + p.copy.return_value = (MockFileMetadata(), True) + p.auth['callback_url'] = 'src_callback' + return p + + +@pytest.fixture +def dest_provider(): + p = MockProvider() + p.copy.return_value = (MockFileMetadata(), True) + p.auth['callback_url'] = 'dest_callback' + return p + + +@pytest.fixture +def providers(monkeypatch, src_provider, dest_provider): + """Mock make_provider to return our mock source and destination providers.""" + def make_provider(name=None, **kwargs): + if name == 'src': + return src_provider + if name == 'dest': + return dest_provider + raise ValueError('Unexpected provider: {}'.format(name)) + monkeypatch.setattr(copy_module.utils, 'make_provider', make_provider) + monkeypatch.setattr(move_module.utils, 'make_provider', make_provider) + return src_provider, dest_provider + + +# --------------------------------------------------------------------------- +# Pre-checks Unit Tests +# --------------------------------------------------------------------------- + +class TestPreChecks: + + @pytest.mark.asyncio + async def test_file_pre_checks_no_checks(self, monkeypatch): + """Pre-checks should return early when no checks (max size or quota) are enabled.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + await run_pre_checks(src_provider, src_path, dest_provider) + src_provider.metadata.assert_not_called() + + @pytest.mark.asyncio + async def test_file_pre_checks_max_size_ok(self, monkeypatch): + """Pre-checks should succeed if the file size is within limits.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(100, name='file.txt') + src_provider.metadata = MockCoroutine(return_value=file_meta) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=200) + src_provider.metadata.assert_called_once_with(src_path, version=None, revision=None) + + @pytest.mark.asyncio + async def test_file_pre_checks_max_size_oversized(self, monkeypatch): + """Pre-checks should raise InvalidParameters (413) if the file exceeds max size.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(300, name='bigfile.txt') + src_provider.metadata = MockCoroutine(return_value=file_meta) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=200) + + assert exc.value.code == 413 + assert exc.value.data['message'] == 'Move/Copy Failed due to oversized files.' + assert exc.value.data['oversized_files'] == [{'name': 'bigfile.txt', 'size': 300}] + + @pytest.mark.asyncio + async def test_file_pre_checks_quota_ok(self, monkeypatch): + """Pre-checks should succeed if the file fits in the destination quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(100) + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + dest_provider.get_quota.assert_called_once_with() + + @pytest.mark.asyncio + async def test_file_pre_checks_quota_exceeded(self, monkeypatch): + """Pre-checks should raise NotEnoughQuotaError if the file exceeds quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/file.txt', prepend=None) + dest_provider = MockProvider() + + file_meta = MockFileMetadataWithSize(600) + src_provider.metadata = MockCoroutine(return_value=file_meta) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + assert exc.value.data['message_key'] == 'quota_exceeded' + + @pytest.mark.asyncio + async def test_fetch_all_pages_pagination(self, monkeypatch): + """Pre-checks should exhaust all pages when fetching paginated metadata.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + # Page 1 contains a file and a pagination token. Page 2 contains a file. + page1 = [MockFileMetadataWithSize(100, name='file1.txt'), 'page2_token'] + page2 = [MockFileMetadataWithSize(200, name='file2.txt')] + + src_provider.metadata = MockCoroutine(side_effect=[page1, page2]) + src_provider.handle_data = mock.Mock(return_value=([page1[0]], 'page2_token')) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=500) + + assert src_provider.metadata.call_count == 2 + src_provider.metadata.assert_has_calls([ + mock.call(src_path, version=None, revision=None, next_token=None), + mock.call(src_path, version=None, revision=None, next_token='page2_token') + ]) + src_provider.handle_data.assert_called_once_with(page1) + + @pytest.mark.asyncio + async def test_folder_pre_checks_max_size_ok(self, monkeypatch): + """Pre-checks should succeed if all files in the folder are within size limits.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(50, name='file1.txt') + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(80, name='file2.txt') + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100) + + assert src_provider.metadata.call_count == 2 + src_provider.validate_path.assert_called_once_with('/folder/subfolder/') + + @pytest.mark.asyncio + async def test_folder_pre_checks_max_size_oversized(self, monkeypatch): + """Pre-checks should raise InvalidParameters listing all oversized files inside folder sorted properly.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file_ok = MockFileMetadataWithSize(50, name='file_ok.txt') + bigfile1 = MockFileMetadataWithSize(150, name='bigfile1.txt') + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + bigfile2 = MockFileMetadataWithSize(180, name='bigfile2.txt') + + src_provider.metadata = MockCoroutine(side_effect=[ + [file_ok, bigfile1, subfolder], + [bigfile2] + ]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + + with pytest.raises(exceptions.InvalidParameters) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, max_size_bytes=100) + + assert exc.value.code == 413 + # subfolder is sorted first (kind=='folder'), then bigfile1.txt + assert exc.value.data['oversized_files'] == [ + {'name': 'bigfile2.txt', 'size': 180}, + {'name': 'bigfile1.txt', 'size': 150} + ] + + @pytest.mark.asyncio + async def test_folder_pre_checks_quota_ok(self, monkeypatch): + """Pre-checks should succeed if folder's recursive size fits within destination quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(100) + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(150) + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + dest_provider.get_quota = MockCoroutine(return_value={'used': 500, 'max': 1000}) + + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + @pytest.mark.asyncio + async def test_folder_pre_checks_quota_exceeded(self, monkeypatch): + """Pre-checks should raise NotEnoughQuotaError if folder's recursive size exceeds quota.""" + monkeypatch.setattr(time, 'sleep', lambda sec: None) + src_provider = MockProvider() + src_path = WaterButlerPath('/folder/', prepend=None) + dest_provider = MockProvider() + + file1 = MockFileMetadataWithSize(200) + subfolder = MockFolderMetadataWithName(name='subfolder', path='/folder/subfolder/') + file2 = MockFileMetadataWithSize(250) + + src_provider.metadata = MockCoroutine(side_effect=[[file1, subfolder], [file2]]) + src_provider.validate_path = MockCoroutine(return_value=WaterButlerPath('/folder/subfolder/', prepend=None)) + dest_provider.get_quota = MockCoroutine(return_value={'used': 600, 'max': 1000}) + + with pytest.raises(exceptions.NotEnoughQuotaError) as exc: + await run_pre_checks(src_provider, src_path, dest_provider, check_quota=True) + + assert exc.value.data['message_key'] == 'quota_exceeded' + + +# --------------------------------------------------------------------------- +# Celery Task Integration Tests +# --------------------------------------------------------------------------- + +class TestPreChecksTaskIntegration: + + def test_copy_task_calls_pre_checks(self, monkeypatch, providers, bundles, callback): + """Copy task should execute pre-checks before triggering copy.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(copy_module, 'run_pre_checks', mock_run_pre_checks) + + copy_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + mock_run_pre_checks.assert_called_once_with( + src, src_bundle['path'], dest, + max_size_bytes=1000, + check_quota=True + ) + assert src.copy.called + + def test_move_task_calls_pre_checks(self, monkeypatch, providers, bundles, callback): + """Move task should execute pre-checks before triggering move.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine() + monkeypatch.setattr(move_module, 'run_pre_checks', mock_run_pre_checks) + + src.move.return_value = (MockFileMetadata(), True) + + move_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + mock_run_pre_checks.assert_called_once_with( + src, src_bundle['path'], dest, + max_size_bytes=1000, + check_quota=True + ) + assert src.move.called + + def test_copy_task_pre_checks_failure_aborts_copy(self, monkeypatch, providers, bundles, callback): + """Copy task should abort and raise if pre-checks raise InvalidParameters.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine(side_effect=exceptions.InvalidParameters('Oversized files', code=413)) + monkeypatch.setattr(copy_module, 'run_pre_checks', mock_run_pre_checks) + + with pytest.raises(exceptions.InvalidParameters): + copy_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + assert not src.copy.called + + def test_move_task_pre_checks_failure_aborts_move(self, monkeypatch, providers, bundles, callback): + """Move task should abort and raise if pre-checks raise NotEnoughQuotaError.""" + src, dest = providers + src_bundle, dest_bundle = bundles + + mock_run_pre_checks = MockCoroutine(side_effect=exceptions.NotEnoughQuotaError('Quota exceeded')) + monkeypatch.setattr(move_module, 'run_pre_checks', mock_run_pre_checks) + + with pytest.raises(exceptions.NotEnoughQuotaError): + move_task( + cp.deepcopy(src_bundle), + cp.deepcopy(dest_bundle), + max_size_bytes=1000, + check_quota=True + ) + + assert not src.move.called diff --git a/waterbutler/providers/osfstorage/provider.py b/waterbutler/providers/osfstorage/provider.py index 8606a202f..88d0760f6 100644 --- a/waterbutler/providers/osfstorage/provider.py +++ b/waterbutler/providers/osfstorage/provider.py @@ -478,8 +478,16 @@ async def _do_intra_move_or_copy(self, action: str, dest_provider, src_path, des """ created = True + replaced_size = 0 if dest_path.identifier: created = False + try: + dest_meta = await dest_provider.metadata(dest_path) + if hasattr(dest_meta, 'size') and dest_meta.size: + replaced_size = int(dest_meta.size) + except Exception as e: + logger.error('Failed to fetch dest_meta for replaced_size calculation: %s', e) + raise exceptions.ProviderError({'message': 'Failed to fetch dest_meta for replaced_size calculation.'}, code=500) await dest_provider.delete(dest_path) resp = await self.make_signed_request( @@ -492,7 +500,8 @@ async def _do_intra_move_or_copy(self, action: str, dest_provider, src_path, des 'name': dest_path.name, 'node': dest_provider.nid, 'parent': dest_path.parent.identifier - } + }, + 'replaced_size': replaced_size, }), headers={'Content-Type': 'application/json'}, expects=(200, 201) diff --git a/waterbutler/server/api/v1/provider/movecopy.py b/waterbutler/server/api/v1/provider/movecopy.py index 8cfd7a638..4bdb89310 100644 --- a/waterbutler/server/api/v1/provider/movecopy.py +++ b/waterbutler/server/api/v1/provider/movecopy.py @@ -12,6 +12,7 @@ from waterbutler.constants import DEFAULT_CONFLICT from waterbutler.auth.osf.handler import EXPORT_DATA_FAKE_NODE_ID from waterbutler.tasks.settings import SYNCHRONOUS_TIMEOUT +from waterbutler.tasks.pre_checks import run_pre_checks auth_handler = AuthHandler(settings.AUTH_HANDLERS) @@ -101,7 +102,7 @@ async def move_or_copy(self): self.auth['settings'] ) self.path = await self.provider.validate_v1_path(self.path, **self.arguments) - + check_kwargs = {} if auth_action == 'rename': # 'rename' implies the file/folder does not change location self.dest_auth = self.auth self.dest_provider = self.provider @@ -145,6 +146,44 @@ async def move_or_copy(self): ) self.dest_path = await self.dest_provider.validate_path(**self.json) + # Check if the file/folder is oversized + max_size_mb = self.dest_auth['settings'].get('max_file_size') + max_size_bytes = (int(max_size_mb) * 1024 * 1024) if max_size_mb else None + + if not self.path.is_dir: + # Single-file path: read metadata once and check inline. + # No recursion needed — the item is guaranteed to be a file. + file_meta = await self.provider.metadata( + self.path, version=None, revision=None + ) + file_size = int(file_meta.size) + + # Check max_file_size + if max_size_bytes and file_size > max_size_bytes: + raise exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': [{'name': file_meta.name, 'size': file_size}], + 'max_size': max_size_bytes, + }, code=413) + + # Check quota (osfstorage only) + if self.dest_provider.NAME == 'osfstorage': + quota = await self.dest_provider.get_quota() + if quota['used'] + file_size > quota['max']: + raise exceptions.NotEnoughQuotaError({ + 'message_key': 'quota_exceeded', + 'message': 'You do not have enough available quota.', + }) + check_kwargs = { + 'max_size_bytes': None, + 'check_quota': False, + } + else: + check_kwargs = { + 'max_size_bytes': max_size_bytes, + 'check_quota': (self.dest_provider.NAME == 'osfstorage'), + } + if not getattr(self.provider, 'can_intra_' + provider_action)(self.dest_provider, self.path): # this weird signature syntax courtesy of py3.4 not liking trailing commas on kwargs conflict = self.json.get('conflict', DEFAULT_CONFLICT) @@ -158,6 +197,7 @@ async def move_or_copy(self): request=remote_logging._serialize_request(self.request), *self.build_args(), **task_kwargs, + **check_kwargs, ) synchronous = self.json.get('synchronous', 'false') synchronous = True if isinstance(synchronous, bool) and synchronous is True else False @@ -168,16 +208,21 @@ async def move_or_copy(self): # Use default timeout value for asynchronous processes metadata, created = await tasks.wait_on_celery(result) else: - metadata, created = ( - await tasks.backgrounded( - getattr(self.provider, provider_action), + async def _intra_task(): + if self.path.is_dir: + await run_pre_checks( + self.provider, self.path, self.dest_provider, + **check_kwargs + ) + return await getattr(self.provider, provider_action)( self.dest_provider, self.path, self.dest_path, rename=self.json.get('rename'), conflict=self.json.get('conflict', DEFAULT_CONFLICT), ) - ) + + metadata, created = await tasks.backgrounded(_intra_task) self.dest_meta = metadata diff --git a/waterbutler/tasks/copy.py b/waterbutler/tasks/copy.py index 878f14aff..2ee5503d7 100644 --- a/waterbutler/tasks/copy.py +++ b/waterbutler/tasks/copy.py @@ -5,12 +5,14 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.core import utils, remote_logging from waterbutler.core.log_payload import LogPayload +from waterbutler.tasks.pre_checks import run_pre_checks logger = logging.getLogger(__name__) @core.celery_task -async def copy(src_bundle, dest_bundle, request=None, start_time=None, **kwargs): +async def copy(src_bundle, dest_bundle, request=None, start_time=None, + max_size_bytes=None, check_quota=False, **kwargs): request = request or {} start_time = start_time or time.time() @@ -26,6 +28,9 @@ async def copy(src_bundle, dest_bundle, request=None, start_time=None, **kwargs) metadata, errors = None, [] try: + # Run pre-checks before attempting the copy to avoid partial copies and ensure we can report all errors at once + await run_pre_checks(src_provider, src_path, dest_provider, + max_size_bytes=max_size_bytes, check_quota=check_quota) metadata, created = await src_provider.copy(dest_provider, src_path, dest_path, **kwargs) except Exception as e: logger.error('Copy failed with error {!r}'.format(e)) diff --git a/waterbutler/tasks/move.py b/waterbutler/tasks/move.py index d25d96ab2..5a8e6c8a7 100644 --- a/waterbutler/tasks/move.py +++ b/waterbutler/tasks/move.py @@ -5,12 +5,14 @@ from waterbutler.core.path import WaterButlerPath from waterbutler.core import utils, remote_logging from waterbutler.core.log_payload import LogPayload +from waterbutler.tasks.pre_checks import run_pre_checks logger = logging.getLogger(__name__) @core.celery_task -async def move(src_bundle, dest_bundle, request=None, start_time=None, **kwargs): +async def move(src_bundle, dest_bundle, request=None, start_time=None, + max_size_bytes=None, check_quota=False, **kwargs): request = request or {} start_time = start_time or time.time() @@ -26,6 +28,9 @@ async def move(src_bundle, dest_bundle, request=None, start_time=None, **kwargs) metadata, errors = None, [] try: + # Run pre-checks before attempting the move to avoid partial moves and ensure we can report all errors at once + await run_pre_checks(src_provider, src_path, dest_provider, + max_size_bytes=max_size_bytes, check_quota=check_quota) metadata, created = await src_provider.move(dest_provider, src_path, dest_path, **kwargs) except Exception as e: logger.error('Move failed with error {!r}'.format(e)) diff --git a/waterbutler/tasks/pre_checks.py b/waterbutler/tasks/pre_checks.py new file mode 100644 index 000000000..b6504ecd6 --- /dev/null +++ b/waterbutler/tasks/pre_checks.py @@ -0,0 +1,81 @@ +from waterbutler.core import exceptions + + +async def _fetch_all_pages(provider, path): + """Fetch all children with pagination support.""" + all_data = [] + next_token = None + while True: + data = await provider.metadata(path, version=None, revision=None, next_token=next_token) + if data and isinstance(data[-1], str): + data, next_token = provider.handle_data(data) + else: + next_token = None + all_data.extend(data) + if not next_token: + break + return all_data + + +async def _get_total_size(provider, data): + """Recursively calculate total size of all files.""" + size = 0 + for item in data: + if item.kind == 'file': + size += int(item.size) + else: + child_path = await provider.validate_path(item.path) + children = await _fetch_all_pages(provider, child_path) + size += await _get_total_size(provider, children) + return size + + +async def _get_oversized_files(provider, data, max_size_bytes): + """Recursively find files exceeding max_size_bytes.""" + oversized = [] + for item in sorted(data, key=lambda i: (0 if i.kind == 'folder' else 1, i.name.lower())): + if item.kind == 'file': + if int(item.size) > max_size_bytes: + oversized.append({'name': item.name, 'size': int(item.size)}) + else: + child_path = await provider.validate_path(item.path) + children = await _fetch_all_pages(provider, child_path) + oversized.extend(await _get_oversized_files(provider, children, max_size_bytes)) + return oversized + + +async def run_pre_checks(src_provider, src_path, dest_provider, + max_size_bytes=None, check_quota=False): + """ + Run max_file_size and quota pre-checks inside the Celery task. + Raises InvalidParameters (413) or NotEnoughQuotaError if checks fail. + """ + # Only fetch data once, reuse for both checks + needs_check = max_size_bytes is not None or check_quota + if not needs_check: + return + + if src_path.is_dir: + data = await _fetch_all_pages(src_provider, src_path) + else: + data = [await src_provider.metadata(src_path, version=None, revision=None)] + + # Check 1: max file size + if max_size_bytes is not None: + oversized = await _get_oversized_files(src_provider, data, max_size_bytes) + if oversized: + raise exceptions.InvalidParameters({ + 'message': 'Move/Copy Failed due to oversized files.', + 'oversized_files': oversized, + 'max_size': max_size_bytes, + }, code=413) + + # Check 2: quota + if check_quota: + file_size = await _get_total_size(src_provider, data) + quota = await dest_provider.get_quota() + if quota['used'] + file_size > quota['max']: + raise exceptions.NotEnoughQuotaError({ + 'message_key': 'quota_exceeded', + 'message': 'You do not have enough available quota.', + })