diff --git a/tests/providers/s3compatsigv4/test_provider.py b/tests/providers/s3compatsigv4/test_provider.py index bd21e0752..ef7b3f284 100644 --- a/tests/providers/s3compatsigv4/test_provider.py +++ b/tests/providers/s3compatsigv4/test_provider.py @@ -8,6 +8,7 @@ import datetime import aiohttpretty from http import client +from http import HTTPStatus from urllib import parse from unittest import mock @@ -1188,15 +1189,191 @@ async def test_chunked_upload_aborted_success(self, provider, upload_parts_heade with pytest.raises(exceptions.UploadError) as exc: await provider._chunked_upload(file_stream, path) + # The abort has SUCCEEDED (return value True), so the "manual clean-up" + # warning must NOT be appended to the error message. msg = 'An unexpected error has occurred during the multi-part upload.' - msg += ' The abort action failed to clean up the temporary file parts generated ' \ - 'during the upload process. Please manually remove them.' assert str(exc.value) == ', '.join(['500', msg]) provider._create_upload_session.assert_called_with(path) provider._upload_parts.assert_called_with(file_stream, path, upload_id) provider._abort_chunked_upload.assert_called_with(path, upload_id) + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload_abort_failure_appends_warning(self, provider, file_stream, + mock_time): + assert file_stream.size == 6 + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 5 + provider.CHUNK_SIZE = 2 + + path = WaterButlerPath('/foobah', prepend=provider.prefix) + upload_id = 'EXAMPLEJZ6e0YupT2h66iePQCc9IEbYbDUy4RTpMeoSMLPRp8Z5o1u' \ + '8feSRonpvnWsKKG35tI2LB9VDPiCgTy.Gq2VxQLYjrue4Nq.NBdqI-' + + provider._create_upload_session = MockCoroutine() + provider._create_upload_session.return_value = upload_id + provider._upload_parts = MockCoroutine() + provider._upload_parts.side_effect = Exception('error') + provider._abort_chunked_upload = MockCoroutine() + provider._abort_chunked_upload.return_value = False + + with pytest.raises(exceptions.UploadError) as exc: + await provider._chunked_upload(file_stream, path) + # The abort has FAILED (return value False), so the "manual clean-up" + # warning must be appended to the error message. + msg = 'An unexpected error has occurred during the multi-part upload.' + msg += ' The abort action failed to clean up the temporary file parts generated ' \ + 'during the upload process. Please manually remove them.' + assert str(exc.value) == ', '.join(['500', msg]) + + provider._abort_chunked_upload.assert_called_with(path, upload_id) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_chunked_upload_storage_quota_exceeded(self, provider, file_stream, mock_time): + assert file_stream.size == 6 + provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 5 + provider.CHUNK_SIZE = 2 + + path = WaterButlerPath('/foobah', prepend=provider.prefix) + upload_id = 'EXAMPLEJZ6e0YupT2h66iePQCc9IEbYbDUy4RTpMeoSMLPRp8Z5o1u' \ + '8feSRonpvnWsKKG35tI2LB9VDPiCgTy.Gq2VxQLYjrue4Nq.NBdqI-' + error_xml = ('' + 'QuotaExceeded' + 'The bucket quota has been exceeded') + + provider._create_upload_session = MockCoroutine() + provider._create_upload_session.return_value = upload_id + provider._upload_parts = MockCoroutine() + provider._upload_parts.side_effect = exceptions.UploadError({'response': error_xml}, + code=403) + provider._abort_chunked_upload = MockCoroutine() + provider._abort_chunked_upload.return_value = True + + with pytest.raises(exceptions.UploadError) as exc: + await provider._chunked_upload(file_stream, path) + + # A storage-side quota error must surface as HTTP 507 with an explicit, + # user-readable message, and the multipart session must be aborted. + assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE + assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message + assert 'QuotaExceeded' in exc.value.message + provider._abort_chunked_upload.assert_called_with(path, upload_id) + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_contiguous_upload_storage_quota_exceeded(self, provider, file_stream, + mock_time): + path = WaterButlerPath('/foobah', prepend=provider.prefix) + error_xml = ('' + 'QuotaExceeded' + 'The bucket quota has been exceeded') + + # ``make_request`` raises ``UploadError`` built by + # ``exception_from_response`` when the storage rejects the PUT. + provider.make_request = MockCoroutine() + provider.make_request.side_effect = exceptions.UploadError({'response': error_xml}, + code=403) + + with pytest.raises(exceptions.UploadError) as exc: + await provider._contiguous_upload(file_stream, path) + + # A storage-side quota error must surface as HTTP 507 with an explicit, + # user-readable message. + assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE + assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message + assert 'QuotaExceeded' in exc.value.message + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_contiguous_upload_other_storage_error(self, provider, file_stream, + mock_time): + path = WaterButlerPath('/foobah', prepend=provider.prefix) + error_xml = ('' + 'AccessDenied' + 'Access Denied') + + provider.make_request = MockCoroutine() + provider.make_request.side_effect = exceptions.UploadError({'response': error_xml}, + code=403) + + with pytest.raises(exceptions.UploadError) as exc: + await provider._contiguous_upload(file_stream, path) + + # Non-quota errors keep the storage's status code but get a readable + # message (not the raw XML body). + assert exc.value.code == 403 + assert 'AccessDenied' in exc.value.message + assert '' + 'QuotaExceeded' + 'The bucket quota has been exceeded') + + provider._create_upload_session = MockCoroutine() + provider._create_upload_session.side_effect = exceptions.UploadError( + {'response': error_xml}, code=403) + provider._abort_chunked_upload = MockCoroutine() + + with pytest.raises(exceptions.UploadError) as exc: + await provider._chunked_upload(file_stream, path) + + assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE + assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message + # No session was created, so nothing must be aborted. + provider._abort_chunked_upload.assert_not_called() + + @pytest.mark.asyncio + @pytest.mark.aiohttpretty + async def test_create_upload_session_invalid_response(self, provider, mock_time): + path = WaterButlerPath('/foobah', prepend=provider.prefix) + + resp = mock.Mock() + resp.read = MockCoroutine(return_value=b'this is not the expected xml') + provider.make_request = MockCoroutine(return_value=resp) + + with pytest.raises(exceptions.UploadError) as exc: + await provider._create_upload_session(path) + + # A malformed 200-range response must become a controlled error, not a + # raw ExpatError/KeyError propagating as HTTP 500. + assert exc.value.code == HTTPStatus.BAD_GATEWAY + assert 'unexpected response' in exc.value.message + @pytest.mark.asyncio @pytest.mark.aiohttpretty async def test_chunked_upload_limit_contiguous(self, provider, file_stream, mock_time): diff --git a/waterbutler/providers/s3compatsigv4/provider.py b/waterbutler/providers/s3compatsigv4/provider.py index 20ff65fd5..468d555ab 100644 --- a/waterbutler/providers/s3compatsigv4/provider.py +++ b/waterbutler/providers/s3compatsigv4/provider.py @@ -10,6 +10,7 @@ from io import BytesIO import base64 +import aiohttp import xmltodict import boto3 from botocore.config import Config @@ -92,6 +93,17 @@ def NAME(self): CHUNK_SIZE = settings.CHUNK_SIZE CONTIGUOUS_UPLOAD_SIZE_LIMIT = settings.CONTIGUOUS_UPLOAD_SIZE_LIMIT + QUOTA_EXCEEDED_MESSAGE = ( + 'Upload failed because the quota or capacity of the cloud storage has been exceeded. ' + 'Please free up storage space or contact the storage administrator.' + ) + CONNECTION_INTERRUPTED_MESSAGE = ( + 'Upload failed because the connection to the cloud storage was interrupted before the ' + 'upload completed. This may indicate that the storage is full or that its quota has ' + 'been exceeded. Please retry the upload, and contact the storage administrator if the ' + 'problem persists.' + ) + def __init__(self, auth, credentials, settings, **kwargs): """ :param dict auth: Not used @@ -307,6 +319,67 @@ async def _get_content_whole_size(self, path: WaterButlerPath, revision=None): raise exceptions.MetadataError('Cannot get content size and ETag') return size, etag + @staticmethod + def _parse_s3_error_body(err): + """Extract the S3 XML error ``Code`` and ``Message`` from an + :class:`waterbutler.core.exceptions.UploadError` raised by ``make_request``. + + ``exception_from_response`` stores an XML error body either as + ``err.data['response']`` (dict) or as ``err.message`` (str). + + :param err: ( :class:`.UploadError` ) The error raised by ``make_request`` + :rtype: tuple(str or None, str or None) + :return: ``(error_code, error_message)``, or ``(None, None)`` when the + response body is not a parsable S3 XML error + """ + body = None + data = getattr(err, 'data', None) + if isinstance(data, dict): + body = data.get('response') + if body is None: + body = getattr(err, 'message', None) + if not isinstance(body, str) or '