diff --git a/tests/providers/s3compatsigv4/test_provider.py b/tests/providers/s3compatsigv4/test_provider.py
index bd21e0752..ef7b3f284 100644
--- a/tests/providers/s3compatsigv4/test_provider.py
+++ b/tests/providers/s3compatsigv4/test_provider.py
@@ -8,6 +8,7 @@
import datetime
import aiohttpretty
from http import client
+from http import HTTPStatus
from urllib import parse
from unittest import mock
@@ -1188,15 +1189,191 @@ async def test_chunked_upload_aborted_success(self, provider, upload_parts_heade
with pytest.raises(exceptions.UploadError) as exc:
await provider._chunked_upload(file_stream, path)
+ # The abort has SUCCEEDED (return value True), so the "manual clean-up"
+ # warning must NOT be appended to the error message.
msg = 'An unexpected error has occurred during the multi-part upload.'
- msg += ' The abort action failed to clean up the temporary file parts generated ' \
- 'during the upload process. Please manually remove them.'
assert str(exc.value) == ', '.join(['500', msg])
provider._create_upload_session.assert_called_with(path)
provider._upload_parts.assert_called_with(file_stream, path, upload_id)
provider._abort_chunked_upload.assert_called_with(path, upload_id)
+ @pytest.mark.asyncio
+ @pytest.mark.aiohttpretty
+ async def test_chunked_upload_abort_failure_appends_warning(self, provider, file_stream,
+ mock_time):
+ assert file_stream.size == 6
+ provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 5
+ provider.CHUNK_SIZE = 2
+
+ path = WaterButlerPath('/foobah', prepend=provider.prefix)
+ upload_id = 'EXAMPLEJZ6e0YupT2h66iePQCc9IEbYbDUy4RTpMeoSMLPRp8Z5o1u' \
+ '8feSRonpvnWsKKG35tI2LB9VDPiCgTy.Gq2VxQLYjrue4Nq.NBdqI-'
+
+ provider._create_upload_session = MockCoroutine()
+ provider._create_upload_session.return_value = upload_id
+ provider._upload_parts = MockCoroutine()
+ provider._upload_parts.side_effect = Exception('error')
+ provider._abort_chunked_upload = MockCoroutine()
+ provider._abort_chunked_upload.return_value = False
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._chunked_upload(file_stream, path)
+ # The abort has FAILED (return value False), so the "manual clean-up"
+ # warning must be appended to the error message.
+ msg = 'An unexpected error has occurred during the multi-part upload.'
+ msg += ' The abort action failed to clean up the temporary file parts generated ' \
+ 'during the upload process. Please manually remove them.'
+ assert str(exc.value) == ', '.join(['500', msg])
+
+ provider._abort_chunked_upload.assert_called_with(path, upload_id)
+
+ @pytest.mark.asyncio
+ @pytest.mark.aiohttpretty
+ async def test_chunked_upload_storage_quota_exceeded(self, provider, file_stream, mock_time):
+ assert file_stream.size == 6
+ provider.CONTIGUOUS_UPLOAD_SIZE_LIMIT = 5
+ provider.CHUNK_SIZE = 2
+
+ path = WaterButlerPath('/foobah', prepend=provider.prefix)
+ upload_id = 'EXAMPLEJZ6e0YupT2h66iePQCc9IEbYbDUy4RTpMeoSMLPRp8Z5o1u' \
+ '8feSRonpvnWsKKG35tI2LB9VDPiCgTy.Gq2VxQLYjrue4Nq.NBdqI-'
+ error_xml = (''
+ 'QuotaExceeded'
+ 'The bucket quota has been exceeded')
+
+ provider._create_upload_session = MockCoroutine()
+ provider._create_upload_session.return_value = upload_id
+ provider._upload_parts = MockCoroutine()
+ provider._upload_parts.side_effect = exceptions.UploadError({'response': error_xml},
+ code=403)
+ provider._abort_chunked_upload = MockCoroutine()
+ provider._abort_chunked_upload.return_value = True
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._chunked_upload(file_stream, path)
+
+ # A storage-side quota error must surface as HTTP 507 with an explicit,
+ # user-readable message, and the multipart session must be aborted.
+ assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE
+ assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message
+ assert 'QuotaExceeded' in exc.value.message
+ provider._abort_chunked_upload.assert_called_with(path, upload_id)
+
+ @pytest.mark.asyncio
+ @pytest.mark.aiohttpretty
+ async def test_contiguous_upload_storage_quota_exceeded(self, provider, file_stream,
+ mock_time):
+ path = WaterButlerPath('/foobah', prepend=provider.prefix)
+ error_xml = (''
+ 'QuotaExceeded'
+ 'The bucket quota has been exceeded')
+
+ # ``make_request`` raises ``UploadError`` built by
+ # ``exception_from_response`` when the storage rejects the PUT.
+ provider.make_request = MockCoroutine()
+ provider.make_request.side_effect = exceptions.UploadError({'response': error_xml},
+ code=403)
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._contiguous_upload(file_stream, path)
+
+ # A storage-side quota error must surface as HTTP 507 with an explicit,
+ # user-readable message.
+ assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE
+ assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message
+ assert 'QuotaExceeded' in exc.value.message
+
+ @pytest.mark.asyncio
+ @pytest.mark.aiohttpretty
+ async def test_contiguous_upload_other_storage_error(self, provider, file_stream,
+ mock_time):
+ path = WaterButlerPath('/foobah', prepend=provider.prefix)
+ error_xml = (''
+ 'AccessDenied'
+ 'Access Denied')
+
+ provider.make_request = MockCoroutine()
+ provider.make_request.side_effect = exceptions.UploadError({'response': error_xml},
+ code=403)
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._contiguous_upload(file_stream, path)
+
+ # Non-quota errors keep the storage's status code but get a readable
+ # message (not the raw XML body).
+ assert exc.value.code == 403
+ assert 'AccessDenied' in exc.value.message
+ assert ''
+ 'QuotaExceeded'
+ 'The bucket quota has been exceeded')
+
+ provider._create_upload_session = MockCoroutine()
+ provider._create_upload_session.side_effect = exceptions.UploadError(
+ {'response': error_xml}, code=403)
+ provider._abort_chunked_upload = MockCoroutine()
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._chunked_upload(file_stream, path)
+
+ assert exc.value.code == HTTPStatus.INSUFFICIENT_STORAGE
+ assert provider.QUOTA_EXCEEDED_MESSAGE in exc.value.message
+ # No session was created, so nothing must be aborted.
+ provider._abort_chunked_upload.assert_not_called()
+
+ @pytest.mark.asyncio
+ @pytest.mark.aiohttpretty
+ async def test_create_upload_session_invalid_response(self, provider, mock_time):
+ path = WaterButlerPath('/foobah', prepend=provider.prefix)
+
+ resp = mock.Mock()
+ resp.read = MockCoroutine(return_value=b'this is not the expected xml')
+ provider.make_request = MockCoroutine(return_value=resp)
+
+ with pytest.raises(exceptions.UploadError) as exc:
+ await provider._create_upload_session(path)
+
+ # A malformed 200-range response must become a controlled error, not a
+ # raw ExpatError/KeyError propagating as HTTP 500.
+ assert exc.value.code == HTTPStatus.BAD_GATEWAY
+ assert 'unexpected response' in exc.value.message
+
@pytest.mark.asyncio
@pytest.mark.aiohttpretty
async def test_chunked_upload_limit_contiguous(self, provider, file_stream, mock_time):
diff --git a/waterbutler/providers/s3compatsigv4/provider.py b/waterbutler/providers/s3compatsigv4/provider.py
index 20ff65fd5..468d555ab 100644
--- a/waterbutler/providers/s3compatsigv4/provider.py
+++ b/waterbutler/providers/s3compatsigv4/provider.py
@@ -10,6 +10,7 @@
from io import BytesIO
import base64
+import aiohttp
import xmltodict
import boto3
from botocore.config import Config
@@ -92,6 +93,17 @@ def NAME(self):
CHUNK_SIZE = settings.CHUNK_SIZE
CONTIGUOUS_UPLOAD_SIZE_LIMIT = settings.CONTIGUOUS_UPLOAD_SIZE_LIMIT
+ QUOTA_EXCEEDED_MESSAGE = (
+ 'Upload failed because the quota or capacity of the cloud storage has been exceeded. '
+ 'Please free up storage space or contact the storage administrator.'
+ )
+ CONNECTION_INTERRUPTED_MESSAGE = (
+ 'Upload failed because the connection to the cloud storage was interrupted before the '
+ 'upload completed. This may indicate that the storage is full or that its quota has '
+ 'been exceeded. Please retry the upload, and contact the storage administrator if the '
+ 'problem persists.'
+ )
+
def __init__(self, auth, credentials, settings, **kwargs):
"""
:param dict auth: Not used
@@ -307,6 +319,67 @@ async def _get_content_whole_size(self, path: WaterButlerPath, revision=None):
raise exceptions.MetadataError('Cannot get content size and ETag')
return size, etag
+ @staticmethod
+ def _parse_s3_error_body(err):
+ """Extract the S3 XML error ``Code`` and ``Message`` from an
+ :class:`waterbutler.core.exceptions.UploadError` raised by ``make_request``.
+
+ ``exception_from_response`` stores an XML error body either as
+ ``err.data['response']`` (dict) or as ``err.message`` (str).
+
+ :param err: ( :class:`.UploadError` ) The error raised by ``make_request``
+ :rtype: tuple(str or None, str or None)
+ :return: ``(error_code, error_message)``, or ``(None, None)`` when the
+ response body is not a parsable S3 XML error
+ """
+ body = None
+ data = getattr(err, 'data', None)
+ if isinstance(data, dict):
+ body = data.get('response')
+ if body is None:
+ body = getattr(err, 'message', None)
+ if not isinstance(body, str) or '