[SVCS-826] S3 Compatablilty

requirements.txt

@@ -1,6 +1,7 @@
 agent==0.1.2
 aiohttp==0.18.2
 git+https://github.com/felliott/boto.git@feature/gen-url-query-params-6#egg=boto
+boto3==1.7.36
 celery==3.1.17
 furl==0.4.2
 google-auth==1.4.1

waterbutler/core/streams/http.py

@@ -179,6 +179,7 @@ def __init__(self, request, inner):
         super().__init__()
         self.inner = inner
         self.request = request
+        self.offset = 0
 
     @property
     def size(self):
@@ -187,12 +188,16 @@ def size(self):
     def at_eof(self):
         return self.inner.at_eof()
 
+    def tell(self):
+        return self.offset
+
     async def _read(self, size):
         if self.inner.at_eof():
             return b''
         if size < 0:
             return (await self.inner.read(size))
         try:
+            self.offset += size
             return (await self.inner.readexactly(size))
         except asyncio.IncompleteReadError as e:
             return e.partial

waterbutler/core/utils.py

@@ -105,15 +105,17 @@ async def send_signed_request(method, url, payload):
     ))
 
 
-def normalize_datetime(date_string):
-    if date_string is None:
+def normalize_datetime(date):
+    if date is None:
         return None
-    parsed_datetime = dateutil.parser.parse(date_string)
-    if not parsed_datetime.tzinfo:
-        parsed_datetime = parsed_datetime.replace(tzinfo=pytz.UTC)
-    parsed_datetime = parsed_datetime.astimezone(tz=pytz.UTC)
-    parsed_datetime = parsed_datetime.replace(microsecond=0)
-    return parsed_datetime.isoformat()
+    if isinstance(date, str):
+        date = dateutil.parser.parse(date)
+    if not date.tzinfo:
+        date = date.replace(tzinfo=pytz.UTC)
+    date = date.astimezone(tz=pytz.UTC)
+    date = date.replace(microsecond=0)
+    return date.isoformat()
+
 
 
 class ZipStreamGenerator:

waterbutler/providers/s3/metadata.py

@@ -1,6 +1,7 @@
 import os
 
 from waterbutler.core import metadata
+from waterbutler.core import utils
 
 
 class S3Metadata(metadata.BaseMetadata):
@@ -18,42 +19,47 @@ class S3FileMetadataHeaders(S3Metadata, metadata.BaseFileMetadata):
 
     def __init__(self, path, headers):
         self._path = path
+        self.obj = headers
+        self._etag = None
         # Cast to dict to clone as the headers will
         # be destroyed when the request leaves scope
-        super().__init__(dict(headers))
+        super().__init__(headers)
 
     @property
     def path(self):
         return '/' + self._path
 
     @property
     def size(self):
-        return self.raw['CONTENT-LENGTH']
+        return self.obj.content_length
 
     @property
     def content_type(self):
-        return self.raw['CONTENT-TYPE']
+        return self.obj.content_type
 
     @property
     def modified(self):
-        return self.raw['LAST-MODIFIED']
+        return utils.normalize_datetime(self.obj.last_modified)
 
     @property
     def created_utc(self):
         return None
 
     @property
     def etag(self):
-        return self.raw['ETAG'].replace('"', '')
+        if self._etag:
+            return self._etag
+        else:
+            self._etag = self.obj.e_tag.replace('"', '')
+            return self._etag
 
     @property
     def extra(self):
-        md5 = self.raw['ETAG'].replace('"', '')
         return {
-            'md5': md5,
-            'encryption': self.raw.get('X-AMZ-SERVER-SIDE-ENCRYPTION', ''),
+            'md5': self.etag,
+            'encryption': self.obj.server_side_encryption,
             'hashes': {
-                'md5': md5,
+                'md5': self.etag,
             },
         }
 
@@ -70,7 +76,7 @@ def size(self):
 
     @property
     def modified(self):
-        return self.raw['LastModified']
+        return self.raw['LastModified'].isoformat()
 
     @property
     def created_utc(self):