From 26b0252355e8d97dfba375bfb188225d59195be3 Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Thu, 12 May 2022 12:05:08 +0100 Subject: [PATCH 1/7] CAmazonS3Client: switch to path-style endpoint, this is closer to how R2 works --- src/amazon/AmazonS3Client.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index 58ae621b..01f3bc78 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -21,9 +21,9 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati Request rq; rq.method = Framework::Http::HTTP_VERB::GET; - rq.host = string_format("%s." S3_HOSTNAME, request.bucket.c_str()); + rq.host = S3_HOSTNAME; rq.urlHost = S3_HOSTNAME; - rq.uri = "/"; + rq.uri = string_format("/%s", request.bucket.c_str()); //We add a bucket parameter even if the S3 API doesn't use it to prevent caching rq.query = string_format("bucket=%s&location=", request.bucket.c_str()); @@ -50,8 +50,8 @@ GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) { Request rq; rq.method = Framework::Http::HTTP_VERB::GET; - rq.uri = "/" + Framework::UrlEncode(request.key); - rq.host = string_format("%s.s3-%s.amazonaws.com", request.bucket.c_str(), m_region.c_str()); + rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); + rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); rq.urlHost = rq.host; if(request.range.first != request.range.second) @@ -79,8 +79,8 @@ HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) { Request rq; rq.method = Framework::Http::HTTP_VERB::HEAD; - rq.uri = "/" + Framework::UrlEncode(request.key); - rq.host = string_format("%s.s3-%s.amazonaws.com", request.bucket.c_str(), m_region.c_str()); + rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); + rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); rq.urlHost = rq.host; auto response = ExecuteRequest(rq); @@ -111,8 +111,8 @@ ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) { Request rq; rq.method = Framework::Http::HTTP_VERB::GET; - rq.uri = "/"; - rq.host = string_format("%s.s3-%s.amazonaws.com", bucket.c_str(), m_region.c_str()); + rq.uri = string_format("/%s", bucket.c_str()); + rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); rq.urlHost = rq.host; auto response = ExecuteRequest(rq); @@ -142,8 +142,8 @@ void CAmazonS3Client::PutObject(const PutObjectRequest& request) { Request rq; rq.method = Framework::Http::HTTP_VERB::PUT; - rq.uri = "/" + Framework::UrlEncode(request.key); - rq.host = string_format("%s.s3-%s.amazonaws.com", request.bucket.c_str(), m_region.c_str()); + rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); + rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); rq.urlHost = rq.host; rq.content = request.data; From 123688305f54f137cec80af801898b38863728fd Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Thu, 12 May 2022 15:40:03 +0100 Subject: [PATCH 2/7] S3: allow us to specify an endpoint --- include/amazon/AmazonS3Client.h | 5 ++++- src/amazon/AmazonS3Client.cpp | 17 ++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/include/amazon/AmazonS3Client.h b/include/amazon/AmazonS3Client.h index 112790b3..b2b5fc64 100644 --- a/include/amazon/AmazonS3Client.h +++ b/include/amazon/AmazonS3Client.h @@ -58,11 +58,14 @@ struct PutObjectRequest class CAmazonS3Client : public CAmazonClient { public: - CAmazonS3Client(CAmazonCredentials, std::string = "us-east-1"); + CAmazonS3Client(CAmazonCredentials, std::string = "us-east-1", std::string = "amazonaws.com"); GetBucketLocationResult GetBucketLocation(const GetBucketLocationRequest&); GetObjectResult GetObject(const GetObjectRequest&); HeadObjectResult HeadObject(const HeadObjectRequest&); ListObjectsResult ListObjects(std::string); void PutObject(const PutObjectRequest&); + +private: + std::string m_endpoint; }; diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index 01f3bc78..ede5f5c5 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -5,10 +5,9 @@ #include "xml/Parser.h" #include "Url.h" -#define S3_HOSTNAME "s3.amazonaws.com" - -CAmazonS3Client::CAmazonS3Client(CAmazonCredentials credentials, std::string region) +CAmazonS3Client::CAmazonS3Client(CAmazonCredentials credentials, std::string region, std::string endpoint) : CAmazonClient("s3", std::move(credentials), std::move(region)) + , m_endpoint(endpoint) { } @@ -21,8 +20,8 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati Request rq; rq.method = Framework::Http::HTTP_VERB::GET; - rq.host = S3_HOSTNAME; - rq.urlHost = S3_HOSTNAME; + rq.host = string_format("s3.%s", m_endpoint.c_str()); + rq.urlHost = rq.host; rq.uri = string_format("/%s", request.bucket.c_str()); //We add a bucket parameter even if the S3 API doesn't use it to prevent caching rq.query = string_format("bucket=%s&location=", request.bucket.c_str()); @@ -51,7 +50,7 @@ GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) Request rq; rq.method = Framework::Http::HTTP_VERB::GET; rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); + rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); rq.urlHost = rq.host; if(request.range.first != request.range.second) @@ -80,7 +79,7 @@ HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) Request rq; rq.method = Framework::Http::HTTP_VERB::HEAD; rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); + rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); rq.urlHost = rq.host; auto response = ExecuteRequest(rq); @@ -112,7 +111,7 @@ ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) Request rq; rq.method = Framework::Http::HTTP_VERB::GET; rq.uri = string_format("/%s", bucket.c_str()); - rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); + rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); rq.urlHost = rq.host; auto response = ExecuteRequest(rq); @@ -143,7 +142,7 @@ void CAmazonS3Client::PutObject(const PutObjectRequest& request) Request rq; rq.method = Framework::Http::HTTP_VERB::PUT; rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.amazonaws.com", m_region.c_str()); + rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); rq.urlHost = rq.host; rq.content = request.data; From 7ec7766f2a2f1a4a63427b41eaa6bc23a32b0ab8 Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Thu, 12 May 2022 15:47:28 +0100 Subject: [PATCH 3/7] rename CAmazonCredentials to CAmazonConfigs and add more options --- include/amazon/AmazonClient.h | 14 +++++++++++--- include/amazon/AmazonS3Client.h | 2 +- src/amazon/AmazonClient.cpp | 16 ++++++++-------- src/amazon/AmazonS3Client.cpp | 4 ++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/include/amazon/AmazonClient.h b/include/amazon/AmazonClient.h index c894d1df..0ac2dc66 100644 --- a/include/amazon/AmazonClient.h +++ b/include/amazon/AmazonClient.h @@ -4,11 +4,19 @@ #include #include "http/HttpClient.h" -struct CAmazonCredentials +struct CAmazonConfigs { + enum S3PROVIDER + { + AWS_S3 = 0, + CF_R2 = 1 + }; + std::string accessKeyId; std::string secretAccessKey; std::string sessionToken; + std::string accountKeyId; + S3PROVIDER m_provider; bool IsValid() const { @@ -19,7 +27,7 @@ struct CAmazonCredentials class CAmazonClient { public: - CAmazonClient(std::string, CAmazonCredentials, std::string); + CAmazonClient(std::string, CAmazonConfigs, std::string); protected: struct Request @@ -36,6 +44,6 @@ class CAmazonClient Framework::Http::RequestResult ExecuteRequest(const Request&); std::string m_service; - CAmazonCredentials m_credentials; + CAmazonConfigs m_configs; std::string m_region; }; diff --git a/include/amazon/AmazonS3Client.h b/include/amazon/AmazonS3Client.h index b2b5fc64..b47d44d0 100644 --- a/include/amazon/AmazonS3Client.h +++ b/include/amazon/AmazonS3Client.h @@ -58,7 +58,7 @@ struct PutObjectRequest class CAmazonS3Client : public CAmazonClient { public: - CAmazonS3Client(CAmazonCredentials, std::string = "us-east-1", std::string = "amazonaws.com"); + CAmazonS3Client(CAmazonConfigs, std::string = "us-east-1", std::string = "amazonaws.com"); GetBucketLocationResult GetBucketLocation(const GetBucketLocationRequest&); GetObjectResult GetObject(const GetObjectRequest&); diff --git a/src/amazon/AmazonClient.cpp b/src/amazon/AmazonClient.cpp index 895c39ce..91f27c9c 100644 --- a/src/amazon/AmazonClient.cpp +++ b/src/amazon/AmazonClient.cpp @@ -137,17 +137,17 @@ static std::string timeToString(const tm* timeInfo) return std::string(output); } -CAmazonClient::CAmazonClient(std::string service, CAmazonCredentials credentials, std::string region) +CAmazonClient::CAmazonClient(std::string service, CAmazonConfigs configs, std::string region) : m_service(std::move(service)) - , m_credentials(std::move(credentials)) + , m_configs(std::move(configs)) , m_region(std::move(region)) { } Framework::Http::RequestResult CAmazonClient::ExecuteRequest(const Request& request) { - assert(!m_credentials.accessKeyId.empty()); - assert(!m_credentials.secretAccessKey.empty()); + assert(!m_configs.accessKeyId.empty()); + assert(!m_configs.secretAccessKey.empty()); assert(!request.host.empty()); assert(!request.urlHost.empty()); @@ -167,9 +167,9 @@ Framework::Http::RequestResult CAmazonClient::ExecuteRequest(const Request& requ headers.insert(std::make_pair("Host", request.host)); headers.insert(std::make_pair("x-amz-content-sha256", contentHashString)); headers.insert(std::make_pair("x-amz-date", timestamp)); - if(!m_credentials.sessionToken.empty()) + if(!m_configs.sessionToken.empty()) { - headers.insert(std::make_pair("x-amz-security-token", m_credentials.sessionToken)); + headers.insert(std::make_pair("x-amz-security-token", m_configs.sessionToken)); } auto canonicalRequest = buildCanonicalRequest(request.method, request.uri, request.query, contentHashString, headers); @@ -183,11 +183,11 @@ Framework::Http::RequestResult CAmazonClient::ExecuteRequest(const Request& requ #endif auto signedHeaders = buildSignedHeadersParam(headers); - auto signingKey = buildSigningKey(m_credentials.secretAccessKey, date, m_region, m_service, requestType); + auto signingKey = buildSigningKey(m_configs.secretAccessKey, date, m_region, m_service, requestType); auto signature = hashToString(Framework::HashUtils::ComputeHmacSha256(signingKey.data(), signingKey.size(), stringToSign.c_str(), stringToSign.length())); auto authorizationString = string_format("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", - m_credentials.accessKeyId.c_str(), scope.c_str(), signedHeaders.c_str(), signature.c_str()); + m_configs.accessKeyId.c_str(), scope.c_str(), signedHeaders.c_str(), signature.c_str()); headers.insert(std::make_pair("Authorization", authorizationString)); headers.insert(request.headers.begin(), request.headers.end()); diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index ede5f5c5..b6345116 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -5,8 +5,8 @@ #include "xml/Parser.h" #include "Url.h" -CAmazonS3Client::CAmazonS3Client(CAmazonCredentials credentials, std::string region, std::string endpoint) - : CAmazonClient("s3", std::move(credentials), std::move(region)) +CAmazonS3Client::CAmazonS3Client(CAmazonConfigs configs, std::string region, std::string endpoint) + : CAmazonClient("s3", std::move(configs), std::move(region)) , m_endpoint(endpoint) { } From 38130062e8ec99b0a9429231ae5c6e4494304bc3 Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Thu, 12 May 2022 18:52:25 +0100 Subject: [PATCH 4/7] Abstract Request creation, makes it easier to isolate the logical differene between R2/S3 --- include/amazon/AmazonS3Client.h | 4 +- src/amazon/AmazonS3Client.cpp | 82 +++++++++++++++++++++------------ 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/include/amazon/AmazonS3Client.h b/include/amazon/AmazonS3Client.h index b47d44d0..75bf6fac 100644 --- a/include/amazon/AmazonS3Client.h +++ b/include/amazon/AmazonS3Client.h @@ -58,7 +58,7 @@ struct PutObjectRequest class CAmazonS3Client : public CAmazonClient { public: - CAmazonS3Client(CAmazonConfigs, std::string = "us-east-1", std::string = "amazonaws.com"); + CAmazonS3Client(CAmazonConfigs, std::string = "us-east-1"); GetBucketLocationResult GetBucketLocation(const GetBucketLocationRequest&); GetObjectResult GetObject(const GetObjectRequest&); @@ -67,5 +67,5 @@ class CAmazonS3Client : public CAmazonClient void PutObject(const PutObjectRequest&); private: - std::string m_endpoint; + Request CreateRequest(Framework::Http::HTTP_VERB, std::string, std::string = ""); }; diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index b6345116..c1e36209 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -1,28 +1,70 @@ #include "amazon/AmazonS3Client.h" #include #include +#include #include "string_format.h" #include "xml/Parser.h" #include "Url.h" -CAmazonS3Client::CAmazonS3Client(CAmazonConfigs configs, std::string region, std::string endpoint) +CAmazonS3Client::CAmazonS3Client(CAmazonConfigs configs, std::string region) : CAmazonClient("s3", std::move(configs), std::move(region)) - , m_endpoint(endpoint) { } +CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB method, std::string bucket, std::string key) +{ + Request rq; + rq.method = method; + rq.uri = string_format("/%s", bucket.c_str()); + if(!key.empty()) + { + rq.uri = string_format("%s/%s", rq.uri.c_str(), key.c_str()); + } + + std::string endpoint; + switch (m_configs.m_provider) + { + case CAmazonConfigs::S3PROVIDER::CF_R2: + endpoint = "r2.cloudflarestorage.com"; + rq.host = string_format("%s.%s", m_configs.accountKeyId.c_str(), endpoint.c_str()); + rq.urlHost = rq.host; + return rq; + default: + assert(false); + case CAmazonConfigs::S3PROVIDER::AWS_S3: + endpoint = "amazonaws.com"; + break; + } + + if(m_region.empty()) + { + rq.host = string_format("s3.%s", endpoint.c_str()); + } + else + { + rq.host = string_format("s3.%s.%s", m_region.c_str(), endpoint.c_str()); + } + rq.urlHost = rq.host; + return rq; +} + GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocationRequest& request) { + GetBucketLocationResult result; + // CF R2 does not support regions, thus doesnt support location call, + // but it seems region is needed for signature verification + if(m_configs.m_provider == CAmazonConfigs::CF_R2) + { + result.locationConstraint = m_region; + return result; + } + if(request.bucket.empty()) { throw new std::runtime_error("Bucket name must be provided."); } - Request rq; - rq.method = Framework::Http::HTTP_VERB::GET; - rq.host = string_format("s3.%s", m_endpoint.c_str()); - rq.urlHost = rq.host; - rq.uri = string_format("/%s", request.bucket.c_str()); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket); //We add a bucket parameter even if the S3 API doesn't use it to prevent caching rq.query = string_format("bucket=%s&location=", request.bucket.c_str()); @@ -32,8 +74,6 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati throw std::runtime_error("Failed to get bucket location."); } - GetBucketLocationResult result; - auto documentNode = std::unique_ptr(Framework::Xml::CParser::ParseDocument(response.data)); auto locationConstraintNode = documentNode->Select("LocationConstraint"); if(locationConstraintNode) @@ -47,11 +87,7 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) { - Request rq; - rq.method = Framework::Http::HTTP_VERB::GET; - rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); - rq.urlHost = rq.host; + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket, Framework::UrlEncode(request.key)); if(request.range.first != request.range.second) { @@ -76,11 +112,7 @@ GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) { - Request rq; - rq.method = Framework::Http::HTTP_VERB::HEAD; - rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); - rq.urlHost = rq.host; + Request rq = CreateRequest(Framework::Http::HTTP_VERB::HEAD, request.bucket, Framework::UrlEncode(request.key)); auto response = ExecuteRequest(rq); if(response.statusCode != Framework::Http::HTTP_STATUS_CODE::OK) @@ -108,11 +140,7 @@ HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) { - Request rq; - rq.method = Framework::Http::HTTP_VERB::GET; - rq.uri = string_format("/%s", bucket.c_str()); - rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); - rq.urlHost = rq.host; + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, bucket); auto response = ExecuteRequest(rq); if(response.statusCode != Framework::Http::HTTP_STATUS_CODE::OK) @@ -139,11 +167,7 @@ ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) void CAmazonS3Client::PutObject(const PutObjectRequest& request) { - Request rq; - rq.method = Framework::Http::HTTP_VERB::PUT; - rq.uri = string_format("/%s/%s", request.bucket.c_str(), Framework::UrlEncode(request.key).c_str()); - rq.host = string_format("s3.%s.%s", m_region.c_str(), m_endpoint.c_str()); - rq.urlHost = rq.host; + Request rq = CreateRequest(Framework::Http::HTTP_VERB::PUT, Framework::UrlEncode(request.key)); rq.content = request.data; auto response = ExecuteRequest(rq); From 1ee856a2a856a21e782b920c3d1b7f9bb9ccc2f9 Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Wed, 18 May 2022 11:59:59 +0100 Subject: [PATCH 5/7] we want to keep aws s3 on virtual-style access, as path-style access is deprecated --- src/amazon/AmazonS3Client.cpp | 41 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index c1e36209..aa2f5df5 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -15,36 +15,41 @@ CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB { Request rq; rq.method = method; - rq.uri = string_format("/%s", bucket.c_str()); - if(!key.empty()) - { - rq.uri = string_format("%s/%s", rq.uri.c_str(), key.c_str()); - } - std::string endpoint; - switch (m_configs.m_provider) + // Path‐style bucket access + if(m_configs.m_provider == CAmazonConfigs::S3PROVIDER::CF_R2) { - case CAmazonConfigs::S3PROVIDER::CF_R2: - endpoint = "r2.cloudflarestorage.com"; - rq.host = string_format("%s.%s", m_configs.accountKeyId.c_str(), endpoint.c_str()); + auto endpoint = "r2.cloudflarestorage.com"; + rq.uri = string_format("/%s", bucket.c_str()); + if(!key.empty()) + { + rq.uri = string_format("%s/%s", rq.uri.c_str(), key.c_str()); + } + rq.host = string_format("%s.%s", m_configs.accountKeyId.c_str(), endpoint); rq.urlHost = rq.host; return rq; - default: - assert(false); - case CAmazonConfigs::S3PROVIDER::AWS_S3: - endpoint = "amazonaws.com"; - break; } + // Virtual hosted‐style bucket access + assert(m_configs.m_provider == CAmazonConfigs::S3PROVIDER::AWS_S3); + + rq.uri = "/"; + if(!key.empty()) + { + rq.uri = string_format("/%s", key.c_str()); + } + + std::string endpoint = "amazonaws.com"; if(m_region.empty()) { - rq.host = string_format("s3.%s", endpoint.c_str()); + rq.host = string_format("%s.s3.%s", bucket.c_str(), endpoint.c_str()); + rq.urlHost = string_format("s3.%s", endpoint.c_str()); } else { - rq.host = string_format("s3.%s.%s", m_region.c_str(), endpoint.c_str()); + rq.host = string_format("%s.s3-%s.%s", bucket.c_str(), m_region.c_str(), endpoint.c_str()); + rq.urlHost = rq.host; } - rq.urlHost = rq.host; return rq; } From 48a398d3833ac559f0b933301c599b4775b47625 Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Wed, 18 May 2022 12:01:00 +0100 Subject: [PATCH 6/7] cleanup --- src/amazon/AmazonS3Client.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index aa2f5df5..811e010c 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -11,7 +11,7 @@ CAmazonS3Client::CAmazonS3Client(CAmazonConfigs configs, std::string region) { } -CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB method, std::string bucket, std::string key) +CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB method, std::string bucket, std::string path) { Request rq; rq.method = method; @@ -21,9 +21,9 @@ CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB { auto endpoint = "r2.cloudflarestorage.com"; rq.uri = string_format("/%s", bucket.c_str()); - if(!key.empty()) + if(!path.empty()) { - rq.uri = string_format("%s/%s", rq.uri.c_str(), key.c_str()); + rq.uri = string_format("%s/%s", rq.uri.c_str(), path.c_str()); } rq.host = string_format("%s.%s", m_configs.accountKeyId.c_str(), endpoint); rq.urlHost = rq.host; @@ -34,9 +34,9 @@ CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB assert(m_configs.m_provider == CAmazonConfigs::S3PROVIDER::AWS_S3); rq.uri = "/"; - if(!key.empty()) + if(!path.empty()) { - rq.uri = string_format("/%s", key.c_str()); + rq.uri = string_format("/%s", path.c_str()); } std::string endpoint = "amazonaws.com"; From d9054eddcfdea7f60ff193eb759cb2fe19b1682a Mon Sep 17 00:00:00 2001 From: Mahmood - Zer0xFF <5013823+Zer0xFF@users.noreply.github.com> Date: Wed, 18 May 2022 12:17:22 +0100 Subject: [PATCH 7/7] correctly evaluate region in CAmazonS3Client::CreateRequest --- include/amazon/AmazonS3Client.h | 2 +- src/amazon/AmazonS3Client.cpp | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/include/amazon/AmazonS3Client.h b/include/amazon/AmazonS3Client.h index 75bf6fac..b73f7f72 100644 --- a/include/amazon/AmazonS3Client.h +++ b/include/amazon/AmazonS3Client.h @@ -67,5 +67,5 @@ class CAmazonS3Client : public CAmazonClient void PutObject(const PutObjectRequest&); private: - Request CreateRequest(Framework::Http::HTTP_VERB, std::string, std::string = ""); + Request CreateRequest(Framework::Http::HTTP_VERB, std::string, std::string, std::string = ""); }; diff --git a/src/amazon/AmazonS3Client.cpp b/src/amazon/AmazonS3Client.cpp index 811e010c..565e6611 100644 --- a/src/amazon/AmazonS3Client.cpp +++ b/src/amazon/AmazonS3Client.cpp @@ -11,7 +11,7 @@ CAmazonS3Client::CAmazonS3Client(CAmazonConfigs configs, std::string region) { } -CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB method, std::string bucket, std::string path) +CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB method, std::string bucket, std::string region, std::string path) { Request rq; rq.method = method; @@ -40,14 +40,14 @@ CAmazonClient::Request CAmazonS3Client::CreateRequest(Framework::Http::HTTP_VERB } std::string endpoint = "amazonaws.com"; - if(m_region.empty()) + if(region.empty()) { rq.host = string_format("%s.s3.%s", bucket.c_str(), endpoint.c_str()); rq.urlHost = string_format("s3.%s", endpoint.c_str()); } else { - rq.host = string_format("%s.s3-%s.%s", bucket.c_str(), m_region.c_str(), endpoint.c_str()); + rq.host = string_format("%s.s3-%s.%s", bucket.c_str(), region.c_str(), endpoint.c_str()); rq.urlHost = rq.host; } return rq; @@ -68,8 +68,7 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati { throw new std::runtime_error("Bucket name must be provided."); } - - Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket, ""); //We add a bucket parameter even if the S3 API doesn't use it to prevent caching rq.query = string_format("bucket=%s&location=", request.bucket.c_str()); @@ -92,7 +91,7 @@ GetBucketLocationResult CAmazonS3Client::GetBucketLocation(const GetBucketLocati GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) { - Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket, Framework::UrlEncode(request.key)); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, request.bucket, m_region, Framework::UrlEncode(request.key)); if(request.range.first != request.range.second) { @@ -117,7 +116,7 @@ GetObjectResult CAmazonS3Client::GetObject(const GetObjectRequest& request) HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) { - Request rq = CreateRequest(Framework::Http::HTTP_VERB::HEAD, request.bucket, Framework::UrlEncode(request.key)); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::HEAD, request.bucket, m_region, Framework::UrlEncode(request.key)); auto response = ExecuteRequest(rq); if(response.statusCode != Framework::Http::HTTP_STATUS_CODE::OK) @@ -145,7 +144,7 @@ HeadObjectResult CAmazonS3Client::HeadObject(const HeadObjectRequest& request) ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) { - Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, bucket); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::GET, bucket, m_region); auto response = ExecuteRequest(rq); if(response.statusCode != Framework::Http::HTTP_STATUS_CODE::OK) @@ -172,7 +171,7 @@ ListObjectsResult CAmazonS3Client::ListObjects(std::string bucket) void CAmazonS3Client::PutObject(const PutObjectRequest& request) { - Request rq = CreateRequest(Framework::Http::HTTP_VERB::PUT, Framework::UrlEncode(request.key)); + Request rq = CreateRequest(Framework::Http::HTTP_VERB::PUT, request.bucket.c_str(), m_region, Framework::UrlEncode(request.key)); rq.content = request.data; auto response = ExecuteRequest(rq);