fix(logs_monitoring): read the API key from the region in its ARN

aws/logs_monitoring/README.md

@@ -492,10 +492,10 @@ The Datadog Forwarder is signed by Datadog. To verify the integrity of the Forwa
 : Your [Datadog API key][20], which can be found under **Organization Settings** > **API Keys**. The API Key is stored in AWS Secrets Manager. If you already have a Datadog API Key stored in Secrets Manager, use `DdApiKeySecretArn` instead.
 
 `DdApiKeySecretArn`
-: The ARN of the secret storing the Datadog API key, if you already have it stored in Secrets Manager. You must store the secret as a plaintext, rather than a key-value pair.
+: The ARN of the secret storing the Datadog API key, if you already have it stored in Secrets Manager. You must store the secret as a plaintext, rather than a key-value pair. The secret may live in a different region than the Forwarder, as the region is read from the ARN.
 
 `DdApiKeySsmParameterName`
-: The name of the SSM parameter containing the Datadog API key. If set, both `DdApiKey` and `DdApiKeySecretArn` are ignored.
+: The name of the SSM parameter containing the Datadog API key, or its full ARN when the parameter lives in a different region than the Forwarder. If set, both `DdApiKey` and `DdApiKeySecretArn` are ignored.
 
 `DdSite`
 : The [Datadog site][13] that your metrics and logs will be sent to. Your Datadog site is {{< region-param key="dd_site" code="true" >}}.
@@ -673,13 +673,13 @@ If you are installing the Forwarder manually, convert the parameter names from P
 : Your [Datadog API key][20], which can be found under **Organization Settings** > **API Keys**. The API Key is stored in AWS Secrets Manager. If you already have a Datadog API Key stored in Secrets Manager, use `DD_API_KEY_SECRET_ARN` instead.
 
 `DD_API_KEY_SECRET_ARN`
-: The ARN of the secret storing the Datadog API key, if you already have it stored in Secrets Manager. You must store the secret as a plaintext, rather than a key-value pair.
+: The ARN of the secret storing the Datadog API key, if you already have it stored in Secrets Manager. You must store the secret as a plaintext, rather than a key-value pair. The secret may live in a different region than the Forwarder, as the region is read from the ARN.
 
 `DD_API_KEY_SSM_NAME`
-: The name of the parameter in AWS Systems Manager (SSM) Parameter Store containing the Datadog API key. Takes precedence over `DD_KMS_API_KEY` and `DD_API_KEY`.
+: The name of the parameter in AWS Systems Manager (SSM) Parameter Store containing the Datadog API key. Takes precedence over `DD_KMS_API_KEY` and `DD_API_KEY`. Pass the full parameter ARN instead of the name to read a parameter from a different region than the Forwarder.
 
 `DD_KMS_API_KEY`
-: The Datadog API key encrypted with AWS KMS. Takes precedence over `DD_API_KEY`.
+: The Datadog API key encrypted with AWS KMS. Takes precedence over `DD_API_KEY`. The KMS key must be in the same region as the Forwarder.
 
 `DD_SITE`
 : The [Datadog site][13] that your metrics and logs will be sent to. Your Datadog site is {{< region-param key="dd_site" code="true" >}}.

aws/logs_monitoring/settings.py

@@ -162,15 +162,37 @@ def __init__(self, name, pattern, placeholder, enabled=True):
 boto3_config = botocore.config.Config(
     connect_timeout=5, read_timeout=5, retries={"max_attempts": 2}
 )
+
+
+def get_region_from_arn(arn):
+    """
+    Return the region encoded in an ARN, or None when the value is not an ARN.
+
+    boto3 resolves the client region from the environment, which on Lambda is
+    always the region the function runs in. Secrets Manager and SSM do not
+    redirect a request based on the ARN it carries, so a secret or parameter
+    that lives in another region is only reachable when its region is passed
+    to the client explicitly. Returning None keeps the default resolution for
+    values that are not ARNs, such as a plain SSM parameter name.
+    """
+    # arn:<partition>:<service>:<region>:<account-id>:<resource>
+    parts = arn.split(":")
+    if len(parts) < 6 or parts[0] != "arn" or not parts[3]:
+        return None
+    return parts[3]
+
+
 # DD API Key
 # Check if the DD_API_KEY_SECRET_ARN environment variable is set
 if "DD_API_KEY_SECRET_ARN" in os.environ:
     SECRET_ARN = os.environ["DD_API_KEY_SECRET_ARN"]
     logger.debug(f"Fetching the Datadog API key from SecretsManager: {SECRET_ARN}")
 
-    # Fetch the secret from Secrets Manager
+    # Fetch the secret from Secrets Manager, from the region the ARN points to
     secret_response = boto3.client(
-        "secretsmanager", config=boto3_config
+        "secretsmanager",
+        region_name=get_region_from_arn(SECRET_ARN),
+        config=boto3_config,
     ).get_secret_value(SecretId=SECRET_ARN)
 
     # The secret could be either a plain string or a JSON object
@@ -200,9 +222,11 @@ def __init__(self, name, pattern, placeholder, enabled=True):
 elif "DD_API_KEY_SSM_NAME" in os.environ:
     SECRET_NAME = os.environ["DD_API_KEY_SSM_NAME"]
     logger.debug(f"Fetching the Datadog API key from SSM: {SECRET_NAME}")
-    DD_API_KEY = boto3.client("ssm", config=boto3_config).get_parameter(
-        Name=SECRET_NAME, WithDecryption=True
-    )["Parameter"]["Value"]
+    DD_API_KEY = boto3.client(
+        "ssm",
+        region_name=get_region_from_arn(SECRET_NAME),
+        config=boto3_config,
+    ).get_parameter(Name=SECRET_NAME, WithDecryption=True)["Parameter"]["Value"]
 elif "DD_KMS_API_KEY" in os.environ:
     ENCRYPTED = os.environ["DD_KMS_API_KEY"]
     logger.debug(f"Fetching the Datadog API key from KMS: {ENCRYPTED}")

aws/logs_monitoring/template.yaml

@@ -19,8 +19,8 @@ Parameters:
   DdApiKeySsmParameterName:
     Type: String
     Default: "/my/parameter/path"
-    AllowedPattern: "^/[a-zA-Z0-9/_.-]*$"
-    Description: The name of the SSM parameter containing Datadog's API key. If set, both DdApiKey and DdApiKeySecretArn will be ignored, the forwarder will use the SSM parameter name to fetch the API key.
+    AllowedPattern: "^(/[a-zA-Z0-9/_.-]*|arn:[a-z0-9-]*:ssm:[a-z0-9-]+:[0-9]{12}:parameter/[a-zA-Z0-9/_.-]+)$"
+    Description: The name of the SSM parameter containing Datadog's API key, or its full ARN when the parameter lives in another region. If set, both DdApiKey and DdApiKeySecretArn will be ignored, the forwarder will use the SSM parameter name to fetch the API key.
   DdSite:
     Type: String
     Default: datadoghq.com

aws/logs_monitoring/tests/test_settings.py

@@ -1,9 +1,15 @@
+import os
+import sys
 import unittest
+from importlib import reload
 from unittest.mock import MagicMock, patch
 
-from settings import is_api_key_valid
+from settings import get_region_from_arn, is_api_key_valid
 
 VALID_API_KEY = "11111111111111111111111111111111"
+# A key that is not also valid JSON, so it round-trips through the plain-string
+# branch of the Secrets Manager and SSM lookups
+STORED_API_KEY = "abcdef1234567890abcdef1234567890"
 
 
 # For the integration tests to work because of other tests set sys.modules["requests"] as a MagicMock.
@@ -59,5 +65,126 @@ def test_on_timeout_exception(self, mock_session_cls, mock_logger):
         self.assertIn("network error", mock_logger.warning.call_args[0][0].lower())
 
 
+class TestGetRegionFromArn(unittest.TestCase):
+    def test_secret_arn(self):
+        self.assertEqual(
+            get_region_from_arn(
+                "arn:aws:secretsmanager:eu-west-1:123456789012:secret:dd-api-key-AbCdEf"
+            ),
+            "eu-west-1",
+        )
+
+    def test_secret_partial_arn(self):
+        self.assertEqual(
+            get_region_from_arn(
+                "arn:aws:secretsmanager:us-west-2:123456789012:secret:dd-api-key"
+            ),
+            "us-west-2",
+        )
+
+    def test_ssm_parameter_arn(self):
+        self.assertEqual(
+            get_region_from_arn(
+                "arn:aws:ssm:ap-southeast-2:123456789012:parameter/datadog/api-key"
+            ),
+            "ap-southeast-2",
+        )
+
+    def test_arn_in_another_partition(self):
+        self.assertEqual(
+            get_region_from_arn(
+                "arn:aws-us-gov:secretsmanager:us-gov-west-1:123456789012:secret:dd-api-key"
+            ),
+            "us-gov-west-1",
+        )
+
+    def test_ssm_parameter_name(self):
+        self.assertIsNone(get_region_from_arn("/datadog/api-key"))
+
+    def test_secret_friendly_name(self):
+        self.assertIsNone(get_region_from_arn("dd-api-key"))
+
+    def test_arn_without_a_region(self):
+        self.assertIsNone(
+            get_region_from_arn("arn:aws:iam::123456789012:role/dd-forwarder")
+        )
+
+    def test_truncated_arn(self):
+        self.assertIsNone(get_region_from_arn("arn:aws:secretsmanager:eu-west-1"))
+
+
+class TestApiKeyClientRegion(unittest.TestCase):
+    """
+    The API key is fetched while settings is imported, so each case reloads the
+    module with a patched boto3 to capture how the client was built.
+    """
+
+    API_KEY_SOURCES = (
+        "DD_API_KEY_SECRET_ARN",
+        "DD_API_KEY_SSM_NAME",
+        "DD_KMS_API_KEY",
+    )
+
+    def tearDown(self):
+        reload(sys.modules["settings"])
+
+    def _reload_settings(self, env, boto3_client):
+        with patch.dict(os.environ, env), patch("boto3.client", boto3_client):
+            # Only the source under test may win the precedence chain
+            for var in self.API_KEY_SOURCES:
+                if var not in env:
+                    os.environ.pop(var, None)
+            reload(sys.modules["settings"])
+
+    def test_secret_client_targets_the_secret_region(self):
+        boto3_client = MagicMock()
+        boto3_client.return_value.get_secret_value.return_value = {
+            "SecretString": STORED_API_KEY
+        }
+
+        self._reload_settings(
+            {
+                "DD_API_KEY_SECRET_ARN": (
+                    "arn:aws:secretsmanager:eu-west-1:123456789012:secret:dd-api-key-AbCdEf"
+                )
+            },
+            boto3_client,
+        )
+
+        self.assertEqual(boto3_client.call_args.args[0], "secretsmanager")
+        self.assertEqual(boto3_client.call_args.kwargs["region_name"], "eu-west-1")
+        self.assertEqual(sys.modules["settings"].DD_API_KEY, STORED_API_KEY)
+
+    def test_ssm_client_targets_the_parameter_region(self):
+        boto3_client = MagicMock()
+        boto3_client.return_value.get_parameter.return_value = {
+            "Parameter": {"Value": STORED_API_KEY}
+        }
+
+        self._reload_settings(
+            {
+                "DD_API_KEY_SSM_NAME": (
+                    "arn:aws:ssm:us-west-2:123456789012:parameter/datadog/api-key"
+                )
+            },
+            boto3_client,
+        )
+
+        self.assertEqual(boto3_client.call_args.args[0], "ssm")
+        self.assertEqual(boto3_client.call_args.kwargs["region_name"], "us-west-2")
+        self.assertEqual(sys.modules["settings"].DD_API_KEY, STORED_API_KEY)
+
+    def test_ssm_client_keeps_the_default_region_for_a_parameter_name(self):
+        boto3_client = MagicMock()
+        boto3_client.return_value.get_parameter.return_value = {
+            "Parameter": {"Value": STORED_API_KEY}
+        }
+
+        self._reload_settings({"DD_API_KEY_SSM_NAME": "/datadog/api-key"}, boto3_client)
+
+        self.assertEqual(boto3_client.call_args.args[0], "ssm")
+        self.assertIsNone(boto3_client.call_args.kwargs["region_name"])
+
+
 if __name__ == "__main__":
     unittest.main()