point at django-log-asim dep at branch for testing

django_app/pyproject.toml

@@ -9,7 +9,7 @@ authors = [
 license = "MIT"
 
 [tool.poetry.dependencies]
-django_log_formatter_asim = "^1.3.0"
+django-log-formatter-asim = { git = "https://github.com/uktrade/django-log-formatter-asim.git", rev = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" }
 python = ">=3.12,<3.13"
 redbox = { path = "../redbox", develop = true }
 django = "5.2.12"
@@ -52,7 +52,6 @@ scikit-learn = "^1.8.0"
 django-staff-sso-client = "^5.2.0"
 dj-database-url = "^3.1.2"
 dbt-copilot-python = "^2.1.0"
-django-log-formatter-asim = "^1.3.0"
 django-chunk-upload-handlers = "^0.0.15"
 mohawk = "^1.1.0"
 uwotm8 = "^0.1.6"

django_app/redbox_app/redbox_core/settings/dd_asim_formatter.py

@@ -0,0 +1,69 @@
+import json
+import os
+import re
+
+import ddtrace
+from ddtrace import tracer
+from django_log_formatter_asim import ASIMFormatter
+
+S3_HEADERS = ("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Security-Token")
+EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
+URL_RE = re.compile(r"https?://\S+")
+
+
+def redact_record(record):
+    record.msg = redact_emails_and_s3_headers(record.getMessage())
+    record.args = None  # args already interpolated into msg
+    return True
+
+
+def redact_emails_and_s3_headers(text):
+    text = EMAIL_RE.sub("[REDACTED_EMAIL]", text)
+    if any(h in text for h in S3_HEADERS):
+        text = URL_RE.sub("[REDACTED_URL]", text)
+    return text
+
+
+class DDASIMFormatter(ASIMFormatter):
+    def _get_container_id(self):
+        """
+        The dockerId (container Id) is available via the metadata endpoint. However, the it looks like it is embedded
+        in the metadata URL,eg:
+        ECS_CONTAINER_METADATA_URI=http://169.254.170.2/v3/709d1c10779d47b2a84db9eef2ebd041-0265927825
+        See: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4-response.html
+        """
+        try:
+            return os.environ["ECS_CONTAINER_METADATA_URI"].split("/")[-1]
+        except (KeyError, IndexError):
+            return ""
+
+    def _datadog_trace_dict(self):
+        # source: https://docs.datadoghq.com/tracing/other_telemetry/connect_logs_and_traces/python/
+
+        event_dict = {}
+
+        span = tracer.current_span()
+        trace_id, span_id = (str((1 << 64) - 1 & span.trace_id), span.span_id) if span else (None, None)
+
+        # add ids to structlog event dictionary
+        event_dict["dd.trace_id"] = str(trace_id or 0)
+        event_dict["dd.span_id"] = str(span_id or 0)
+
+        # add the env, service, and version configured for the tracer
+        event_dict["env"] = ddtrace.config.env or ""
+        event_dict["service"] = ddtrace.config.service or ""
+        event_dict["version"] = ddtrace.config.version or ""
+
+        event_dict["container_id"] = self._get_container_id()
+
+        return event_dict
+
+    def format(self, record):
+        log_dict = json.loads(super().format(record))
+
+        if record.exc_info:
+            log_dict["exception"] = self.formatException(record.exc_info)
+
+        log_dict.update(self._datadog_trace_dict())
+
+        return redact_emails_and_s3_headers(json.dumps(log_dict, default=str))

django_app/redbox_app/settings.py

@@ -1,7 +1,6 @@
 # mypy: ignore-errors
 import logging
 import os
-import re
 import socket
 from pathlib import Path
 from urllib.parse import urlparse
@@ -11,13 +10,13 @@
 from dbt_copilot_python.database import database_from_env
 from dbt_copilot_python.error_tracking import DatadogErrorTrackingFilter
 from django.urls import reverse_lazy
-from django_log_formatter_asim import ASIMFormatter
 from dotenv import find_dotenv, load_dotenv
 from import_export.formats.base_formats import CSV
 from sentry_sdk.integrations.django import DjangoIntegration
 from storages.backends import s3boto3
 from yarl import URL
 
+from redbox_app.redbox_core.settings.dd_asim_formatter import DDASIMFormatter, redact_record
 from redbox_app.setting_enums import Classification, Environment
 
 logger = logging.getLogger(__name__)
@@ -361,23 +360,13 @@ def filter_transactions(event, _hint):
     "formatters": {
         "verbose": {"format": "%(asctime)s %(levelname)s %(module)s: %(message)s"},
         "asim_formatter": {
-            "()": ASIMFormatter,
+            "()": DDASIMFormatter,
         },
     },
     "filters": {
         "exclude_s3_urls_and_emails": {
             "": "django.utils.log.CallbackFilter",
-            "callback": lambda record: (
-                (
-                    all(
-                        header not in record.getMessage()
-                        for header in ["X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Security-Token"]
-                    )
-                    and not re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", record.getMessage())
-                )
-                if hasattr(record, "getMessage")
-                else True
-            ),
+            "callback": redact_record,
         },
         "error_tracking": {"()": DatadogErrorTrackingFilter},
     },

pyproject.toml

@@ -21,7 +21,7 @@ django-staff-sso-client = "^5.2.0"
 dbt-copilot-python = "^2.1.0"
 uwotm8 = "^0.1.6"
 ddtrace = "^4.10.3"
-django-log-formatter-asim = "^1.3.0"
+django-log-formatter-asim = {git = "https://github.com/uktrade/django-log-formatter-asim.git", rev = "58b52cd82debfd4fe3a8d3867c63fae7597e658d"}
 django-requestlogs = "^0.8"
 openpyxl = "^3.1.5"
 coverage = "^7.13.5"