diff --git a/django_app/poetry.lock b/django_app/poetry.lock index c6bec269e..049a2a901 100644 --- a/django_app/poetry.lock +++ b/django_app/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "accelerate" @@ -2408,17 +2408,21 @@ name = "django-log-formatter-asim" version = "1.3.0" description = "Formats Django logs in ASIM format." optional = false -python-versions = "<4,>=3.10" +python-versions = ">=3.10,<4" groups = ["main"] -files = [ - {file = "django_log_formatter_asim-1.3.0-py3-none-any.whl", hash = "sha256:59bdd288e2482a74c2c41d0e0b2f8c85221a96a631d11f55924b07b37ff3ed09"}, - {file = "django_log_formatter_asim-1.3.0.tar.gz", hash = "sha256:10e03b75f020a3176fd565da1b3b5b8276808c3ac13ce8dc7d39dd40a32206da"}, -] +files = [] +develop = false [package.dependencies] ddtrace = ">=3.2.1,<5" django = ">=4.2,<7" -django-ipware = ">=7.0.1,<8.0.0" +django-ipware = "^7.0.1" + +[package.source] +type = "git" +url = "https://github.com/uktrade/django-log-formatter-asim.git" +reference = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" +resolved_reference = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" [[package]] name = "django-magic-link" @@ -11465,4 +11469,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.13" -content-hash = "fa16631f516a858827a987eeb5d50ec37aba45aa095581c8480fd57a653f5f09" +content-hash = "8118769c0bd49672ae498066ba41852f5d562ca931a2f4623bcf84d905302d8c" diff --git a/django_app/pyproject.toml b/django_app/pyproject.toml index eb8857a9c..db2fdac6b 100644 --- a/django_app/pyproject.toml +++ b/django_app/pyproject.toml @@ -9,7 +9,7 @@ authors = [ license = "MIT" [tool.poetry.dependencies] -django_log_formatter_asim = "^1.3.0" +django-log-formatter-asim = { git = "https://github.com/uktrade/django-log-formatter-asim.git", rev = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" } python = ">=3.12,<3.13" redbox = { path = "../redbox", develop = true } django = "5.2.12" @@ -52,7 +52,6 @@ scikit-learn = "^1.8.0" django-staff-sso-client = "^5.2.0" dj-database-url = "^3.1.2" dbt-copilot-python = "^2.1.0" -django-log-formatter-asim = "^1.3.0" django-chunk-upload-handlers = "^0.0.15" mohawk = "^1.1.0" uwotm8 = "^0.1.6" diff --git a/django_app/redbox_app/redbox_core/settings/dd_asim_formatter.py b/django_app/redbox_app/redbox_core/settings/dd_asim_formatter.py new file mode 100644 index 000000000..b29051afd --- /dev/null +++ b/django_app/redbox_app/redbox_core/settings/dd_asim_formatter.py @@ -0,0 +1,69 @@ +import json +import os +import re + +import ddtrace +from ddtrace import tracer +from django_log_formatter_asim import ASIMFormatter + +S3_HEADERS = ("X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Security-Token") +EMAIL_RE = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") +URL_RE = re.compile(r"https?://\S+") + + +def redact_record(record): + record.msg = redact_emails_and_s3_headers(record.getMessage()) + record.args = None # args already interpolated into msg + return True + + +def redact_emails_and_s3_headers(text): + text = EMAIL_RE.sub("[REDACTED_EMAIL]", text) + if any(h in text for h in S3_HEADERS): + text = URL_RE.sub("[REDACTED_URL]", text) + return text + + +class DDASIMFormatter(ASIMFormatter): + def _get_container_id(self): + """ + The dockerId (container Id) is available via the metadata endpoint. However, the it looks like it is embedded + in the metadata URL,eg: + ECS_CONTAINER_METADATA_URI=http://169.254.170.2/v3/709d1c10779d47b2a84db9eef2ebd041-0265927825 + See: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v4-response.html + """ + try: + return os.environ["ECS_CONTAINER_METADATA_URI"].split("/")[-1] + except (KeyError, IndexError): + return "" + + def _datadog_trace_dict(self): + # source: https://docs.datadoghq.com/tracing/other_telemetry/connect_logs_and_traces/python/ + + event_dict = {} + + span = tracer.current_span() + trace_id, span_id = (str((1 << 64) - 1 & span.trace_id), span.span_id) if span else (None, None) + + # add ids to structlog event dictionary + event_dict["dd.trace_id"] = str(trace_id or 0) + event_dict["dd.span_id"] = str(span_id or 0) + + # add the env, service, and version configured for the tracer + event_dict["env"] = ddtrace.config.env or "" + event_dict["service"] = ddtrace.config.service or "" + event_dict["version"] = ddtrace.config.version or "" + + event_dict["container_id"] = self._get_container_id() + + return event_dict + + def format(self, record): + log_dict = json.loads(super().format(record)) + + if record.exc_info: + log_dict["exception"] = self.formatException(record.exc_info) + + log_dict.update(self._datadog_trace_dict()) + + return redact_emails_and_s3_headers(json.dumps(log_dict, default=str)) diff --git a/django_app/redbox_app/settings.py b/django_app/redbox_app/settings.py index f13dfa671..b41981415 100644 --- a/django_app/redbox_app/settings.py +++ b/django_app/redbox_app/settings.py @@ -1,7 +1,6 @@ # mypy: ignore-errors import logging import os -import re import socket from pathlib import Path from urllib.parse import urlparse @@ -11,13 +10,13 @@ from dbt_copilot_python.database import database_from_env from dbt_copilot_python.error_tracking import DatadogErrorTrackingFilter from django.urls import reverse_lazy -from django_log_formatter_asim import ASIMFormatter from dotenv import find_dotenv, load_dotenv from import_export.formats.base_formats import CSV from sentry_sdk.integrations.django import DjangoIntegration from storages.backends import s3boto3 from yarl import URL +from redbox_app.redbox_core.settings.dd_asim_formatter import DDASIMFormatter, redact_record from redbox_app.setting_enums import Classification, Environment logger = logging.getLogger(__name__) @@ -361,23 +360,13 @@ def filter_transactions(event, _hint): "formatters": { "verbose": {"format": "%(asctime)s %(levelname)s %(module)s: %(message)s"}, "asim_formatter": { - "()": ASIMFormatter, + "()": DDASIMFormatter, }, }, "filters": { "exclude_s3_urls_and_emails": { "": "django.utils.log.CallbackFilter", - "callback": lambda record: ( - ( - all( - header not in record.getMessage() - for header in ["X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Security-Token"] - ) - and not re.search(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", record.getMessage()) - ) - if hasattr(record, "getMessage") - else True - ), + "callback": redact_record, }, "error_tracking": {"()": DatadogErrorTrackingFilter}, }, diff --git a/poetry.lock b/poetry.lock index c06c5eff8..f14691498 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "accelerate" @@ -2237,17 +2237,21 @@ name = "django-log-formatter-asim" version = "1.3.0" description = "Formats Django logs in ASIM format." optional = false -python-versions = "<4,>=3.10" +python-versions = ">=3.10,<4" groups = ["main"] -files = [ - {file = "django_log_formatter_asim-1.3.0-py3-none-any.whl", hash = "sha256:59bdd288e2482a74c2c41d0e0b2f8c85221a96a631d11f55924b07b37ff3ed09"}, - {file = "django_log_formatter_asim-1.3.0.tar.gz", hash = "sha256:10e03b75f020a3176fd565da1b3b5b8276808c3ac13ce8dc7d39dd40a32206da"}, -] +files = [] +develop = false [package.dependencies] ddtrace = ">=3.2.1,<5" django = ">=4.2,<7" -django-ipware = ">=7.0.1,<8.0.0" +django-ipware = "^7.0.1" + +[package.source] +type = "git" +url = "https://github.com/uktrade/django-log-formatter-asim.git" +reference = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" +resolved_reference = "58b52cd82debfd4fe3a8d3867c63fae7597e658d" [[package]] name = "django-requestlogs" @@ -10571,4 +10575,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.12,<3.13" -content-hash = "bccb8a2d50820660e2e43bdaceff3708577cf3f92480c5630d3a4b42d935b6cb" +content-hash = "6c877bc3773d67dcc151b1838fa85a1187832d62098e907782cc99c6101a3b1b" diff --git a/pyproject.toml b/pyproject.toml index 7b4657b68..b185cea19 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ django-staff-sso-client = "^5.2.0" dbt-copilot-python = "^2.1.0" uwotm8 = "^0.1.6" ddtrace = "^4.10.3" -django-log-formatter-asim = "^1.3.0" +django-log-formatter-asim = {git = "https://github.com/uktrade/django-log-formatter-asim.git", rev = "58b52cd82debfd4fe3a8d3867c63fae7597e658d"} django-requestlogs = "^0.8" openpyxl = "^3.1.5" coverage = "^7.13.5"