chore: Fix datadog vulnerabilities (off-ticket)

code-security.datadog.yml

@@ -0,0 +1,9 @@
+schema-version: v1.0
+sast:
+  ruleset-configs:
+    python-security:
+      rule-configs:
+        variable-sql-statement-injection:
+          ignore-paths:
+            - "terraform/postgres/manage_users.py"
+            - "dbt_platform_helper/providers/copilot.py"

dbt_platform_helper/providers/load_balancers.py

@@ -321,8 +321,11 @@ def create_forward_rule(
         rule_name: str,
         priority: int,
         conditions: list,
-        additional_tags: list = [],
+        additional_tags=None,
     ):
+        if additional_tags is None:
+            additional_tags = []
+
         return self.create_rule(
             listener_arn=listener_arn,
             priority=priority,

dbt_platform_helper/templates/.copilot/phases/pre_build.sh

@@ -5,8 +5,8 @@ set -e
 
 if [ -f "./.gitmodules" ]; then
   echo ".gitmodules file exists. Modifying URLs..."
-    account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5)
-    connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2)
+    account_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d':' -f5)
+    connection_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d'/' -f2)
     git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade"
 
     git config --global credential.helper '!aws codecommit credential-helper $@'

images/conduit/opensearch/entrypoint.sh

@@ -5,7 +5,7 @@ mkdir -p /root/.opensearch-cli/
 
 function decode_url() { : "${*//+/ }"; echo "${_//%/\\x}"; }
 
-echo $(decode_url "$CONNECTION_SECRET") | gawk '{
+decode_url "$CONNECTION_SECRET" | gawk '{
   match($0, /^https:\/\/([A-Za-z0-9_]+):([^@]+)@(.+)$/, arr);
   print "profiles:"
   print "    - name: connection"
@@ -30,7 +30,7 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   TASKS_RUNNING="$(ps -e -o pid,comm,args | awk '{if ($4 != "/entrypoint.sh" && $2 == "bash" && $3 == "bash") {print $1}}' | wc -l | xargs)"
 
   if [[ $TASKS_RUNNING == 0 ]]; then
-     CHECK_COUNT=$(( $CHECK_COUNT + 1 ))
+     CHECK_COUNT=$(( CHECK_COUNT + 1 ))
      TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))"
      echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds"
   else
@@ -39,9 +39,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   fi
 done
 
-# Trigger CloudFormation stack delete before shutting down
-if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then
-  aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name")
-fi
-
 echo "Shutting down"

images/conduit/opensearch/shell-profile.sh

@@ -19,7 +19,7 @@ function os_run_next_command() {
   echo -n "opensearch => "
   read
   echo "Running opensearch-cli $REPLY"
-  /root/opensearch-cli $REPLY --profile connection
+  /root/opensearch-cli "$REPLY" --profile connection
 }
 
 os_entry

images/conduit/postgres/entrypoint.sh

@@ -10,10 +10,10 @@ CLIENT_TASK="psql"
 
 while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   sleep $CHECK_INTERVAL
-  TASKS_RUNNING="$(ps -e -o pid,comm | grep -c "$CLIENT_TASK")"
+  TASKS_RUNNING="$(pgrep -c "$CLIENT_TASK")"
 
   if [[ $TASKS_RUNNING == 0 ]]; then
-     CHECK_COUNT=$(( $CHECK_COUNT + 1 ))
+     CHECK_COUNT=$(( CHECK_COUNT + 1 ))
      TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))"
      echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds"
   else
@@ -22,9 +22,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   fi
 done
 
-# Trigger CloudFormation stack delete before shutting down
-if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then
-  aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name")
-fi
-
 echo "Shutting down"

images/conduit/postgres/shell-profile.sh

@@ -1,7 +1,7 @@
 echo "Welcome to the client container for postgres services."
-echo $CONNECTION_SECRET | jq -rc '"Connecting to database \"\(.dbname)\" on \"\(.host)\""'
+echo "$CONNECTION_SECRET" | jq -rc '"Connecting to database \"\(.dbname)\" on \"\(.host)\""'
 echo
 
-psql "$(echo $CONNECTION_SECRET | jq -rc '"postgres://\(.username):\(.password)@\(.host):\(.port)/\(.dbname)"')"
+psql "$(echo "$CONNECTION_SECRET" | jq -rc '"postgres://\(.username):\(.password)@\(.host):\(.port)/\(.dbname)"')"
 
 exit

images/conduit/redis/entrypoint.sh

@@ -10,10 +10,10 @@ CLIENT_TASK="redis-cli"
 
 while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   sleep $CHECK_INTERVAL
-  TASKS_RUNNING="$(ps -e -o pid,comm | grep -c "$CLIENT_TASK")"
+  TASKS_RUNNING="$(pgrep -c "$CLIENT_TASK")"
 
   if [[ $TASKS_RUNNING == 0 ]]; then
-     CHECK_COUNT=$(( $CHECK_COUNT + 1 ))
+     CHECK_COUNT=$(( CHECK_COUNT + 1 ))
      TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))"
      echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds"
   else
@@ -22,9 +22,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do
   fi
 done
 
-# Trigger CloudFormation stack delete before shutting down
-if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then
-  aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name")
-fi
-
 echo "Shutting down"

images/conduit/redis/shell-profile.sh

@@ -1,6 +1,6 @@
 echo "Welcome to the client container for redis services."
 echo
 
-redis-cli -u $CONNECTION_SECRET
+redis-cli -u "$CONNECTION_SECRET"
 
 exit

images/tools/database-copy/entrypoint.sh

@@ -4,60 +4,60 @@ clean_up(){
   echo "Cleaning up dump file"
   rm "${DUMP_FILE_NAME}.sql"
   echo "Removing dump file from S3"
-  aws s3 rm s3://${S3_BUCKET_NAME}/"${DUMP_FILE_NAME}.sql"
+  aws s3 rm s3://"${S3_BUCKET_NAME}"/"${DUMP_FILE_NAME}.sql"
   exit_code=$?
-  if [ ${exit_code} -ne 0 ]
+  if [ "${exit_code}" -ne 0 ]
   then
     echo "Aborting data load: Clean up failed"
-    exit $exit_code
+    exit "${exit_code}"
   fi
 }
 
 handle_errors(){
   exit_code=$1
   message=$2
-  if [ ${exit_code} -ne 0 ]
+  if [ "${exit_code}" -ne 0 ]
   then
     clean_up
     echo "Aborting data load: {$message}"
-    exit $exit_code    
+    exit "${exit_code}"
   fi
 }
 
 echo "Detecting postgres version"
-PG_VERSION="$(psql "$DB_CONNECTION_STRING" -c "SELECT version();" | grep "PostgreSQL" | awk '{print $2;}' | awk -F"." '{print $1}')"
-if [ -z "$PG_VERSION" ]; then
+PG_VERSION="$(psql "${DB_CONNECTION_STRING}" -c "SELECT version();" | grep "PostgreSQL" | awk '{print $2;}' | awk -F"." '{print $1}')"
+if [ -z "${PG_VERSION}" ]; then
   PG_VERSION="17"
 fi
-echo "Detected postgres version $PG_VERSION"
+echo "Detected postgres version ${}PG_VERSION}"
 
 if [ "${DATA_COPY_OPERATION:-DUMP}" != "LOAD" ]
 then
   echo "Starting data dump"
-  /usr/lib/postgresql/$PG_VERSION/bin/pg_dump --no-owner --no-acl --format c "${DB_CONNECTION_STRING}" > "${DUMP_FILE_NAME}.sql"
+  /usr/lib/postgresql/"${PG_VERSION}"/bin/pg_dump --no-owner --no-acl --format c "${DB_CONNECTION_STRING}" > "${DUMP_FILE_NAME}.sql"
   exit_code=$?
 
-  if [ ${exit_code} -ne 0 ]
+  if [ "${exit_code}" -ne 0 ]
   then
     echo "Aborting data dump"
-    exit $exit_code
+    exit "${exit_code}"
   fi
 
-  aws s3 cp "${DUMP_FILE_NAME}.sql" s3://${S3_BUCKET_NAME}/
+  aws s3 cp "${DUMP_FILE_NAME}.sql" s3://"${S3_BUCKET_NAME}"/
   exit_code=$?
 
-  if [ ${exit_code} -ne 0 ]
+  if [ "${exit_code}" -ne 0 ]
   then
     echo "Aborting data dump"
-    exit $exit_code
+    exit "${exit_code}"
   fi
 
   echo "Stopping data dump"
 else
   echo "Starting data load"
 
   echo "Copying data dump from S3"
-  aws s3 cp s3://${S3_BUCKET_NAME}/"${DUMP_FILE_NAME}.sql" "${DUMP_FILE_NAME}.sql"
+  aws s3 cp s3://"${S3_BUCKET_NAME}"/"${DUMP_FILE_NAME}.sql" "${DUMP_FILE_NAME}.sql"
 
   handle_errors $? "Copy failed"
 
@@ -76,20 +76,20 @@ else
     CONFIG_FILE="${SERVICE_NAME}.desired_count"
     echo "${COUNT}" > "${CONFIG_FILE}"
 
-    echo ${SERVICE_NAME}
+    echo "${SERVICE_NAME}"
     UPDATE_DATA=$(aws ecs update-service --cluster "${ECS_CLUSTER}" --service "${service}" --desired-count 0)
     handle_errors $? "Failed to update service ${SERVICE_NAME}"
     echo "${UPDATE_DATA}" | jq -r '"  Desired Count: \(.service.desiredCount)\n  Running Count: \(.service.runningCount)"'
 
   done
 
   echo "Clearing down the database prior to loading new data"
-  /usr/lib/postgresql/$PG_VERSION/bin/psql "${DB_CONNECTION_STRING}" -f /clear_db.sql
+  /usr/lib/postgresql/"${PG_VERSION}"/bin/psql "${DB_CONNECTION_STRING}" -f /clear_db.sql
 
   handle_errors $? "Clear down failed"
 
   echo "Restoring data from dump file"
-  /usr/lib/postgresql/$PG_VERSION/bin/pg_restore --format c --dbname "${DB_CONNECTION_STRING}" "${DUMP_FILE_NAME}.sql"
+  /usr/lib/postgresql/"${PG_VERSION}"/bin/pg_restore --format c --dbname "${DB_CONNECTION_STRING}" "${DUMP_FILE_NAME}.sql"
 
   handle_errors $? "Restore failed"
   for service in ${SERVICES}
@@ -98,7 +98,7 @@ else
     COUNT=$(cat "${CONFIG_FILE}")
     SERVICE_NAME=$(basename "${service}")
     echo "Scaling up services"
-    echo ${SERVICE_NAME}
+    echo "${SERVICE_NAME}"
     UPDATE_DATA=$(aws ecs update-service --cluster "${ECS_CLUSTER}" --service "${service}" --desired-count "${COUNT}")
     handle_errors $? "Failed to update service ${SERVICE_NAME}"
     echo "${UPDATE_DATA}" | jq -r '"  Desired Count: \(.service.desiredCount)\n  Running Count: \(.service.runningCount)"'

publish_to_pypi.sh

@@ -13,14 +13,14 @@ if ! python utils/check_pypi.py
 then
   echo Building Python package
   poetry build
-  echo Publishing Python package ${VERSION}
-  poetry config pypi-token.pypi ${PYPI_TOKEN}
+  echo Publishing Python package "${VERSION}"
+  poetry config pypi-token.pypi "${PYPI_TOKEN}"
   poetry publish
   echo Checking the package has reached PyPI
   python utils/check_pypi.py --max-attempts 20
   echo -e "\nSending slack notification"
-  VERSION_NUMBER=($VERSION)
-  poetry run python utils/notify/publish_notification.py --publish-version ${VERSION_NUMBER[1]}
+  VERSION_NUMBER=("${VERSION}")
+  poetry run python utils/notify/publish_notification.py --publish-version "${VERSION_NUMBER[1]}"
 else
-  echo ${VERSION} of the package has already been published
+  echo "${VERSION}" of the package has already been published
 fi

terraform/application-load-balancer/lambda_function/rotate_secret/secret_rotator.py

@@ -546,7 +546,7 @@ def set_secret(self, service_client, arn, token):
                 f"Failed to update resources CloudFront Distro Id {distro['Id']} , WAF WebACL Id {self.waf_acl_id}"
             ) from e
 
-    def run_test_secret(self, service_client, arn, token, test_domains=[]):
+    def run_test_secret(self, service_client, arn, token, test_domains=None):
         """
         Test the secret This method validates that the AWSPENDING secret works
         in the service.
@@ -555,6 +555,9 @@ def run_test_secret(self, service_client, arn, token, test_domains=[]):
         1. Attempts to send a Slack notification (notification failure won't stop the rotation process)
         2. If Lambda event contains key TestDomains and provided domains to test, then you can trigger a Slack notification to the configured Slack channel
         """
+        if test_domains is None:
+            test_domains = []
+
         test_failures = []
 
         # Check for TestDomains key in the Lambda event - currently only used in console to test Slack message is emitted

terraform/scripts/install_platform_helper.sh

@@ -4,7 +4,7 @@ echo -e "Installing dependencies"
 pip install poetry --quiet
 poetry install --quiet
 echo -e "\nBuild platform-helper\n"
-poetry build --no-interaction --format sdist --no-ansi --local-version ${PLATFORM_HELPER_VERSION} --quiet
+poetry build --no-interaction --format sdist --no-ansi --local-version "${PLATFORM_HELPER_VERSION}" --quiet
 echo -e "\nInstall platform-helper\n"
 most_recent_built_wheel_package=$(ls -t1 dist | grep ".whl" | head -1)
 pip install "dist/${most_recent_built_wheel_package}" --quiet

tests/platform_helper/expected_files/.copilot/phases/pre_build.sh

@@ -5,8 +5,8 @@ set -e
 
 if [ -f "./.gitmodules" ]; then
   echo ".gitmodules file exists. Modifying URLs..."
-    account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5)
-    connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2)
+    account_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d':' -f5)
+    connection_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d'/' -f2)
     git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade"
 
     git config --global credential.helper '!aws codecommit credential-helper $@'

tests/platform_helper/integration/test_secrets.py

@@ -95,8 +95,11 @@ def put_parameter_called_with(
         description="",
         secret="SECRET",
         platform="platform",
-        tags=[],
+        tags=None,
     ):
+        if tags is None:
+            tags = []
+
         called_with = dict(
             Name=f"/{platform}/test-application/{env}/secrets/{secret.upper()}",
             Value=str(value),
@@ -119,7 +122,10 @@ def put_parameter_called_with(
         return called_with
 
     @staticmethod
-    def put_parameter_copied_called_with(platform, source, target, secret, tags=[]):
+    def put_parameter_copied_called_with(platform, source, target, secret, tags=None):
+        if tags is None:
+            tags = []
+
         tags = [{"Key": "copied-from", "Value": source}] + tags
         description = f"Copied from {source} environment."
         return AWSTestFixtures.put_parameter_called_with(
@@ -132,7 +138,10 @@ def put_parameter_copied_called_with(platform, source, target, secret, tags=[]):
         )
 
     @staticmethod
-    def simulate_principal_policy_called_with(account_id, role_name, actions=["ssm:PutParameter"]):
+    def simulate_principal_policy_called_with(account_id, role_name, actions=None):
+        if actions is None:
+            actions = ["ssm:PutParameter"]
+
         return dict(
             PolicySourceArn=f"arn:aws:iam::{account_id}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}",
             ActionNames=actions,
@@ -163,10 +172,13 @@ def get_ssm_parameters_by_path_called_with(
 class AWSMocks:
     def __init__(
         self,
-        has_access={"all": True},
+        has_access=None,
         put_parameter_unexpected=False,
         create_existing_params="none",
     ):
+        if has_access is None:
+            has_access = {"all": True}
+
         self.create_existing_params = create_existing_params
         self.has_access = has_access
         self.mocks = None
@@ -315,7 +327,10 @@ def setup_copy(self, mock_application, source, target):
                 ]
             if stage == "target":
 
-                def _create_ssm_mock_with_failing_put_parameter(ssm_client, calls_to_fail_on=[2]):
+                def _create_ssm_mock_with_failing_put_parameter(ssm_client, calls_to_fail_on=None):
+                    if calls_to_fail_on is None:
+                        calls_to_fail_on = [2]
+
                     return_value = ssm_client.put_parameter.return_value
 
                     def mock_put_parameter(*args, **kwargs):