diff --git a/code-security.datadog.yml b/code-security.datadog.yml new file mode 100644 index 000000000..c78ff7e06 --- /dev/null +++ b/code-security.datadog.yml @@ -0,0 +1,9 @@ +schema-version: v1.0 +sast: + ruleset-configs: + python-security: + rule-configs: + variable-sql-statement-injection: + ignore-paths: + - "terraform/postgres/manage_users.py" + - "dbt_platform_helper/providers/copilot.py" diff --git a/dbt_platform_helper/providers/load_balancers.py b/dbt_platform_helper/providers/load_balancers.py index 17afcb8fa..141e34c33 100644 --- a/dbt_platform_helper/providers/load_balancers.py +++ b/dbt_platform_helper/providers/load_balancers.py @@ -321,8 +321,11 @@ def create_forward_rule( rule_name: str, priority: int, conditions: list, - additional_tags: list = [], + additional_tags=None, ): + if additional_tags is None: + additional_tags = [] + return self.create_rule( listener_arn=listener_arn, priority=priority, diff --git a/dbt_platform_helper/templates/.copilot/phases/pre_build.sh b/dbt_platform_helper/templates/.copilot/phases/pre_build.sh index 18b19394e..f6fe04aae 100755 --- a/dbt_platform_helper/templates/.copilot/phases/pre_build.sh +++ b/dbt_platform_helper/templates/.copilot/phases/pre_build.sh @@ -5,8 +5,8 @@ set -e if [ -f "./.gitmodules" ]; then echo ".gitmodules file exists. Modifying URLs..." - account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5) - connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2) + account_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d':' -f5) + connection_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d'/' -f2) git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade" git config --global credential.helper '!aws codecommit credential-helper $@' diff --git a/images/conduit/opensearch/entrypoint.sh b/images/conduit/opensearch/entrypoint.sh index 663f89ec5..9e5e53d2b 100644 --- a/images/conduit/opensearch/entrypoint.sh +++ b/images/conduit/opensearch/entrypoint.sh @@ -5,7 +5,7 @@ mkdir -p /root/.opensearch-cli/ function decode_url() { : "${*//+/ }"; echo "${_//%/\\x}"; } -echo $(decode_url "$CONNECTION_SECRET") | gawk '{ +decode_url "$CONNECTION_SECRET" | gawk '{ match($0, /^https:\/\/([A-Za-z0-9_]+):([^@]+)@(.+)$/, arr); print "profiles:" print " - name: connection" @@ -30,7 +30,7 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do TASKS_RUNNING="$(ps -e -o pid,comm,args | awk '{if ($4 != "/entrypoint.sh" && $2 == "bash" && $3 == "bash") {print $1}}' | wc -l | xargs)" if [[ $TASKS_RUNNING == 0 ]]; then - CHECK_COUNT=$(( $CHECK_COUNT + 1 )) + CHECK_COUNT=$(( CHECK_COUNT + 1 )) TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))" echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds" else @@ -39,9 +39,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do fi done -# Trigger CloudFormation stack delete before shutting down -if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then - aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name") -fi - echo "Shutting down" diff --git a/images/conduit/opensearch/shell-profile.sh b/images/conduit/opensearch/shell-profile.sh index 368adf695..0afc55fbb 100644 --- a/images/conduit/opensearch/shell-profile.sh +++ b/images/conduit/opensearch/shell-profile.sh @@ -19,7 +19,7 @@ function os_run_next_command() { echo -n "opensearch => " read echo "Running opensearch-cli $REPLY" - /root/opensearch-cli $REPLY --profile connection + /root/opensearch-cli "$REPLY" --profile connection } os_entry diff --git a/images/conduit/postgres/entrypoint.sh b/images/conduit/postgres/entrypoint.sh index d7095eb63..c1e8c326b 100644 --- a/images/conduit/postgres/entrypoint.sh +++ b/images/conduit/postgres/entrypoint.sh @@ -10,10 +10,10 @@ CLIENT_TASK="psql" while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do sleep $CHECK_INTERVAL - TASKS_RUNNING="$(ps -e -o pid,comm | grep -c "$CLIENT_TASK")" + TASKS_RUNNING="$(pgrep -c "$CLIENT_TASK")" if [[ $TASKS_RUNNING == 0 ]]; then - CHECK_COUNT=$(( $CHECK_COUNT + 1 )) + CHECK_COUNT=$(( CHECK_COUNT + 1 )) TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))" echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds" else @@ -22,9 +22,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do fi done -# Trigger CloudFormation stack delete before shutting down -if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then - aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name") -fi - echo "Shutting down" diff --git a/images/conduit/postgres/shell-profile.sh b/images/conduit/postgres/shell-profile.sh index 989a57edb..c93de0650 100644 --- a/images/conduit/postgres/shell-profile.sh +++ b/images/conduit/postgres/shell-profile.sh @@ -1,7 +1,7 @@ echo "Welcome to the client container for postgres services." -echo $CONNECTION_SECRET | jq -rc '"Connecting to database \"\(.dbname)\" on \"\(.host)\""' +echo "$CONNECTION_SECRET" | jq -rc '"Connecting to database \"\(.dbname)\" on \"\(.host)\""' echo -psql "$(echo $CONNECTION_SECRET | jq -rc '"postgres://\(.username):\(.password)@\(.host):\(.port)/\(.dbname)"')" +psql "$(echo "$CONNECTION_SECRET" | jq -rc '"postgres://\(.username):\(.password)@\(.host):\(.port)/\(.dbname)"')" exit diff --git a/images/conduit/redis/entrypoint.sh b/images/conduit/redis/entrypoint.sh index 06774df3d..f63976856 100644 --- a/images/conduit/redis/entrypoint.sh +++ b/images/conduit/redis/entrypoint.sh @@ -10,10 +10,10 @@ CLIENT_TASK="redis-cli" while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do sleep $CHECK_INTERVAL - TASKS_RUNNING="$(ps -e -o pid,comm | grep -c "$CLIENT_TASK")" + TASKS_RUNNING="$(pgrep -c "$CLIENT_TASK")" if [[ $TASKS_RUNNING == 0 ]]; then - CHECK_COUNT=$(( $CHECK_COUNT + 1 )) + CHECK_COUNT=$(( CHECK_COUNT + 1 )) TIME_TO_SHUTDOWN="$(( (CHECK_NUMBER - CHECK_COUNT) * CHECK_INTERVAL ))" echo "No clients connected, will shutdown in approximately $TIME_TO_SHUTDOWN seconds" else @@ -22,9 +22,4 @@ while [ $CHECK_COUNT -lt $CHECK_NUMBER ]; do fi done -# Trigger CloudFormation stack delete before shutting down -if [[ ! -z $ECS_CONTAINER_METADATA_URI_V4 ]]; then - aws cloudformation delete-stack --stack-name task-$(curl $ECS_CONTAINER_METADATA_URI_V4 -s | jq -r ".Name") -fi - echo "Shutting down" diff --git a/images/conduit/redis/shell-profile.sh b/images/conduit/redis/shell-profile.sh index b15c0ec0c..5b19c9626 100644 --- a/images/conduit/redis/shell-profile.sh +++ b/images/conduit/redis/shell-profile.sh @@ -1,6 +1,6 @@ echo "Welcome to the client container for redis services." echo -redis-cli -u $CONNECTION_SECRET +redis-cli -u "$CONNECTION_SECRET" exit diff --git a/images/tools/database-copy/entrypoint.sh b/images/tools/database-copy/entrypoint.sh index c54d024a6..eb2073f05 100644 --- a/images/tools/database-copy/entrypoint.sh +++ b/images/tools/database-copy/entrypoint.sh @@ -4,52 +4,52 @@ clean_up(){ echo "Cleaning up dump file" rm "${DUMP_FILE_NAME}.sql" echo "Removing dump file from S3" - aws s3 rm s3://${S3_BUCKET_NAME}/"${DUMP_FILE_NAME}.sql" + aws s3 rm s3://"${S3_BUCKET_NAME}"/"${DUMP_FILE_NAME}.sql" exit_code=$? - if [ ${exit_code} -ne 0 ] + if [ "${exit_code}" -ne 0 ] then echo "Aborting data load: Clean up failed" - exit $exit_code + exit "${exit_code}" fi } handle_errors(){ exit_code=$1 message=$2 - if [ ${exit_code} -ne 0 ] + if [ "${exit_code}" -ne 0 ] then clean_up echo "Aborting data load: {$message}" - exit $exit_code + exit "${exit_code}" fi } echo "Detecting postgres version" -PG_VERSION="$(psql "$DB_CONNECTION_STRING" -c "SELECT version();" | grep "PostgreSQL" | awk '{print $2;}' | awk -F"." '{print $1}')" -if [ -z "$PG_VERSION" ]; then +PG_VERSION="$(psql "${DB_CONNECTION_STRING}" -c "SELECT version();" | grep "PostgreSQL" | awk '{print $2;}' | awk -F"." '{print $1}')" +if [ -z "${PG_VERSION}" ]; then PG_VERSION="17" fi -echo "Detected postgres version $PG_VERSION" +echo "Detected postgres version ${PG_VERSION}" if [ "${DATA_COPY_OPERATION:-DUMP}" != "LOAD" ] then echo "Starting data dump" - /usr/lib/postgresql/$PG_VERSION/bin/pg_dump --no-owner --no-acl --format c "${DB_CONNECTION_STRING}" > "${DUMP_FILE_NAME}.sql" + /usr/lib/postgresql/"${PG_VERSION}"/bin/pg_dump --no-owner --no-acl --format c "${DB_CONNECTION_STRING}" > "${DUMP_FILE_NAME}.sql" exit_code=$? - if [ ${exit_code} -ne 0 ] + if [ "${exit_code}" -ne 0 ] then echo "Aborting data dump" - exit $exit_code + exit "${exit_code}" fi - aws s3 cp "${DUMP_FILE_NAME}.sql" s3://${S3_BUCKET_NAME}/ + aws s3 cp "${DUMP_FILE_NAME}.sql" s3://"${S3_BUCKET_NAME}"/ exit_code=$? - if [ ${exit_code} -ne 0 ] + if [ "${exit_code}" -ne 0 ] then echo "Aborting data dump" - exit $exit_code + exit "${exit_code}" fi echo "Stopping data dump" @@ -57,7 +57,7 @@ else echo "Starting data load" echo "Copying data dump from S3" - aws s3 cp s3://${S3_BUCKET_NAME}/"${DUMP_FILE_NAME}.sql" "${DUMP_FILE_NAME}.sql" + aws s3 cp s3://"${S3_BUCKET_NAME}"/"${DUMP_FILE_NAME}.sql" "${DUMP_FILE_NAME}.sql" handle_errors $? "Copy failed" @@ -76,7 +76,7 @@ else CONFIG_FILE="${SERVICE_NAME}.desired_count" echo "${COUNT}" > "${CONFIG_FILE}" - echo ${SERVICE_NAME} + echo "${SERVICE_NAME}" UPDATE_DATA=$(aws ecs update-service --cluster "${ECS_CLUSTER}" --service "${service}" --desired-count 0) handle_errors $? "Failed to update service ${SERVICE_NAME}" echo "${UPDATE_DATA}" | jq -r '" Desired Count: \(.service.desiredCount)\n Running Count: \(.service.runningCount)"' @@ -84,12 +84,12 @@ else done echo "Clearing down the database prior to loading new data" - /usr/lib/postgresql/$PG_VERSION/bin/psql "${DB_CONNECTION_STRING}" -f /clear_db.sql + /usr/lib/postgresql/"${PG_VERSION}"/bin/psql "${DB_CONNECTION_STRING}" -f /clear_db.sql handle_errors $? "Clear down failed" echo "Restoring data from dump file" - /usr/lib/postgresql/$PG_VERSION/bin/pg_restore --format c --dbname "${DB_CONNECTION_STRING}" "${DUMP_FILE_NAME}.sql" + /usr/lib/postgresql/"${PG_VERSION}"/bin/pg_restore --format c --dbname "${DB_CONNECTION_STRING}" "${DUMP_FILE_NAME}.sql" handle_errors $? "Restore failed" for service in ${SERVICES} @@ -98,7 +98,7 @@ else COUNT=$(cat "${CONFIG_FILE}") SERVICE_NAME=$(basename "${service}") echo "Scaling up services" - echo ${SERVICE_NAME} + echo "${SERVICE_NAME}" UPDATE_DATA=$(aws ecs update-service --cluster "${ECS_CLUSTER}" --service "${service}" --desired-count "${COUNT}") handle_errors $? "Failed to update service ${SERVICE_NAME}" echo "${UPDATE_DATA}" | jq -r '" Desired Count: \(.service.desiredCount)\n Running Count: \(.service.runningCount)"' diff --git a/publish_to_pypi.sh b/publish_to_pypi.sh index bb59583ba..ad2e5dd70 100755 --- a/publish_to_pypi.sh +++ b/publish_to_pypi.sh @@ -13,14 +13,13 @@ if ! python utils/check_pypi.py then echo Building Python package poetry build - echo Publishing Python package ${VERSION} - poetry config pypi-token.pypi ${PYPI_TOKEN} + echo Publishing Python package "${VERSION}" + poetry config pypi-token.pypi "${PYPI_TOKEN}" poetry publish echo Checking the package has reached PyPI python utils/check_pypi.py --max-attempts 20 echo -e "\nSending slack notification" - VERSION_NUMBER=($VERSION) - poetry run python utils/notify/publish_notification.py --publish-version ${VERSION_NUMBER[1]} + poetry run python utils/notify/publish_notification.py --publish-version "${VERSION#* }" else - echo ${VERSION} of the package has already been published + echo "${VERSION}" of the package has already been published fi diff --git a/terraform/application-load-balancer/lambda_function/rotate_secret/secret_rotator.py b/terraform/application-load-balancer/lambda_function/rotate_secret/secret_rotator.py index 6fb4054c5..559e8d425 100644 --- a/terraform/application-load-balancer/lambda_function/rotate_secret/secret_rotator.py +++ b/terraform/application-load-balancer/lambda_function/rotate_secret/secret_rotator.py @@ -546,7 +546,7 @@ def set_secret(self, service_client, arn, token): f"Failed to update resources CloudFront Distro Id {distro['Id']} , WAF WebACL Id {self.waf_acl_id}" ) from e - def run_test_secret(self, service_client, arn, token, test_domains=[]): + def run_test_secret(self, service_client, arn, token, test_domains=None): """ Test the secret This method validates that the AWSPENDING secret works in the service. @@ -555,6 +555,9 @@ def run_test_secret(self, service_client, arn, token, test_domains=[]): 1. Attempts to send a Slack notification (notification failure won't stop the rotation process) 2. If Lambda event contains key TestDomains and provided domains to test, then you can trigger a Slack notification to the configured Slack channel """ + if test_domains is None: + test_domains = [] + test_failures = [] # Check for TestDomains key in the Lambda event - currently only used in console to test Slack message is emitted diff --git a/terraform/scripts/install_platform_helper.sh b/terraform/scripts/install_platform_helper.sh index 63a26bfad..f56c86656 100755 --- a/terraform/scripts/install_platform_helper.sh +++ b/terraform/scripts/install_platform_helper.sh @@ -4,7 +4,7 @@ echo -e "Installing dependencies" pip install poetry --quiet poetry install --quiet echo -e "\nBuild platform-helper\n" -poetry build --no-interaction --format sdist --no-ansi --local-version ${PLATFORM_HELPER_VERSION} --quiet +poetry build --no-interaction --format sdist --no-ansi --local-version "${PLATFORM_HELPER_VERSION}" --quiet echo -e "\nInstall platform-helper\n" most_recent_built_wheel_package=$(ls -t1 dist | grep ".whl" | head -1) pip install "dist/${most_recent_built_wheel_package}" --quiet diff --git a/tests/platform_helper/expected_files/.copilot/phases/pre_build.sh b/tests/platform_helper/expected_files/.copilot/phases/pre_build.sh index 18b19394e..f6fe04aae 100755 --- a/tests/platform_helper/expected_files/.copilot/phases/pre_build.sh +++ b/tests/platform_helper/expected_files/.copilot/phases/pre_build.sh @@ -5,8 +5,8 @@ set -e if [ -f "./.gitmodules" ]; then echo ".gitmodules file exists. Modifying URLs..." - account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5) - connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2) + account_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d':' -f5) + connection_id=$(echo "$CODESTAR_CONNECTION_ARN" | cut -d'/' -f2) git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade" git config --global credential.helper '!aws codecommit credential-helper $@' diff --git a/tests/platform_helper/integration/test_secrets.py b/tests/platform_helper/integration/test_secrets.py index 27143967f..6546e562b 100644 --- a/tests/platform_helper/integration/test_secrets.py +++ b/tests/platform_helper/integration/test_secrets.py @@ -95,8 +95,11 @@ def put_parameter_called_with( description="", secret="SECRET", platform="platform", - tags=[], + tags=None, ): + if tags is None: + tags = [] + called_with = dict( Name=f"/{platform}/test-application/{env}/secrets/{secret.upper()}", Value=str(value), @@ -119,7 +122,10 @@ def put_parameter_called_with( return called_with @staticmethod - def put_parameter_copied_called_with(platform, source, target, secret, tags=[]): + def put_parameter_copied_called_with(platform, source, target, secret, tags=None): + if tags is None: + tags = [] + tags = [{"Key": "copied-from", "Value": source}] + tags description = f"Copied from {source} environment." return AWSTestFixtures.put_parameter_called_with( @@ -132,7 +138,10 @@ def put_parameter_copied_called_with(platform, source, target, secret, tags=[]): ) @staticmethod - def simulate_principal_policy_called_with(account_id, role_name, actions=["ssm:PutParameter"]): + def simulate_principal_policy_called_with(account_id, role_name, actions=None): + if actions is None: + actions = ["ssm:PutParameter"] + return dict( PolicySourceArn=f"arn:aws:iam::{account_id}:role/aws-reserved/sso.amazonaws.com/eu-west-2/{role_name}", ActionNames=actions, @@ -163,10 +172,13 @@ def get_ssm_parameters_by_path_called_with( class AWSMocks: def __init__( self, - has_access={"all": True}, + has_access=None, put_parameter_unexpected=False, create_existing_params="none", ): + if has_access is None: + has_access = {"all": True} + self.create_existing_params = create_existing_params self.has_access = has_access self.mocks = None @@ -315,7 +327,10 @@ def setup_copy(self, mock_application, source, target): ] if stage == "target": - def _create_ssm_mock_with_failing_put_parameter(ssm_client, calls_to_fail_on=[2]): + def _create_ssm_mock_with_failing_put_parameter(ssm_client, calls_to_fail_on=None): + if calls_to_fail_on is None: + calls_to_fail_on = [2] + return_value = ssm_client.put_parameter.return_value def mock_put_parameter(*args, **kwargs):