migrate runtime secrets from ssm to aws secrets manager

.github/scripts/verify-runtime-secrets.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+usage() {
+  echo "Usage: $0 <environment> <enable-additional-parameters>" >&2
+  exit 1
+}
+
+[[ $# -eq 2 ]] || usage
+
+environment=$1
+enable_additional_parameters=$2
+
+if [[ "$enable_additional_parameters" != "true" && "$enable_additional_parameters" != "false" ]]; then
+  usage
+fi
+
+check_secret() {
+  local secret_id=$1
+  local secret_metadata
+
+  # Check version metadata without retrieving or printing the secret value.
+  if ! secret_metadata=$(aws secretsmanager describe-secret \
+    --secret-id "$secret_id" \
+    --query 'VersionIdsToStages' \
+    --output json); then
+    echo "::error::Unable to describe required secret: ${secret_id}" >&2
+    exit 1
+  fi
+
+  if ! jq -e 'any(.[]; index("AWSCURRENT") != null)' <<<"$secret_metadata" >/dev/null; then
+    echo "::error::Secret has no AWSCURRENT version: ${secret_id}" >&2
+    exit 1
+  fi
+}
+
+secret_names=(
+  DJANGO_ALGOLIA_WRITE_API_KEY
+  DJANGO_OPEN_AI_SECRET_KEY
+  DJANGO_REDIS_PASSWORD
+  DJANGO_SECRET_KEY
+  DJANGO_SENTRY_DSN
+  DJANGO_SLACK_BOT_TOKEN
+  DJANGO_SLACK_SIGNING_SECRET
+  GITHUB_TOKEN
+  NEXTAUTH_SECRET
+  NEXT_SERVER_GITHUB_CLIENT_SECRET
+)
+
+if [[ "$enable_additional_parameters" == "true" ]]; then
+  secret_names+=(NEST_GITHUB_APP_PRIVATE_KEY SLACK_BOT_TOKEN_T04T40NHX)
+fi
+
+for secret_name in "${secret_names[@]}"; do
+  check_secret "/nest/${environment}/${secret_name}"
+done
+
+check_secret "nest-${environment}-db-credentials"
+
+echo "All required runtime secrets have an AWSCURRENT version."

.github/workflows/run-deploy.yaml

@@ -140,6 +140,7 @@ jobs:
         }}
       FRONTEND_IMAGE: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_REGION }}.amazonaws.com/nest-${{ inputs.environment }}-frontend:${{ inputs.release_version
         }}
+      RUNTIME_SECRETS_MODE: ${{ vars.RUNTIME_SECRETS_MODE || 'prepare' }}
       TF_IN_AUTOMATION: true
       TF_INPUT: false
     permissions:
@@ -163,6 +164,23 @@ jobs:
           role-skip-session-tagging: true
           role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ inputs.aws_role_name }}
 
+      - name: Validate runtime secrets migration mode
+        run: |
+          if [[ "$RUNTIME_SECRETS_MODE" != "prepare" && "$RUNTIME_SECRETS_MODE" != "complete" ]]; then
+            echo "::error::RUNTIME_SECRETS_MODE must be prepare or complete."
+            exit 1
+          fi
+
+      - name: Verify runtime secrets are populated
+        if: env.RUNTIME_SECRETS_MODE == 'complete'
+        env:
+          DEPLOY_ENVIRONMENT: ${{ inputs.environment }}
+          ENABLE_ADDITIONAL_PARAMETERS: ${{ inputs.enable_additional_parameters }}
+        run: |
+          bash .github/scripts/verify-runtime-secrets.sh \
+            "$DEPLOY_ENVIRONMENT" \
+            "$ENABLE_ADDITIONAL_PARAMETERS"
+
       - name: Login to Amazon ECR
         uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64  # v2.1.6
 
@@ -220,6 +238,7 @@ jobs:
             frontend_use_fargate_spot=${{ inputs.frontend_use_fargate_spot }}
             frontend_image_tag="${{ inputs.release_version }}"
             project_name="nest"
+            runtime_secrets_mode="${{ vars.RUNTIME_SECRETS_MODE || 'prepare' }}"
             tasks_use_fargate_spot=${{ inputs.tasks_use_fargate_spot }}
           working-directory: infrastructure/live
 

infrastructure/bootstrap/main.tf

@@ -307,7 +307,7 @@ data "aws_iam_policy_document" "part_two" {
   for_each = local.environments
 
   statement {
-    sid    = "AppAutoscalingManagement"
+    sid    = "AppAutoscalingMgmt"
     effect = "Allow"
     actions = [
       "application-autoscaling:DeleteScalingPolicy",
@@ -365,7 +365,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "ELBManagement"
+    sid    = "ELBMgmt"
     effect = "Allow"
     actions = [
       "elasticloadbalancing:AddTags",
@@ -393,7 +393,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "EventBridgeManagement"
+    sid    = "EventBridgeMgmt"
     effect = "Allow"
     actions = [
       "events:DeleteRule",
@@ -412,7 +412,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "IAMManagement"
+    sid    = "IAMMgmt"
     effect = "Allow"
     actions = [
       "iam:AttachRolePolicy",
@@ -471,7 +471,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "KMSManagement"
+    sid    = "KMSMgmt"
     effect = "Allow"
     actions = [
       "kms:CreateKey",
@@ -525,7 +525,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "S3Management"
+    sid    = "S3Mgmt"
     effect = "Allow"
     actions = [
       "s3:CreateBucket",
@@ -571,7 +571,7 @@ data "aws_iam_policy_document" "part_two" {
   }
 
   statement {
-    sid    = "SecretsManagerManagement"
+    sid    = "SecretsManagerMgmt"
     effect = "Allow"
     actions = [
       "secretsmanager:CreateSecret",
@@ -585,11 +585,14 @@ data "aws_iam_policy_document" "part_two" {
       "secretsmanager:UntagResource",
       "secretsmanager:UpdateSecret",
     ]
-    resources = ["arn:aws:secretsmanager:${var.aws_region}:${data.aws_caller_identity.current.account_id}:secret:${var.project_name}-${each.key}-*"]
+    resources = [
+      "arn:aws:secretsmanager:${var.aws_region}:${data.aws_caller_identity.current.account_id}:secret:${var.project_name}-${each.key}-*",
+      "arn:aws:secretsmanager:${var.aws_region}:${data.aws_caller_identity.current.account_id}:secret:/${var.project_name}/${each.key}/*",
+    ]
   }
 
   statement {
-    sid    = "SSMManagement"
+    sid    = "SSMMgmt"
     effect = "Allow"
     actions = [
       "ssm:AddTagsToResource",

infrastructure/bootstrap/tests/bootstrap.tftest.hcl

@@ -57,6 +57,42 @@ run "test_part_two_policy_size_production" {
   }
 }
 
+run "test_secrets_manager_namespace_staging" {
+  command = plan
+
+  assert {
+    condition = alltrue([
+      strcontains(
+        data.aws_iam_policy_document.part_two["staging"].json,
+        "arn:aws:secretsmanager:${var.aws_region}:160885282306:secret:${var.project_name}-staging-*",
+      ),
+      strcontains(
+        data.aws_iam_policy_document.part_two["staging"].json,
+        "arn:aws:secretsmanager:${var.aws_region}:160885282306:secret:/${var.project_name}/staging/*",
+      ),
+    ])
+    error_message = "The staging Terraform policy must allow management of the staging Secrets Manager namespace."
+  }
+}
+
+run "test_secrets_manager_namespace_production" {
+  command = plan
+
+  assert {
+    condition = alltrue([
+      strcontains(
+        data.aws_iam_policy_document.part_two["production"].json,
+        "arn:aws:secretsmanager:${var.aws_region}:160885282306:secret:${var.project_name}-production-*",
+      ),
+      strcontains(
+        data.aws_iam_policy_document.part_two["production"].json,
+        "arn:aws:secretsmanager:${var.aws_region}:160885282306:secret:/${var.project_name}/production/*",
+      ),
+    ])
+    error_message = "The production Terraform policy must allow management of the production Secrets Manager namespace."
+  }
+}
+
 run "test_minified_json_is_smaller_than_pretty_json" {
   command = plan
 

infrastructure/live/README.md

@@ -123,6 +123,7 @@ No resources.
 | <a name="input_redis_node_type"></a> [redis\_node\_type](#input\_redis\_node\_type) | The node type for the Redis cache. | `string` | `"cache.t3.micro"` | no |
 | <a name="input_redis_num_cache_nodes"></a> [redis\_num\_cache\_nodes](#input\_redis\_num\_cache\_nodes) | The number of cache nodes in the Redis cluster. | `number` | `1` | no |
 | <a name="input_redis_port"></a> [redis\_port](#input\_redis\_port) | The port for the Redis cache. | `number` | `6379` | no |
+| <a name="input_runtime_secrets_mode"></a> [runtime\_secrets\_mode](#input\_runtime\_secrets\_mode) | Runtime secret migration phase: 'prepare' retains SSM injection, while 'complete' uses Secrets Manager. | `string` | n/a | yes |
 | <a name="input_secret_recovery_window_in_days"></a> [secret\_recovery\_window\_in\_days](#input\_secret\_recovery\_window\_in\_days) | The number of days that Secrets Manager waits before it can delete the secret. Set to 0 to delete immediately. | `number` | `7` | no |
 | <a name="input_slack_bot_token_suffix"></a> [slack\_bot\_token\_suffix](#input\_slack\_bot\_token\_suffix) | The Suffix for the Slack bot token. | `string` | `"T04T40NHX"` | no |
 | <a name="input_tasks_use_fargate_spot"></a> [tasks\_use\_fargate\_spot](#input\_tasks\_use\_fargate\_spot) | Whether to use Fargate Spot for ECS tasks. | `bool` | `true` | no |

infrastructure/live/main.tf

@@ -51,6 +51,7 @@ module "backend" {
   container_cpu                   = 1024
   container_memory                = 2048
   container_port                  = 8000
+  container_secrets               = module.parameters.django_container_secrets
   desired_count                   = var.backend_desired_count
   enable_auto_scaling             = var.backend_enable_auto_scaling
   environment                     = var.environment
@@ -59,8 +60,8 @@ module "backend" {
   kms_key_arn                     = module.kms.key_arn
   max_count                       = var.backend_max_count
   min_count                       = var.backend_min_count
-  parameters_arns                 = module.parameters.django_ssm_parameter_arns
   project_name                    = var.project_name
+  secretsmanager_secret_arns      = module.parameters.django_secretsmanager_secret_arns
   security_group_id               = module.security.backend_sg_id
   service_name                    = "backend"
   subnet_ids                      = var.enable_nat_gateway ? module.networking.private_subnet_ids : module.networking.public_subnet_ids
@@ -72,16 +73,18 @@ module "backend" {
 module "cache" {
   source = "../modules/cache"
 
-  common_tags           = local.common_tags
-  environment           = var.environment
-  kms_key_arn           = module.kms.key_arn
-  project_name          = var.project_name
-  redis_engine_version  = var.redis_engine_version
-  redis_node_type       = var.redis_node_type
-  redis_num_cache_nodes = var.redis_num_cache_nodes
-  redis_port            = var.redis_port
-  security_group_ids    = [module.security.redis_sg_id]
-  subnet_ids            = module.networking.private_subnet_ids
+  common_tags                    = local.common_tags
+  environment                    = var.environment
+  kms_key_arn                    = module.kms.key_arn
+  project_name                   = var.project_name
+  redis_engine_version           = var.redis_engine_version
+  redis_node_type                = var.redis_node_type
+  redis_num_cache_nodes          = var.redis_num_cache_nodes
+  redis_port                     = var.redis_port
+  runtime_secrets_mode           = var.runtime_secrets_mode
+  secret_recovery_window_in_days = var.secret_recovery_window_in_days
+  security_group_ids             = [module.security.redis_sg_id]
+  subnet_ids                     = module.networking.private_subnet_ids
 }
 
 module "database" {
@@ -104,6 +107,7 @@ module "database" {
   kms_key_arn                    = module.kms.key_arn
   project_name                   = var.project_name
   proxy_security_group_ids       = [module.security.rds_proxy_sg_id]
+  runtime_secrets_mode           = var.runtime_secrets_mode
   secret_recovery_window_in_days = var.secret_recovery_window_in_days
   security_group_ids             = [module.security.rds_sg_id]
 }
@@ -118,6 +122,7 @@ module "frontend" {
   aws_region                      = var.aws_region
   common_tags                     = local.common_tags
   container_port                  = 3000
+  container_secrets               = module.parameters.frontend_container_secrets
   desired_count                   = var.frontend_desired_count
   enable_auto_scaling             = var.frontend_enable_auto_scaling
   environment                     = var.environment
@@ -126,10 +131,10 @@ module "frontend" {
   kms_key_arn                     = module.kms.key_arn
   max_count                       = var.frontend_max_count
   min_count                       = var.frontend_min_count
-  parameters_arns                 = module.parameters.frontend_ssm_parameter_arns
   project_name                    = var.project_name
   security_group_id               = module.security.frontend_sg_id
   service_name                    = "frontend"
+  secretsmanager_secret_arns      = module.parameters.frontend_secretsmanager_secret_arns
   subnet_ids                      = var.enable_nat_gateway ? module.networking.private_subnet_ids : module.networking.public_subnet_ids
   target_group_arn                = module.alb.frontend_target_group_arn
   use_fargate_spot                = var.frontend_use_fargate_spot
@@ -182,27 +187,32 @@ module "networking" {
 module "parameters" {
   source = "../modules/parameters"
 
-  common_tags                   = local.common_tags
-  db_password_arn               = module.database.db_password_arn
-  django_configuration          = var.django_configuration
-  django_allowed_hosts          = var.domain_name
-  django_allowed_origins        = "https://${var.domain_name}"
-  django_aws_static_bucket_name = module.storage.static_s3_bucket_name
-  django_db_host                = module.database.db_proxy_endpoint
-  django_db_name                = var.db_name
-  django_db_port                = var.db_port
-  django_db_user                = var.db_user
-  django_redis_host             = module.cache.redis_primary_endpoint
-  django_release_version        = var.django_release_version
-  django_settings_module        = var.django_settings_module
-  enable_additional_parameters  = var.enable_additional_parameters
-  environment                   = var.environment
-  next_server_csrf_url          = "https://${var.domain_name}/csrf/"
-  next_server_graphql_url       = "https://${var.domain_name}/graphql/"
-  nextauth_url                  = "https://${var.domain_name}"
-  project_name                  = var.project_name
-  redis_password_arn            = module.cache.redis_password_arn
-  slack_bot_token_suffix        = var.slack_bot_token_suffix
+  common_tags                    = local.common_tags
+  db_password_arn                = module.database.db_password_arn
+  db_credentials_secret_arn      = module.database.db_credentials_secret_arn
+  django_configuration           = var.django_configuration
+  django_allowed_hosts           = var.domain_name
+  django_allowed_origins         = "https://${var.domain_name}"
+  django_aws_static_bucket_name  = module.storage.static_s3_bucket_name
+  django_db_host                 = module.database.db_proxy_endpoint
+  django_db_name                 = var.db_name
+  django_db_port                 = var.db_port
+  django_db_user                 = var.db_user
+  django_redis_host              = module.cache.redis_primary_endpoint
+  django_release_version         = var.django_release_version
+  django_settings_module         = var.django_settings_module
+  enable_additional_parameters   = var.enable_additional_parameters
+  environment                    = var.environment
+  kms_key_arn                    = module.kms.key_arn
+  next_server_csrf_url           = "https://${var.domain_name}/csrf/"
+  next_server_graphql_url        = "https://${var.domain_name}/graphql/"
+  nextauth_url                   = "https://${var.domain_name}"
+  project_name                   = var.project_name
+  redis_password_arn             = module.cache.redis_password_arn
+  redis_password_secret_arn      = module.cache.redis_password_secret_arn
+  runtime_secrets_mode           = var.runtime_secrets_mode
+  secret_recovery_window_in_days = var.secret_recovery_window_in_days
+  slack_bot_token_suffix         = var.slack_bot_token_suffix
 }
 
 module "security" {
@@ -235,7 +245,7 @@ module "tasks" {
   assign_public_ip              = local.assign_public_ip
   aws_region                    = var.aws_region
   common_tags                   = local.common_tags
-  container_parameters_arns     = module.parameters.django_ssm_parameter_arns
+  container_secrets             = module.parameters.django_container_secrets
   ecr_repository_arn            = module.backend.ecr_repository_arn
   ecr_repository_url            = module.backend.ecr_repository_url
   ecs_sg_id                     = module.security.tasks_sg_id
@@ -246,6 +256,7 @@ module "tasks" {
   image_tag                     = var.backend_image_tag
   kms_key_arn                   = module.kms.key_arn
   project_name                  = var.project_name
+  secretsmanager_secret_arns    = module.parameters.django_secretsmanager_secret_arns
   subnet_ids                    = var.enable_nat_gateway ? module.networking.private_subnet_ids : module.networking.public_subnet_ids
   use_fargate_spot              = var.tasks_use_fargate_spot
 }

infrastructure/live/terraform.production.tfvars.example

@@ -25,4 +25,5 @@ frontend_max_count              = 6
 frontend_min_count              = 2
 frontend_use_fargate_spot       = false
 project_name                  = "nest"
+runtime_secrets_mode = "prepare"
 tasks_use_fargate_spot        = false

infrastructure/live/terraform.staging.tfvars.example

@@ -23,4 +23,5 @@ frontend_max_count              = 3
 frontend_min_count              = 1
 frontend_use_fargate_spot       = true
 project_name                  = "nest"
+runtime_secrets_mode = "prepare"
 tasks_use_fargate_spot        = true