diff --git a/.trivyignore.yaml b/.trivyignore.yaml index 5079ce5db1..fdcf518131 100644 --- a/.trivyignore.yaml +++ b/.trivyignore.yaml @@ -14,6 +14,8 @@ misconfigurations: - docker/osv-scanner/Dockerfile - docker/semgrep/Dockerfile - docker/trivy/Dockerfile + # Container runs as non-root (user 65532) via the ECS task definition; the image is a thin version-pin for dependabot. + - docker/victoriametrics/Dockerfile - docker/zap/Dockerfile - id: AVD-AWS-0053 # Public ALB intentional for nest.owasp.dev paths: diff --git a/docker-compose/local/compose.o11y.yaml b/docker-compose/local/compose.o11y.yaml index 3873f69c7f..ccd2acc025 100644 --- a/docker-compose/local/compose.o11y.yaml +++ b/docker-compose/local/compose.o11y.yaml @@ -35,7 +35,9 @@ services: command: - -retentionPeriod=5y - -storageDataPath=/data - image: victoriametrics/victoria-metrics:v1.145.0@sha256:c014fb5a711d38cb24fd0673197592cd1394bb903dbb16aea565620c9c8a3d70 + build: + context: ../../docker/victoriametrics + dockerfile: Dockerfile healthcheck: interval: 5s retries: 5 diff --git a/docker/victoriametrics/Dockerfile b/docker/victoriametrics/Dockerfile new file mode 100644 index 0000000000..6fbce0da07 --- /dev/null +++ b/docker/victoriametrics/Dockerfile @@ -0,0 +1 @@ +FROM victoriametrics/victoria-metrics:v1.145.0@sha256:c014fb5a711d38cb24fd0673197592cd1394bb903dbb16aea565620c9c8a3d70 diff --git a/infrastructure/live/README.md b/infrastructure/live/README.md index e34c754f6b..52bb4793c8 100644 --- a/infrastructure/live/README.md +++ b/infrastructure/live/README.md @@ -59,6 +59,7 @@ No providers. | [frontend\_build\_cache](#module\_frontend\_build\_cache) | ../modules/ecr-cache | n/a | | [kms](#module\_kms) | ../modules/kms | n/a | | [networking](#module\_networking) | ../modules/networking | n/a | +| [observability](#module\_observability) | ../modules/observability | n/a | | [parameters](#module\_parameters) | ../modules/parameters | n/a | | [security](#module\_security) | ../modules/security | n/a | | [storage](#module\_storage) | ../modules/storage | n/a | @@ -101,6 +102,7 @@ No resources. | [enable\_additional\_parameters](#input\_enable\_additional\_parameters) | Whether to enable additional parameters (e.g. for production). | `bool` | `false` | no | | [enable\_cron\_tasks](#input\_enable\_cron\_tasks) | Whether to enable scheduled cron tasks. | `bool` | n/a | yes | | [enable\_nat\_gateway](#input\_enable\_nat\_gateway) | Whether to enable a NAT Gateway. | `bool` | `true` | no | +| [enable\_observability](#input\_enable\_observability) | Whether to create the observability stack. | `bool` | `false` | no | | [enable\_rds\_proxy](#input\_enable\_rds\_proxy) | Whether to create an RDS proxy. | `bool` | `false` | no | | [enable\_vpc\_cloudwatch\_logs\_endpoint](#input\_enable\_vpc\_cloudwatch\_logs\_endpoint) | Whether to create CloudWatch Logs VPC endpoint. | `bool` | `false` | no | | [enable\_vpc\_ecr\_api\_endpoint](#input\_enable\_vpc\_ecr\_api\_endpoint) | Whether to create ECR API VPC endpoint. | `bool` | `false` | no | diff --git a/infrastructure/live/main.tf b/infrastructure/live/main.tf index 04178f5f9f..e18090b398 100644 --- a/infrastructure/live/main.tf +++ b/infrastructure/live/main.tf @@ -21,7 +21,8 @@ locals { ManagedBy = "Terraform" Project = var.project_name } - fixtures_bucket_name = coalesce(var.fixtures_bucket_name, "${var.project_name}-${var.environment}-fixtures") + fixtures_bucket_name = coalesce(var.fixtures_bucket_name, "${var.project_name}-${var.environment}-fixtures") + observability_vm_image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("${path.root}/../../docker/victoriametrics/Dockerfile"))[0] } module "alb" { @@ -179,6 +180,27 @@ module "networking" { vpc_cidr = var.vpc_cidr } +module "observability" { + count = var.enable_observability ? 1 : 0 + source = "../modules/observability" + + app_security_group_ids = [ + module.security.backend_sg_id, + module.security.frontend_sg_id, + module.security.tasks_sg_id, + ] + assign_public_ip = local.assign_public_ip + aws_region = var.aws_region + common_tags = local.common_tags + environment = var.environment + kms_key_arn = module.kms.key_arn + project_name = var.project_name + # TODO(#5429): Use private_subnet_ids unconditionally once NAT is enabled in all environments. + subnet_ids = var.enable_nat_gateway ? module.networking.private_subnet_ids : module.networking.public_subnet_ids + vm_image = local.observability_vm_image + vpc_id = module.networking.vpc_id +} + module "parameters" { source = "../modules/parameters" diff --git a/infrastructure/live/variables.tf b/infrastructure/live/variables.tf index 7bedd26ed4..3fe0115074 100644 --- a/infrastructure/live/variables.tf +++ b/infrastructure/live/variables.tf @@ -193,6 +193,12 @@ variable "enable_nat_gateway" { default = true } +variable "enable_observability" { + description = "Whether to create the observability stack." + type = bool + default = false +} + variable "enable_rds_proxy" { description = "Whether to create an RDS proxy." type = bool diff --git a/infrastructure/modules/observability/.terraform.lock.hcl b/infrastructure/modules/observability/.terraform.lock.hcl new file mode 100644 index 0000000000..6656dd31ca --- /dev/null +++ b/infrastructure/modules/observability/.terraform.lock.hcl @@ -0,0 +1,28 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.53.0" + constraints = "~> 6.53.0" + hashes = [ + "h1:UFEhEEFJcR/pAOZcwdR11gN9W3X8VwvSl1IS0vbj2G0=", + "h1:bvlWCSuVJQshwuA/vJONdjEUIzGdsW3uSfLxoP9RnIw=", + "h1:nZ85OLLO0sNw/76mgQQQmnOodiQGaaGVxq1NaV3ol4g=", + "zh:0757ce9d5a30e8225521857924f5d6c49e5885fd9e309e56193c9c9920b3f8f0", + "zh:10ddb3a20e0779788e8002bc67d184eb166dec70f9d07220bfc55e15c3e5e206", + "zh:1e943f59a8c3f3b04f09fd5d967fef2518e0ce3ec4959e31f5d7f820660e8524", + "zh:21d55ad3b28f48c6dd34faceeb1fd1fe2ec3b1aaecfc5eb0e0841884f6520d69", + "zh:2e551bc0ff29ea608a99f63d899afbdf2f23c14035f27dc0c0bdca60209d9575", + "zh:581db963564364200426f2a4be866470a975cde6a4bd6fd09362ec2bab119cbb", + "zh:71a7abc88e16ccbe4410fc4966bd35fae9e0161c1385a9b3c19800e3db626b01", + "zh:8767767776c45590d4cc74b4c37f6377e05ce75039a9c2e834de0bbcc29f7a1a", + "zh:99c8e37beb9b1017f67c41e402031559b745fa797848f1a4ef8b40421dee3f05", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ba1477eda2f3ee51846492449338146f80806088d2952c08c2f8af874550a75d", + "zh:c389f68ba4d39fa2bf8a54d4af7f7b2132da2a964c260892c1dd1a89b67e61f8", + "zh:d7381490a637a1fa45769d0ee6b3e0578cce488d078077ec63d9090113784f2a", + "zh:df3b82fb1a675fc6548fac2fff10a8741efa5bcb5a53d01d8f0659179a2f717f", + "zh:e5d78b2ac3dc5477cf21d7cd4c8b9e3d944ff091e90b62c1b2ea9cc7c7eef57c", + "zh:fd224078287c6d82de2ec30ea03d3d550c2c554e372b234f7a6ac2b07c23dba0", + ] +} diff --git a/infrastructure/modules/observability/README.md b/infrastructure/modules/observability/README.md new file mode 100644 index 0000000000..9ebb3e1578 --- /dev/null +++ b/infrastructure/modules/observability/README.md @@ -0,0 +1,69 @@ + +## Requirements + +| Name | Version | +| ---- | ------- | +| [terraform](#requirement\_terraform) | ~> 1.15.0 | +| [aws](#requirement\_aws) | ~> 6.53.0 | + +## Providers + +| Name | Version | +| ---- | ------- | +| [aws](#provider\_aws) | 6.53.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +| ---- | ---- | +| [aws_cloudwatch_log_group.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | +| [aws_ecs_cluster.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster) | resource | +| [aws_ecs_cluster_capacity_providers.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_cluster_capacity_providers) | resource | +| [aws_ecs_service.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource | +| [aws_ecs_task_definition.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource | +| [aws_efs_access_point.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_access_point) | resource | +| [aws_efs_file_system.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_file_system) | resource | +| [aws_efs_mount_target.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/efs_mount_target) | resource | +| [aws_iam_policy.ecs_task_execution_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.ecs_task_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.ecs_task_execution_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_security_group.efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group.vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.efs_from_vm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_egress_https](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_ingest_from_apps](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.vm_to_efs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +| ---- | ----------- | ---- | ------- | :------: | +| [app\_security\_group\_ids](#input\_app\_security\_group\_ids) | Security group IDs of the application tasks allowed to send metrics to VictoriaMetrics. | `list(string)` | n/a | yes | +| [assign\_public\_ip](#input\_assign\_public\_ip) | Whether to assign a public IP to the VictoriaMetrics task. | `bool` | `false` | no | +| [aws\_region](#input\_aws\_region) | The AWS region where the module is deployed. | `string` | n/a | yes | +| [common\_tags](#input\_common\_tags) | A map of common tags to apply to all resources. | `map(string)` | `{}` | no | +| [environment](#input\_environment) | The environment (e.g., staging, production). | `string` | n/a | yes | +| [kms\_key\_arn](#input\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the EFS file system. | `string` | n/a | yes | +| [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The number of days to retain VictoriaMetrics container logs. | `number` | `90` | no | +| [project\_name](#input\_project\_name) | The name of the project. | `string` | n/a | yes | +| [subnet\_ids](#input\_subnet\_ids) | The private subnet IDs for the EFS mount targets and the VictoriaMetrics task. | `list(string)` | n/a | yes | +| [vm\_cpu](#input\_vm\_cpu) | The CPU units for the VictoriaMetrics Fargate task. | `number` | `512` | no | +| [vm\_desired\_count](#input\_vm\_desired\_count) | The number of VictoriaMetrics tasks to run (0 or 1; it is a single-node store). | `number` | `1` | no | +| [vm\_image](#input\_vm\_image) | The VictoriaMetrics container image (including digest). | `string` | n/a | yes | +| [vm\_memory](#input\_vm\_memory) | The memory (in MiB) for the VictoriaMetrics Fargate task. | `number` | `1024` | no | +| [vm\_port](#input\_vm\_port) | The port VictoriaMetrics listens on for ingest and queries. | `number` | `8428` | no | +| [vm\_retention\_period](#input\_vm\_retention\_period) | The VictoriaMetrics data retention period. A value without a suffix is in months, so the default "12" means 12 months (duration suffixes like 1y, 30d, 1w are also supported). | `string` | `"12"` | no | +| [vpc\_id](#input\_vpc\_id) | The VPC ID where the VictoriaMetrics security group is created. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +| ---- | ----------- | +| [efs\_file\_system\_id](#output\_efs\_file\_system\_id) | The ID of the EFS file system backing VictoriaMetrics storage. | +| [vm\_cluster\_name](#output\_vm\_cluster\_name) | The name of the ECS cluster running VictoriaMetrics. | +| [vm\_security\_group\_id](#output\_vm\_security\_group\_id) | The ID of the VictoriaMetrics security group. | + diff --git a/infrastructure/modules/observability/main.tf b/infrastructure/modules/observability/main.tf new file mode 100644 index 0000000000..e1e3c3a93e --- /dev/null +++ b/infrastructure/modules/observability/main.tf @@ -0,0 +1,287 @@ +terraform { + required_version = "~> 1.15.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.53.0" + } + } +} + +locals { + name_prefix = "${var.project_name}-${var.environment}-observability" + + vm_container_definition = { + command = [ + "-storageDataPath=/data", + "-retentionPeriod=${var.vm_retention_period}", + "-httpListenAddr=:${var.vm_port}", + ] + essential = true + healthCheck = { + command = ["CMD-SHELL", "wget --spider -q http://localhost:${var.vm_port}/health || exit 1"] + interval = 30 + retries = 3 + startPeriod = 30 + timeout = 5 + } + image = var.vm_image + logConfiguration = { + logDriver = "awslogs" + options = { + "awslogs-group" = aws_cloudwatch_log_group.vm.name + "awslogs-region" = var.aws_region + "awslogs-stream-prefix" = "ecs" + } + } + mountPoints = [ + { + containerPath = "/data" + readOnly = false + sourceVolume = "vm-data" + } + ] + name = "victoriametrics" + portMappings = [ + { + containerPort = var.vm_port + hostPort = var.vm_port + protocol = "tcp" + } + ] + user = "65532" + } +} + +resource "aws_security_group" "vm" { + description = "Security group for the VictoriaMetrics task" + name = "${local.name_prefix}-vm-sg" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm-sg" + }) + vpc_id = var.vpc_id +} + +resource "aws_security_group_rule" "vm_ingest_from_apps" { + count = length(var.app_security_group_ids) + + description = "Allow metrics ingest and queries from application tasks" + from_port = var.vm_port + protocol = "tcp" + security_group_id = aws_security_group.vm.id + source_security_group_id = var.app_security_group_ids[count.index] + to_port = var.vm_port + type = "ingress" +} + +resource "aws_security_group_rule" "vm_egress_https" { + cidr_blocks = ["0.0.0.0/0"] + description = "Allow HTTPS egress for container image pulls" + from_port = 443 + protocol = "tcp" + security_group_id = aws_security_group.vm.id + to_port = 443 + type = "egress" +} + +resource "aws_security_group_rule" "vm_to_efs" { + description = "Allow NFS to the observability EFS" + from_port = 2049 + protocol = "tcp" + security_group_id = aws_security_group.vm.id + source_security_group_id = aws_security_group.efs.id + to_port = 2049 + type = "egress" +} + +resource "aws_security_group" "efs" { + description = "Security group for the observability EFS file system" + name = "${local.name_prefix}-efs-sg" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-efs-sg" + }) + vpc_id = var.vpc_id +} + +resource "aws_security_group_rule" "efs_from_vm" { + description = "Allow NFS from the VictoriaMetrics task" + from_port = 2049 + protocol = "tcp" + security_group_id = aws_security_group.efs.id + source_security_group_id = aws_security_group.vm.id + to_port = 2049 + type = "ingress" +} + +resource "aws_efs_file_system" "vm" { + encrypted = true + kms_key_id = var.kms_key_arn + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm" + }) + + lifecycle { + prevent_destroy = true + } +} + +resource "aws_efs_mount_target" "vm" { + count = length(var.subnet_ids) + + file_system_id = aws_efs_file_system.vm.id + security_groups = [aws_security_group.efs.id] + subnet_id = var.subnet_ids[count.index] +} + +resource "aws_efs_access_point" "vm" { + file_system_id = aws_efs_file_system.vm.id + + posix_user { + gid = 65532 + uid = 65532 + } + + root_directory { + path = "/victoriametrics" + + creation_info { + owner_gid = 65532 + owner_uid = 65532 + permissions = "0755" + } + } + + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-vm" + }) +} + +resource "aws_cloudwatch_log_group" "vm" { + kms_key_id = var.kms_key_arn + name = "/aws/ecs/${local.name_prefix}" + retention_in_days = var.log_retention_in_days + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-logs" + }) +} + +resource "aws_ecs_cluster" "vm" { + name = "${local.name_prefix}-cluster" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-cluster" + }) + + setting { + name = "containerInsights" + value = "enabled" + } +} + +resource "aws_ecs_cluster_capacity_providers" "vm" { + capacity_providers = ["FARGATE"] + cluster_name = aws_ecs_cluster.vm.name + + default_capacity_provider_strategy { + base = 0 + capacity_provider = "FARGATE" + weight = 1 + } +} + +resource "aws_iam_role" "ecs_task_execution_role" { + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = "ecs-tasks.amazonaws.com" + } + } + ] + }) + name = "${local.name_prefix}-execution-role" + tags = var.common_tags +} + +resource "aws_iam_policy" "ecs_task_execution_policy" { + description = "Policy for the VictoriaMetrics ECS task execution - CloudWatch Logs access." + name = "${local.name_prefix}-execution-policy" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + Effect = "Allow" + Resource = "${aws_cloudwatch_log_group.vm.arn}:*" + } + ] + }) +} + +resource "aws_iam_role_policy_attachment" "ecs_task_execution_policy_attachment" { + policy_arn = aws_iam_policy.ecs_task_execution_policy.arn + role = aws_iam_role.ecs_task_execution_role.name +} + +resource "aws_ecs_task_definition" "vm" { + container_definitions = jsonencode([local.vm_container_definition]) + cpu = var.vm_cpu + execution_role_arn = aws_iam_role.ecs_task_execution_role.arn + family = local.name_prefix + memory = var.vm_memory + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + runtime_platform { + cpu_architecture = "ARM64" + operating_system_family = "LINUX" + } + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-task-def" + }) + + volume { + name = "vm-data" + + efs_volume_configuration { + file_system_id = aws_efs_file_system.vm.id + transit_encryption = "ENABLED" + + authorization_config { + access_point_id = aws_efs_access_point.vm.id + } + } + } +} + +resource "aws_ecs_service" "vm" { + cluster = aws_ecs_cluster.vm.id + deployment_maximum_percent = 100 + deployment_minimum_healthy_percent = 0 + desired_count = var.vm_desired_count + name = "${local.name_prefix}-service" + tags = merge(var.common_tags, { + Name = "${local.name_prefix}-service" + }) + task_definition = aws_ecs_task_definition.vm.arn + + capacity_provider_strategy { + base = 0 + capacity_provider = "FARGATE" + weight = 1 + } + + network_configuration { + assign_public_ip = var.assign_public_ip + security_groups = [aws_security_group.vm.id] + subnets = var.subnet_ids + } + + depends_on = [aws_efs_mount_target.vm] +} diff --git a/infrastructure/modules/observability/outputs.tf b/infrastructure/modules/observability/outputs.tf new file mode 100644 index 0000000000..df41aebd2c --- /dev/null +++ b/infrastructure/modules/observability/outputs.tf @@ -0,0 +1,14 @@ +output "efs_file_system_id" { + description = "The ID of the EFS file system backing VictoriaMetrics storage." + value = aws_efs_file_system.vm.id +} + +output "vm_cluster_name" { + description = "The name of the ECS cluster running VictoriaMetrics." + value = aws_ecs_cluster.vm.name +} + +output "vm_security_group_id" { + description = "The ID of the VictoriaMetrics security group." + value = aws_security_group.vm.id +} diff --git a/infrastructure/modules/observability/tests/integration.tftest.hcl b/infrastructure/modules/observability/tests/integration.tftest.hcl new file mode 100644 index 0000000000..2602d48641 --- /dev/null +++ b/infrastructure/modules/observability/tests/integration.tftest.hcl @@ -0,0 +1,77 @@ +provider "aws" { + access_key = "test" + region = "us-east-1" + s3_use_path_style = true + secret_key = "test" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_requesting_account_id = true +} + +variables { + aws_region = "us-east-1" + common_tags = { Environment = "test", Project = "nest" } + environment = "test" + project_name = "nest" + vm_image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("../../../docker/victoriametrics/Dockerfile"))[0] +} + +run "setup" { + command = apply + + module { + source = "./tests/setup" + } +} + +run "observability_integration_apply" { + command = apply + + variables { + app_security_group_ids = run.setup.app_security_group_ids + kms_key_arn = run.setup.kms_key_arn + subnet_ids = run.setup.subnet_ids + vm_desired_count = 0 + vpc_id = run.setup.vpc_id + } + + assert { + condition = can(aws_efs_file_system.vm.id) + error_message = "EFS file system was not created." + } + + assert { + condition = aws_efs_file_system.vm.encrypted == true + error_message = "EFS must be encrypted at rest." + } + + assert { + condition = length(aws_efs_mount_target.vm) == length(var.subnet_ids) + error_message = "There must be one EFS mount target per subnet." + } + + assert { + condition = aws_security_group_rule.efs_from_vm.source_security_group_id == aws_security_group.vm.id + error_message = "EFS ingress must come only from the VictoriaMetrics security group." + } + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].file_system_id == aws_efs_file_system.vm.id + error_message = "The vm-data volume must reference the module's EFS file system." + } + + assert { + condition = can(aws_efs_access_point.vm.id) + error_message = "EFS access point was not created." + } + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].authorization_config[0].access_point_id == aws_efs_access_point.vm.id + error_message = "The vm-data volume must mount through the EFS access point (for non-root UID enforcement)." + } + + assert { + condition = can(aws_ecs_service.vm.id) + error_message = "ECS service was not created." + } +} diff --git a/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl b/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl new file mode 100644 index 0000000000..af840c017f --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/.terraform.lock.hcl @@ -0,0 +1,29 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "6.53.0" + constraints = "~> 6.53.0" + hashes = [ + "h1:UFEhEEFJcR/pAOZcwdR11gN9W3X8VwvSl1IS0vbj2G0=", + "h1:bvlWCSuVJQshwuA/vJONdjEUIzGdsW3uSfLxoP9RnIw=", + "h1:nZ85OLLO0sNw/76mgQQQmnOodiQGaaGVxq1NaV3ol4g=", + "h1:sSfqLt0XIbqfTvOSZr2gkyiv1Ysg6uuiVXFd/btmsoY=", + "zh:0757ce9d5a30e8225521857924f5d6c49e5885fd9e309e56193c9c9920b3f8f0", + "zh:10ddb3a20e0779788e8002bc67d184eb166dec70f9d07220bfc55e15c3e5e206", + "zh:1e943f59a8c3f3b04f09fd5d967fef2518e0ce3ec4959e31f5d7f820660e8524", + "zh:21d55ad3b28f48c6dd34faceeb1fd1fe2ec3b1aaecfc5eb0e0841884f6520d69", + "zh:2e551bc0ff29ea608a99f63d899afbdf2f23c14035f27dc0c0bdca60209d9575", + "zh:581db963564364200426f2a4be866470a975cde6a4bd6fd09362ec2bab119cbb", + "zh:71a7abc88e16ccbe4410fc4966bd35fae9e0161c1385a9b3c19800e3db626b01", + "zh:8767767776c45590d4cc74b4c37f6377e05ce75039a9c2e834de0bbcc29f7a1a", + "zh:99c8e37beb9b1017f67c41e402031559b745fa797848f1a4ef8b40421dee3f05", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:ba1477eda2f3ee51846492449338146f80806088d2952c08c2f8af874550a75d", + "zh:c389f68ba4d39fa2bf8a54d4af7f7b2132da2a964c260892c1dd1a89b67e61f8", + "zh:d7381490a637a1fa45769d0ee6b3e0578cce488d078077ec63d9090113784f2a", + "zh:df3b82fb1a675fc6548fac2fff10a8741efa5bcb5a53d01d8f0659179a2f717f", + "zh:e5d78b2ac3dc5477cf21d7cd4c8b9e3d944ff091e90b62c1b2ea9cc7c7eef57c", + "zh:fd224078287c6d82de2ec30ea03d3d550c2c554e372b234f7a6ac2b07c23dba0", + ] +} diff --git a/infrastructure/modules/observability/tests/setup/main.tf b/infrastructure/modules/observability/tests/setup/main.tf new file mode 100644 index 0000000000..7bf5e3a170 --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/main.tf @@ -0,0 +1,37 @@ +terraform { + required_version = "~> 1.15.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.53.0" + } + } +} + +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_vpc" "main" { + cidr_block = "10.0.0.0/16" +} + +resource "aws_subnet" "main" { + count = 2 + + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.main.cidr_block, 8, count.index) + vpc_id = aws_vpc.main.id +} + +resource "aws_security_group" "app" { + count = 3 + + name = "nest-test-app-${count.index}" + vpc_id = aws_vpc.main.id +} + +resource "aws_kms_key" "main" { + description = "Test key for the observability integration tests." +} diff --git a/infrastructure/modules/observability/tests/setup/outputs.tf b/infrastructure/modules/observability/tests/setup/outputs.tf new file mode 100644 index 0000000000..50efb66319 --- /dev/null +++ b/infrastructure/modules/observability/tests/setup/outputs.tf @@ -0,0 +1,19 @@ +output "app_security_group_ids" { + description = "The IDs of the stand-in application security groups." + value = aws_security_group.app[*].id +} + +output "kms_key_arn" { + description = "The ARN of the stand-in KMS key." + value = aws_kms_key.main.arn +} + +output "subnet_ids" { + description = "The IDs of the stand-in subnets." + value = aws_subnet.main[*].id +} + +output "vpc_id" { + description = "The ID of the stand-in VPC." + value = aws_vpc.main.id +} diff --git a/infrastructure/modules/observability/tests/setup/variables.tf b/infrastructure/modules/observability/tests/setup/variables.tf new file mode 100644 index 0000000000..e69de29bb2 diff --git a/infrastructure/modules/observability/tests/unit.tftest.hcl b/infrastructure/modules/observability/tests/unit.tftest.hcl new file mode 100644 index 0000000000..6e15bbcad8 --- /dev/null +++ b/infrastructure/modules/observability/tests/unit.tftest.hcl @@ -0,0 +1,173 @@ +mock_provider "aws" {} + +variables { + app_security_group_ids = ["sg-backend", "sg-frontend", "sg-tasks"] + aws_region = "us-east-2" + common_tags = { Environment = "test", Project = "nest" } + environment = "test" + kms_key_arn = "arn:aws:kms:us-east-2:123456789012:key/12345678-1234-1234-1234-123456789012" + project_name = "nest" + subnet_ids = ["subnet-1", "subnet-2"] + vm_image = regex("(?m)^FROM (victoriametrics/victoria-metrics:\\S+)", file("../../../docker/victoriametrics/Dockerfile"))[0] + vm_port = 8428 + vpc_id = "vpc-12345" +} + +run "test_efs_encryption_enabled" { + command = plan + + assert { + condition = aws_efs_file_system.vm.encrypted == true + error_message = "EFS must be encrypted at rest." + } +} + +run "test_efs_uses_kms_key" { + command = plan + + assert { + condition = aws_efs_file_system.vm.kms_key_id == var.kms_key_arn + error_message = "EFS must be encrypted with the provided KMS key." + } +} + +run "test_efs_mount_target_per_subnet" { + command = plan + + assert { + condition = length(aws_efs_mount_target.vm) == length(var.subnet_ids) + error_message = "There must be one EFS mount target per subnet." + } +} + +run "test_vm_ingest_rule_per_app_security_group" { + command = plan + + assert { + condition = length(aws_security_group_rule.vm_ingest_from_apps) == length(var.app_security_group_ids) + error_message = "There must be one VM ingest rule per application security group." + } +} + +run "test_vm_ingest_from_source_security_group_only" { + command = plan + + assert { + condition = aws_security_group_rule.vm_ingest_from_apps[0].source_security_group_id == var.app_security_group_ids[0] + error_message = "VM ingest must be restricted to application security groups, not public CIDRs." + } + + assert { + condition = aws_security_group_rule.vm_ingest_from_apps[0].from_port == var.vm_port + error_message = "VM ingest must be allowed on the configured VictoriaMetrics port." + } +} + +run "test_efs_ingress_from_vm_only" { + command = plan + + assert { + condition = aws_security_group_rule.efs_from_vm.from_port == 2049 && aws_security_group_rule.efs_from_vm.type == "ingress" + error_message = "EFS must only allow NFS ingress on port 2049." + } +} + +run "test_vm_service_is_single_task" { + command = plan + + assert { + condition = aws_ecs_service.vm.desired_count == 1 + error_message = "VictoriaMetrics must run as a single task." + } +} + +run "test_vm_service_stops_before_starting" { + command = plan + + assert { + condition = aws_ecs_service.vm.deployment_minimum_healthy_percent == 0 && aws_ecs_service.vm.deployment_maximum_percent == 100 + error_message = "Deployments must stop the old task before starting the new one to avoid two writers on EFS." + } +} + +run "test_vm_uses_on_demand_fargate_only" { + command = plan + + assert { + condition = length(aws_ecs_cluster_capacity_providers.vm.capacity_providers) == 1 && contains(aws_ecs_cluster_capacity_providers.vm.capacity_providers, "FARGATE") + error_message = "VictoriaMetrics must use on-demand FARGATE only, never FARGATE_SPOT." + } +} + +run "test_task_uses_arm64" { + command = plan + + assert { + condition = aws_ecs_task_definition.vm.runtime_platform[0].cpu_architecture == "ARM64" + error_message = "The VictoriaMetrics task must run on ARM64." + } +} + +run "test_container_runs_as_non_root" { + command = plan + + assert { + condition = jsondecode(aws_ecs_task_definition.vm.container_definitions)[0].user == "65532" + error_message = "The VictoriaMetrics container must run as the non-root user 65532." + } +} + +run "test_access_point_enforces_non_root_owner" { + command = plan + + assert { + condition = aws_efs_access_point.vm.posix_user[0].uid == 65532 && aws_efs_access_point.vm.posix_user[0].gid == 65532 + error_message = "The EFS access point must enforce the non-root POSIX user 65532." + } +} + +run "test_task_mounts_encrypted_efs_volume" { + command = plan + + assert { + condition = one([for v in aws_ecs_task_definition.vm.volume : v if v.name == "vm-data"]).efs_volume_configuration[0].transit_encryption == "ENABLED" + error_message = "The vm-data volume must enable transit encryption." + } +} + +run "test_log_group_name_and_retention" { + command = plan + + assert { + condition = aws_cloudwatch_log_group.vm.name == "/aws/ecs/${var.project_name}-${var.environment}-observability" + error_message = "CloudWatch log group name must follow the /aws/ecs/{project}-{environment}-observability format." + } + + assert { + condition = aws_cloudwatch_log_group.vm.retention_in_days == var.log_retention_in_days + error_message = "CloudWatch log group must use the configured retention." + } +} + +run "test_cluster_name_format" { + command = plan + + assert { + condition = aws_ecs_cluster.vm.name == "${var.project_name}-${var.environment}-observability-cluster" + error_message = "ECS cluster name must follow the {project}-{environment}-observability-cluster format." + } +} + +run "test_common_tags_applied" { + command = plan + + assert { + condition = alltrue([for k, v in var.common_tags : lookup(aws_efs_file_system.vm.tags, k, null) == v]) + error_message = "common_tags must be applied to the EFS file system." + } + + assert { + condition = alltrue([for k, v in var.common_tags : lookup(aws_ecs_cluster.vm.tags, k, null) == v]) + error_message = "common_tags must be applied to the ECS cluster." + } +} diff --git a/infrastructure/modules/observability/variables.tf b/infrastructure/modules/observability/variables.tf new file mode 100644 index 0000000000..9d4e405f19 --- /dev/null +++ b/infrastructure/modules/observability/variables.tf @@ -0,0 +1,112 @@ +variable "app_security_group_ids" { + description = "Security group IDs of the application tasks allowed to send metrics to VictoriaMetrics." + type = list(string) + + validation { + condition = length(var.app_security_group_ids) > 0 + error_message = "app_security_group_ids must contain at least one security group." + } +} + +variable "assign_public_ip" { + description = "Whether to assign a public IP to the VictoriaMetrics task." + type = bool + default = false +} + +variable "aws_region" { + description = "The AWS region where the module is deployed." + type = string +} + +variable "common_tags" { + description = "A map of common tags to apply to all resources." + type = map(string) + default = {} +} + +variable "environment" { + description = "The environment (e.g., staging, production)." + type = string +} + +variable "kms_key_arn" { + description = "The ARN of the KMS key used to encrypt the EFS file system." + type = string +} + +variable "log_retention_in_days" { + description = "The number of days to retain VictoriaMetrics container logs." + type = number + default = 90 +} + +variable "project_name" { + description = "The name of the project." + type = string +} + +variable "subnet_ids" { + description = "The private subnet IDs for the EFS mount targets and the VictoriaMetrics task." + type = list(string) + + validation { + condition = length(var.subnet_ids) > 0 + error_message = "subnet_ids must contain at least one subnet." + } +} + +variable "vm_cpu" { + description = "The CPU units for the VictoriaMetrics Fargate task." + type = number + default = 512 +} + +variable "vm_desired_count" { + description = "The number of VictoriaMetrics tasks to run (0 or 1; it is a single-node store)." + type = number + default = 1 + + validation { + condition = contains([0, 1], var.vm_desired_count) + error_message = "vm_desired_count must be 0 or 1 (VictoriaMetrics is a single-node store)." + } +} + +variable "vm_image" { + description = "The VictoriaMetrics container image (including digest)." + type = string + + validation { + condition = can(regex("^[^@]+@sha256:[0-9a-f]{64}$", var.vm_image)) + error_message = "vm_image must be an image reference pinned to an immutable digest (e.g., repo:tag@sha256:...)." + } +} + +variable "vm_memory" { + description = "The memory (in MiB) for the VictoriaMetrics Fargate task." + type = number + default = 1024 +} + +variable "vm_port" { + description = "The port VictoriaMetrics listens on for ingest and queries." + type = number + default = 8428 + + validation { + condition = var.vm_port > 0 && var.vm_port < 65536 && floor(var.vm_port) == var.vm_port + error_message = "vm_port must be a whole number between 1 and 65535." + } +} + +variable "vm_retention_period" { + description = "The VictoriaMetrics data retention period. A value without a suffix is in months, so the default \"12\" means 12 months (duration suffixes like 1y, 30d, 1w are also supported)." + type = string + default = "12" # 12 months +} + +variable "vpc_id" { + description = "The VPC ID where the VictoriaMetrics security group is created." + type = string +} diff --git a/infrastructure/scripts/localstack.py b/infrastructure/scripts/localstack.py index a793b0f1da..ee6f82feb9 100644 --- a/infrastructure/scripts/localstack.py +++ b/infrastructure/scripts/localstack.py @@ -256,6 +256,10 @@ class OverrideManager: # Temporary Terraform overrides that disable prevent_destroy during integration tests. OVERRIDES: list[tuple[str, str]] = [ + ( + "infrastructure/modules/observability/test_override.tf", + "aws_efs_file_system.vm", + ), ( "infrastructure/modules/storage/modules/s3-bucket/test_override.tf", "aws_s3_bucket.this",