diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87cd185d..6d4d8eb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,22 @@ jobs: working-directory: ./evaluate-automerge run: make test + test-resolve-stack-config: + name: Test resolve-stack-config composite action + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + + - name: Run tests + working-directory: ./resolve-stack-config + run: make test + test-terraform-deploy: name: Test terraform-deploy composite action runs-on: ubuntu-24.04 @@ -402,6 +418,122 @@ jobs: test "$(echo "$PAYLOAD" | jq -r '.data.attributes.git.commit_sha')" = "$GITHUB_SHA" test "$(echo "$PAYLOAD" | jq -r '.data.attributes.git.repository_url')" = "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" + test-e2e-terraform-deploy-stack-config: + name: Test E2E terraform-deploy (stack-config integration) + runs-on: ubuntu-24.04 + permissions: + contents: read + services: + moto: + image: motoserver/moto:5.1.22@sha256:117238c6e7e3b566387c4c74e6fb0b6fdc34a094920c2154cf06f11dc72d9f37 + ports: + - 5000:5000 + env: + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL: http://localhost:5000 + steps: + - name: Checkout + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + + - name: Prepare ephemeral SSH deploy key + id: ssh-key + run: | + ssh-keygen -t ed25519 -f "$RUNNER_TEMP/test-key" -N "" -q + { + echo "private-key<> "$GITHUB_OUTPUT" + + - name: Prepare two identical Terraform stacks under nested paths + run: | + cat > /tmp/main.tf << 'EOF' + terraform { + required_version = ">= 1.9.0" + } + + variable "echoed" { + type = string + default = "default-value" + } + + output "echoed" { + value = var.echoed + } + EOF + # Nested paths so the pattern `stacks/dev/**` matches the stack-dir. + # `**` requires at least one segment after it, so a flat layout like + # `stack-match` would not match `stack-match/**`. + mkdir -p stacks/dev/app-match stacks/dev/app-no-match + cp /tmp/main.tf stacks/dev/app-match/main.tf + cp /tmp/main.tf stacks/dev/app-no-match/main.tf + + - name: Deploy stack with matching stack-config pattern + id: deploy-match + uses: ./terraform-deploy + with: + config: | + { + "dev": { + "accountId": "123456789012", + "defaultRegion": "us-east-1", + "deploymentRoleArn": "arn:aws:iam::123456789012:role/test", + "name": "test-dev" + } + } + stack-dir: "stacks/dev/app-match" + environment: "dev" + github-deploy-key: ${{ steps.ssh-key.outputs.private-key }} + cancel-if-stale: "false" + stack-config: | + [ + { + "pattern": "stacks/dev/**", + "envVars": { "TF_VAR_echoed": "injected" } + } + ] + + - name: Verify injected TF_VAR reached Terraform output + env: + OUTPUTS: ${{ steps.deploy-match.outputs.terraform-outputs }} + run: | + echo "$OUTPUTS" + test "$(echo "$OUTPUTS" | jq -r '.echoed')" = "injected" + + - name: Deploy stack with non-matching stack-config pattern + id: deploy-no-match + uses: ./terraform-deploy + with: + config: | + { + "dev": { + "accountId": "123456789012", + "defaultRegion": "us-east-1", + "deploymentRoleArn": "arn:aws:iam::123456789012:role/test", + "name": "test-dev" + } + } + stack-dir: "stacks/dev/app-no-match" + environment: "dev" + github-deploy-key: ${{ steps.ssh-key.outputs.private-key }} + cancel-if-stale: "false" + stack-config: | + [ + { + "pattern": "stacks/prod/**", + "envVars": { "TF_VAR_echoed": "should-not-be-set" } + } + ] + + - name: Verify Terraform output is the variable default when pattern does not match + env: + OUTPUTS: ${{ steps.deploy-no-match.outputs.terraform-outputs }} + run: | + echo "$OUTPUTS" + test "$(echo "$OUTPUTS" | jq -r '.echoed')" = "default-value" + test-e2e-terraform-deploy-regression-test-terraform-v1-15-0: # Regression test for hashicorp/terraform#38484: Terraform 1.15.0 emits the # S3 backend's `dynamodb_table` deprecation warning to stdout, contaminating diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5e4acaa..5e65cea7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,7 @@ on: - "determine-stacks/*" - "detect-stale-job/*" - "evaluate-automerge/*" + - "resolve-stack-config/*" jobs: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 07055827..95cfb8dd 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -12,5 +12,6 @@ "terraform-deploy": "1.7.3", "determine-stacks": "1.2.2", "detect-stale-job": "1.0.1", - "evaluate-automerge": "1.0.1" + "evaluate-automerge": "1.0.1", + "resolve-stack-config": "0.0.0" } diff --git a/release-please-config.json b/release-please-config.json index eddf677f..bcb85326 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -62,6 +62,10 @@ "evaluate-automerge": { "component": "evaluate-automerge", "release-type": "simple" + }, + "resolve-stack-config": { + "component": "resolve-stack-config", + "release-type": "simple" } } } diff --git a/resolve-stack-config/Makefile b/resolve-stack-config/Makefile new file mode 100644 index 00000000..11066ba9 --- /dev/null +++ b/resolve-stack-config/Makefile @@ -0,0 +1,6 @@ +.PHONY: test + +# Run unit tests +test: + @echo "Running tests..." + uv run python -m unittest test_resolve_stack_config -v diff --git a/resolve-stack-config/action.yml b/resolve-stack-config/action.yml new file mode 100644 index 00000000..85b0aa9a --- /dev/null +++ b/resolve-stack-config/action.yml @@ -0,0 +1,37 @@ +name: "Resolve stack-config" +description: "Resolve stack-config blocks against a stack-dir and emit a sourceable env file" + +inputs: + stack-config: + description: "JSON array of stack configurations ( { pattern, envVars } objects). First matching pattern wins." + required: false + default: "" + stack-dir: + description: "Path of the current Terraform stack (e.g., stacks/dev/app)" + required: true + +outputs: + env-file: + description: "Absolute path to a sourceable env file. Always created; empty if no patterns matched." + value: ${{ steps.resolve.outputs.env-file }} + +runs: + using: "composite" + steps: + - name: Install uv + if: ${{ inputs.stack-config != '' }} + uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + with: + working-directory: ${{ github.action_path }} + + - name: Resolve stack-config + id: resolve + if: ${{ inputs.stack-config != '' }} + working-directory: ${{ github.action_path }} + shell: bash --noprofile --norc -euo pipefail {0} + env: + STACK_CONFIG: ${{ inputs.stack-config }} + STACK_DIR: ${{ inputs.stack-dir }} + run: | + env_file="$(uv run resolve_stack_config.py --stack-config "$STACK_CONFIG" --stack-dir "$STACK_DIR")" + echo "env-file=$env_file" >> "$GITHUB_OUTPUT" diff --git a/resolve-stack-config/pyproject.toml b/resolve-stack-config/pyproject.toml new file mode 100644 index 00000000..4a3ff728 --- /dev/null +++ b/resolve-stack-config/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "resolve-stack-config" +version = "0.1.0" +description = "Resolve stack-config blocks against a stack-dir and emit a sourceable env file" +requires-python = ">=3.13" +dependencies = [] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/resolve-stack-config/resolve_stack_config.py b/resolve-stack-config/resolve_stack_config.py new file mode 100644 index 00000000..b923ce31 --- /dev/null +++ b/resolve-stack-config/resolve_stack_config.py @@ -0,0 +1,159 @@ +"""Resolve stack-config blocks against a stack-dir and emit a sourceable env file. + +Iterates the block list, finds the first block whose pattern matches the current +stack-dir, validates the envVars map, and writes shell-quoted KEY=VALUE lines to +a private tempfile. The path is printed to stdout so the caller can capture it. + +Usage: + python3 resolve_stack_config.py --stack-config --stack-dir + [--env-file-dir ] + +Output: prints the absolute path of the env-file to stdout. +""" + +import argparse +import json +import os +import re +import shlex +import stat +import sys +import tempfile +from pathlib import PurePosixPath +from typing import NotRequired, TypedDict + + +class Block(TypedDict): + pattern: str + envVars: NotRequired[dict[str, str]] + + +# Env var names must look like a conventional shell identifier. This rejects +# garbage (spaces, semicolons, leading digits) that would produce baffling +# downstream errors. Not a security boundary — the caller workflow is already +# trusted with full job privileges. +KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# Only envVars is recognized today; unknown keys are warned-and-ignored so that +# future extensions can be added without breaking older versions of this action. +KNOWN_BLOCK_KEYS = frozenset({"pattern", "envVars"}) + + +def find_match(stack_dir: str, blocks: list[Block]) -> Block | None: + """Return the first block whose pattern matches stack_dir, or None.""" + path = PurePosixPath(stack_dir) + for block in blocks: + if path.full_match(block["pattern"]): + return block + return None + + +def validate_block_shape(raw_blocks: object) -> list[Block]: + """Validate that the parsed JSON matches the expected schema.""" + if not isinstance(raw_blocks, list): + raise ValueError("stack-config must be a JSON array") + result: list[Block] = [] + for i, block in enumerate(raw_blocks): + if not isinstance(block, dict): + raise ValueError(f"stack-config[{i}] must be an object") + pattern = block.get("pattern") + if not isinstance(pattern, str): + raise ValueError(f"stack-config[{i}] requires a string 'pattern'") + env_vars = block.get("envVars", {}) + if not isinstance(env_vars, dict): + raise ValueError(f"stack-config[{i}].envVars must be an object") + unknown = set(block) - KNOWN_BLOCK_KEYS + for key in sorted(unknown): + print( + f"::warning::stack-config[{i}]: unknown key '{key}' ignored " + f"(recognized keys: {sorted(KNOWN_BLOCK_KEYS)})", + file=sys.stderr, + ) + result.append(block) # type: ignore[arg-type] + return result + + +def validate_env_vars(env_vars: dict[str, object]) -> dict[str, str]: + """Validate keys and values of an envVars map.""" + validated: dict[str, str] = {} + for key, value in env_vars.items(): + if not isinstance(key, str) or not KEY_RE.match(key): + raise ValueError( + f"envVars key {key!r} must match {KEY_RE.pattern}" + ) + if not isinstance(value, str): + raise ValueError( + f"envVars[{key!r}] must be a string, got {type(value).__name__}" + ) + validated[key] = value + return validated + + +def write_env_file(env_vars: dict[str, str], env_file_dir: str) -> str: + """Write shell-quoted KEY=VALUE lines to a private tempfile, return its path.""" + fd, path = tempfile.mkstemp(suffix=".env", dir=env_file_dir, text=True) + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + with os.fdopen(fd, "w") as f: + for key in sorted(env_vars): + f.write(f"{key}={shlex.quote(env_vars[key])}\n") + return path + + +def resolve( + stack_config: str, stack_dir: str, env_file_dir: str +) -> tuple[str, list[str]]: + """Resolve the matching block and produce an env file. + + Returns (env_file_path, exported_keys). When no pattern matches (or the + input has no blocks), returns ("", []) and no file is written; the caller + is expected to guard its source against an empty path. + """ + try: + raw_blocks = json.loads(stack_config) if stack_config.strip() else [] + except json.JSONDecodeError as e: + raise ValueError(f"stack-config is not valid JSON: {e}") from e + + blocks = validate_block_shape(raw_blocks) + match = find_match(stack_dir, blocks) + if match is None: + return "", [] + + env_vars = validate_env_vars(match.get("envVars", {})) + if not env_vars: + return "", [] + return write_env_file(env_vars, env_file_dir), sorted(env_vars) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Resolve a stack-config block list against a stack-dir" + ) + parser.add_argument( + "--stack-config", + required=True, + help="JSON array of stack-config blocks", + ) + parser.add_argument( + "--stack-dir", + required=True, + help="Path of the current Terraform stack (e.g., stacks/dev/app)", + ) + parser.add_argument( + "--env-file-dir", + default=os.environ.get("RUNNER_TEMP", tempfile.gettempdir()), + help="Directory in which to create the env file (default: $RUNNER_TEMP)", + ) + args = parser.parse_args() + + try: + env_file_path, exported = resolve( + args.stack_config, args.stack_dir, args.env_file_dir + ) + except ValueError as e: + print(f"::error::{e}", file=sys.stderr) + sys.exit(1) + + for key in exported: + print(f"Exported {key}", file=sys.stderr) + + print(env_file_path) diff --git a/resolve-stack-config/test_resolve_stack_config.py b/resolve-stack-config/test_resolve_stack_config.py new file mode 100644 index 00000000..16b18767 --- /dev/null +++ b/resolve-stack-config/test_resolve_stack_config.py @@ -0,0 +1,144 @@ +import json +import os +import shlex +import stat +import tempfile +import unittest + +import resolve_stack_config as rsc + + +def _resolve(stack_config_obj, stack_dir: str) -> tuple[str, list[str]]: + """Run resolve() with the test's tempdir as env-file location.""" + return rsc.resolve( + json.dumps(stack_config_obj), + stack_dir, + env_file_dir=tempfile.gettempdir(), + ) + + +def _read_env_file(path: str) -> dict[str, str]: + """Parse a written env file back into a dict for assertions.""" + result: dict[str, str] = {} + with open(path) as f: + for line in f: + line = line.rstrip("\n") + if not line: + continue + key, _, raw_value = line.partition("=") + parsed = shlex.split(raw_value) + result[key] = parsed[0] if parsed else "" + return result + + +class TestMatching(unittest.TestCase): + def test_first_match_wins(self): + """Critical: later catch-all patterns must not override an earlier specific match.""" + blocks = [ + {"pattern": "stacks/dev/**", "envVars": {"FOO": "dev"}}, + {"pattern": "**", "envVars": {"FOO": "fallback"}}, + ] + path, exported = _resolve(blocks, "stacks/dev/app") + self.assertEqual(exported, ["FOO"]) + self.assertEqual(_read_env_file(path)["FOO"], "dev") + + def test_no_match_produces_empty_path(self): + """No match returns an empty path (no file written) so the caller can skip sourcing.""" + blocks = [{"pattern": "stacks/dev/**", "envVars": {"FOO": "bar"}}] + path, exported = _resolve(blocks, "stacks/prod/app") + self.assertEqual(exported, []) + self.assertEqual(path, "") + + +class TestSchemaValidation(unittest.TestCase): + def test_invalid_json_raises(self): + with self.assertRaisesRegex(ValueError, "not valid JSON"): + rsc.resolve("{not json", "stacks/dev/app", tempfile.gettempdir()) + + def test_empty_string_is_valid_and_means_no_blocks(self): + """Empty string is the action's default — must not be rejected as invalid JSON. + Returns the same "" path as no-match, so the caller's source-guard handles both.""" + path, exported = rsc.resolve("", "stacks/dev/app", tempfile.gettempdir()) + self.assertEqual(exported, []) + self.assertEqual(path, "") + + def test_malformed_blocks_rejected(self): + """Each shape error should produce a distinct, actionable message.""" + cases = [ + ('{"pattern": "x"}', "must be a JSON array"), + ('["not-an-object"]', "must be an object"), + ('[{"envVars": {"FOO": "bar"}}]', "requires a string 'pattern'"), + ('[{"pattern": "**", "envVars": "foo"}]', "envVars must be an object"), + ] + for raw, expected_msg in cases: + with self.subTest(raw=raw): + with self.assertRaisesRegex(ValueError, expected_msg): + rsc.resolve(raw, "stacks/dev/app", tempfile.gettempdir()) + + +class TestKeyValidation(unittest.TestCase): + def test_keys_with_shell_metachars_rejected(self): + """Shell metachars in keys would break out of KEY=VAL when sourcing — must be blocked at the regex layer.""" + for bad_key in ("X; rm -rf /; FOO", "FOO BAR", "FOO=BAR", "1FOO"): + with self.subTest(bad_key=bad_key): + with self.assertRaisesRegex(ValueError, "must match"): + _resolve( + [{"pattern": "**", "envVars": {bad_key: "x"}}], + "stacks/dev/app", + ) + + def test_tf_var_with_lowercase_suffix_allowed(self): + """Terraform's TF_VAR_ convention uses lowercase suffixes; the regex must permit this.""" + path, _ = _resolve( + [{"pattern": "**", "envVars": {"TF_VAR_region": "eu-west-1"}}], + "stacks/dev/app", + ) + self.assertEqual(_read_env_file(path)["TF_VAR_region"], "eu-west-1") + + +class TestValueValidation(unittest.TestCase): + def test_non_string_value_rejected(self): + """Non-string values would crash shlex.quote with an unhelpful TypeError; + convert to a clean ValueError at validation time.""" + with self.assertRaisesRegex(ValueError, "must be a string"): + _resolve([{"pattern": "**", "envVars": {"FOO": 123}}], "stacks/dev/app") + + +class TestEnvFileSecurity(unittest.TestCase): + def test_shell_injection_via_value_does_not_execute(self): + """The crown-jewel security test: a value designed to break out of single-quoting + must round-trip as a literal string after `source`.""" + evil = "'; rm -rf /; echo $(uname) `id` \"x\"" + path, _ = _resolve( + [{"pattern": "**", "envVars": {"FOO": evil}}], "stacks/dev/app" + ) + self.assertEqual(_read_env_file(path)["FOO"], evil) + + def test_env_file_is_user_only_readable(self): + """File contains values that aren't strictly secret today but might be tomorrow; + keep it 0600 as a baseline.""" + path, _ = _resolve( + [{"pattern": "**", "envVars": {"FOO": "bar"}}], "stacks/dev/app" + ) + self.assertEqual(stat.S_IMODE(os.stat(path).st_mode), 0o600) + + +class TestForwardCompatibility(unittest.TestCase): + def test_unknown_block_keys_are_warned_not_rejected(self): + """Unknown keys must be warned-and-ignored, not rejected, so that future + extensions can be added without breaking older versions of this action.""" + path, _ = _resolve( + [ + { + "pattern": "**", + "envVars": {"FOO": "bar"}, + "futureFeature": {"some": "value"}, + } + ], + "stacks/dev/app", + ) + self.assertEqual(_read_env_file(path)["FOO"], "bar") + + +if __name__ == "__main__": + unittest.main() diff --git a/resolve-stack-config/uv.lock b/resolve-stack-config/uv.lock new file mode 100644 index 00000000..3b59fb30 --- /dev/null +++ b/resolve-stack-config/uv.lock @@ -0,0 +1,8 @@ +version = 1 +revision = 3 +requires-python = ">=3.13" + +[[package]] +name = "resolve-stack-config" +version = "0.1.0" +source = { editable = "." } diff --git a/terraform-deploy/action.yml b/terraform-deploy/action.yml index 5f2de127..f1a6698c 100644 --- a/terraform-deploy/action.yml +++ b/terraform-deploy/action.yml @@ -42,6 +42,10 @@ inputs: datadog-api-key: description: "Datadog API key for sending deployment events" required: false + stack-config: + description: "JSON array of stack configurations ( { pattern, envVars } objects). Matched against stack-dir; the first matching block's envVars are exported as env vars for `terraform apply`. See the resolve-stack-config action for the full schema." + required: false + default: "" outputs: # NOTE: A composite action can't have dynamic outputs, so @@ -151,6 +155,20 @@ runs: with: terraform_version: ${{ steps.v.outputs.TERRAFORM_VERSION }} + - name: Resolve stack-config + id: stack-config + # TODO: replace ./resolve-stack-config with a SHA-pinned reference to + # oslokommune/composite-actions/resolve-stack-config@ before tagging + # a release. The local path only works inside this repo's own CI; + # consumers calling this action from another repo do not have + # resolve-stack-config in their checkout, so the local reference will + # not resolve there. + if: ${{ inputs.stack-config != '' }} + uses: ./resolve-stack-config + with: + stack-config: ${{ inputs.stack-config }} + stack-dir: ${{ inputs.stack-dir }} + - name: Initialize the working directory containing Terraform configuration files id: init shell: bash --noprofile --norc -euo pipefail {0} @@ -207,7 +225,12 @@ runs: working-directory: ${{ steps.get-stack-dir.outputs.stack-dir }} env: EXTRACT_OUTPUTS: ${{ github.action_path }}/extract_outputs.py + STACK_ENV_FILE: ${{ steps.stack-config.outputs.env-file }} run: | + if [ "$STACK_ENV_FILE" != "" ]; then + echo "Sourcing environment variables from file '$STACK_ENV_FILE'" + set -a; source "$STACK_ENV_FILE"; set +a + fi terraform apply -auto-approve -lock-timeout=5m # Pipe through extract_outputs.py instead of jq directly: Terraform 1.15.0 # may emit deprecation warnings on stdout (hashicorp/terraform#38484),