feat(ngrh): add an opt-in DevOps Agent connect to make deploy

README.md

@@ -227,6 +227,8 @@ make devops-agent-connect DEVOPS_AGENT_SERVICES="orders"
 
 Once a service is connected, a test run of it starts an investigation, and each of its failure mode assessment findings offers **Chat with DevOps Agent**.
 
+**In one command.** `make deploy DEVOPS_AGENT_CONNECT="orders"` deploys the sample and then does the two steps above: `make devops-agent`, and `make devops-agent-connect` for the services named. It takes the same names as `DEVOPS_AGENT_SERVICES`, and `all` means every service. Without the variable `make deploy` does neither, and the `e2e` workflow does not set it. Before it deploys anything it checks the names and that your AWS CLI is 2.38.0 or later, so a typo or an old CLI is found at once and not after the deploy. The Agent Space and the connection come last, after the NGRH model is deployed, and the step waits a minute between them (`DEVOPS_AGENT_IAM_WAIT`, in seconds) for IAM to give the invoker role its new policy, because a connection that cannot use the policy yet is not sent again. Like the separate command, it prints each service's finding count before it connects. `make deploy` runs no assessment, so the count is whatever the services already have: on a deployment that has been assessed, naming `orders` starts as many investigations as it has findings, so name one service first. With `DEVOPS_AGENT_SPACE_ARN=<arn>`, an Agent Space you made yourself, the Agent Space stack is not created and the services connect to yours. If the connection fails, the deploy is done and nothing is lost: fix the cause and run `make devops-agent-connect` as above.
+
 The operator web app is at the address `aws devops-agent get-operator-app --agent-space-id <AgentSpaceId> --query operatorAppUrl` prints; the `AgentSpaceId` is an output of the stack.
 
 **What you need.**

deployment/Makefile

@@ -28,6 +28,32 @@ PRIMARY_ONLY=false
 endif
 ####
 
+####
+# Optional: make deploy DEVOPS_AGENT_CONNECT="orders" is the one-command way to use AWS DevOps Agent (README,
+# "Investigating with AWS DevOps Agent"). Before it deploys anything it checks the names and that the AWS CLI can
+# connect a service (devops-agent-connect-preflight). After the deploy it runs make devops-agent, which creates the Agent
+# Space and gives the invoker role access to it, waits DEVOPS_AGENT_IAM_WAIT seconds for IAM to catch up, and runs make
+# devops-agent-connect for the services named. That last step is what costs money: DevOps Agent starts one investigation
+# for each finding of each service connected. Without the variable make deploy does none of this, which is what the e2e
+# workflow gets. With DEVOPS_AGENT_SPACE_ARN=<arn>, an Agent Space you made, make devops-agent is left out and the
+# services connect to that one; set to nothing it takes the access away, which cannot go with connecting, so the check
+# refuses it. The names reach the recipes as an environment variable, so the shell never parses them as text.
+# The steps are kept in a variable, not written in the recipe of deploy: make runs a recipe line that has $(MAKE) in it
+# even under -n, so make -n deploy DEVOPS_AGENT_CONNECT=orders would deploy and connect for real.
+####
+DEVOPS_AGENT_IAM_WAIT ?= 60
+ifneq ($(strip $(DEVOPS_AGENT_CONNECT)),)
+export DEVOPS_AGENT_CONNECT
+DEVOPS_AGENT_PREFLIGHT = devops-agent-connect-preflight
+DEVOPS_AGENT_CONNECT_STEP = echo "Waiting $(DEVOPS_AGENT_IAM_WAIT) seconds for IAM to give the invoker role its new policy, because a connection that cannot use it yet is not sent again..." \
+	&& sleep $(DEVOPS_AGENT_IAM_WAIT) && $(MAKE) devops-agent-connect DEVOPS_AGENT_SERVICES="$$DEVOPS_AGENT_CONNECT"
+ifeq ($(origin DEVOPS_AGENT_SPACE_ARN),undefined)
+DEVOPS_AGENT_AFTER_DEPLOY = @$(MAKE) devops-agent && $(DEVOPS_AGENT_CONNECT_STEP)
+else
+DEVOPS_AGENT_AFTER_DEPLOY = @$(DEVOPS_AGENT_CONNECT_STEP)
+endif
+endif
+
 deploy-all: test-creds create-peer primary_infrastructure primary_ecs standby_infrastructure standby_ecs global_routing \
 	canaries_primary canaries_standby restore-reconciliation monitoring \
 	region-switch arc-dns-status secrets-rotation ngrh
@@ -40,7 +66,7 @@ deploy-primary: test-creds create-peer primary_infrastructure primary_ecs
 	--stack-name canaries${ENV} --parameter-overrides Env=${ENV} RemoteRegion=${STANDBY_REGION} PrimaryOnly=true VpcId=$(_CAN_VPC) --capabilities CAPABILITY_IAM --s3-bucket $(result) \
 	--tags service=shared
 
-deploy: deploy-all
+deploy: $(DEVOPS_AGENT_PREFLIGHT) deploy-all
 	@echo "*** Deployment Outputs ****"
 	@echo ""
 	@echo "Canaries:"
@@ -52,6 +78,7 @@ deploy: deploy-all
 	@echo ""
 	@echo "Region Switch Plan:"
 	@echo "Use 'aws arc-region-switch start-plan-execution' to initiate failover"
+	$(DEVOPS_AGENT_AFTER_DEPLOY)
 
 create-peer:
 	@echo "Verifying if the AWSServiceRoleForECS exists; if not; this will be created for you..."
@@ -946,6 +973,8 @@ ngrh:
 # make devops-agent-connect DEVOPS_AGENT_SERVICES="orders cart" (or all) is what
 # costs money: DevOps Agent starts one investigation for each finding of each
 # service connected, and bills it by the agent-second.
+# make deploy DEVOPS_AGENT_CONNECT="orders" does the deploy, this, and the connect in one command (the comment above
+# deploy-all says how).
 ####
 devops-agent:
 	@aws cloudformation deploy --template ./devops-agent.yaml --region $(PRIMARY_REGION) \
@@ -976,6 +1005,17 @@ devops-agent-connect:
 	fi; \
 	python3 -m ngrh_testing devops-agent connect $(NGRH_TESTING_ARGS) --space-arn "$$space" --services "$$DEVOPS_AGENT_SERVICES"
 
+# What make deploy DEVOPS_AGENT_CONNECT=<services> asks before it deploys anything (see the comment above deploy-all):
+# DEVOPS_AGENT_SPACE_ARN set to nothing takes the invoker role's access away, which cannot go with connecting, and the tool
+# checks that the names are services and that the AWS CLI can connect one. Connect would refuse either after a deploy of
+# about two hours. It reads nothing from the deployment, because on a first deploy there is none.
+devops-agent-connect-preflight:
+	@if [ "$${DEVOPS_AGENT_SPACE_ARN+set}" = set ] && [ -z "$$DEVOPS_AGENT_SPACE_ARN" ]; then \
+		echo "ERROR: DEVOPS_AGENT_CONNECT connects services to an Agent Space, and DEVOPS_AGENT_SPACE_ARN= (set to nothing) takes the access to it away. Use one or the other." >&2; \
+		exit 1; \
+	fi
+	@python3 -m ngrh_testing devops-agent check $(NGRH_TESTING_ARGS) --services "$$DEVOPS_AGENT_CONNECT"
+
 # Takes the NGRH services off their Agent Space. It does nothing unless the ngrh
 # stack was given one, so it is safe to run on any deployment (destroy-ngrh does).
 # The shell variable ngrh_disconnect_options is empty unless the recipe line sets it before this text (destroy-ngrh

deployment/ngrh_testing/__init__.py

@@ -17,5 +17,7 @@
   stop          stop the test's active run (make ngrh-test-stop).
   report        collect a run's report as JSON and markdown (make ngrh-test-report).
   devops-agent  connect named NGRH services to an AWS DevOps Agent Agent Space, disconnect them, or show
-                their state and finding counts (make devops-agent-connect, -disconnect and -status).
+                their state and finding counts (make devops-agent-connect, -disconnect and -status); check
+                that the names and the AWS CLI would let a connect through, before a deploy (make deploy
+                DEVOPS_AGENT_CONNECT=<services>).
 """

deployment/ngrh_testing/cli.py

@@ -117,7 +117,7 @@ def parser() -> argparse.ArgumentParser:
     g.add_argument("action", choices=devops_agent.ACTIONS)
     g.add_argument("--space-arn", default="", help="the Agent Space's ARN (connect needs it)")
     g.add_argument("--services", default="",
-                   help="the services to connect (connect needs it): names from "
+                   help="the services to connect (connect and check need it): names from "
                         f"{', '.join(devops_agent.SERVICE_NAMES)}, separated by spaces or commas, or {devops_agent.ALL}. "
                         "Connecting starts one paid investigation for each finding of each service")
     g.add_argument("--if-supported", action="store_true",
@@ -312,6 +312,17 @@ def _failback_command(aws: AwsCli, args: argparse.Namespace, sleep: Callable[[fl
                         lambda line: sys.stderr.write(line + "\n"))
 
 
+def _devops_agent_check(aws: AwsCli, args: argparse.Namespace) -> int:
+    """check: exit 0 when connect would not refuse these services or this AWS CLI. It reads nothing from the deployment
+    (make deploy DEVOPS_AGENT_CONNECT=<services> runs it before the first stack exists) and writes nothing."""
+    names = devops_agent.check(aws, args.services)
+    shown = f"all {len(devops_agent.SERVICE_NAMES)} services" if names == [devops_agent.ALL] else ", ".join(names)
+    _say(f"DevOps Agent: after the deploy, {shown} will be connected to the Agent Space. This AWS CLI can do that.")
+    _say("Connecting starts one paid DevOps Agent investigation for each finding of each service; "
+         "the counts are shown before anything is changed.")
+    return EXIT_OK
+
+
 def _devops_agent_command(aws: AwsCli, env: Environment, args: argparse.Namespace, sleep: Callable[[float], None],
                           clock: Callable[[], float]) -> int:
     """connect and disconnect exit 3 when a service is left in a state the output names; status only reads."""
@@ -378,6 +389,8 @@ def now() -> datetime:
             return EXIT_OK
         if args.command == "failback":
             return _failback_command(aws, args, sleep, clock, now)
+        if args.command == "devops-agent" and args.action == "check":
+            return _devops_agent_check(aws, args)  # before load_environment: on a first deploy there is no ngrh stack yet
         if args.command == "preflight":  # check 1: the credential sentinel
             try:
                 aws.account_id()

deployment/ngrh_testing/devops_agent.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: MIT-0
 """Connect NGRH services to an AWS DevOps Agent Agent Space, disconnect them, or show their state
-(make devops-agent-connect, make devops-agent-disconnect, make devops-agent-status).
+(make devops-agent-connect, make devops-agent-disconnect, make devops-agent-status). ``check`` is the one action that
+needs no deployment: make deploy DEVOPS_AGENT_CONNECT=<services> runs it before it deploys anything.
 
 Resilience Hub starts a DevOps Agent investigation for a service once the service names an Agent Space in its
 DevOps Agent configuration. That configuration is set here, not in ngrh.yaml: CloudFormation cannot take it away
@@ -29,7 +30,7 @@
 from .aws import AwsCli, AwsCliError
 from .context import Environment
 
-ACTIONS = ("connect", "disconnect", "status")
+ACTIONS = ("connect", "disconnect", "status", "check")
 
 # The services of ngrh.yaml, by the short names its outputs use (OrdersServiceArn, UiServiceArn, ...).
 SERVICE_NAMES = ("ui", "catalog", "cart", "checkout", "orders", "assets")
@@ -142,16 +143,26 @@ def all_services(env: Environment) -> List[str]:
     return arns
 
 
-def select_services(env: Environment, names: str) -> List[str]:
-    """The ARNs of the services named in ``names`` (separated by spaces or commas), or of all of them for ``all``."""
+def parse_service_names(names: str) -> List[str]:
+    """The service names in ``names`` (separated by spaces or commas, in any case), in the order given and without repeats,
+    or ``["all"]`` when ``all`` is among them. Refuses a list with nothing in it and a name that is not a service. This is the
+    part of select_services that needs no deployment, so ``check`` can run it before there is one."""
     wanted = [n for n in re.split(r"[,\s]+", names.strip().lower()) if n]
     if not wanted:
         raise DevOpsAgentError(f"name the services to connect: {', '.join(SERVICE_NAMES)}, or {ALL}")
     if ALL in wanted:
-        return all_services(env)
+        return [ALL]
     unknown = [n for n in wanted if n not in SERVICE_NAMES]
     if unknown:
         raise DevOpsAgentError(f"unknown service {', '.join(unknown)}: the services are {', '.join(SERVICE_NAMES)}")
+    return list(dict.fromkeys(wanted))
+
+
+def select_services(env: Environment, names: str) -> List[str]:
+    """The ARNs of the services named in ``names`` (separated by spaces or commas), or of all of them for ``all``."""
+    wanted = parse_service_names(names)
+    if wanted == [ALL]:
+        return all_services(env)
     arns: List[str] = []
     for n in wanted:
         arn = env.service_arn(n)
@@ -160,6 +171,16 @@ def select_services(env: Environment, names: str) -> List[str]:
     return arns
 
 
+def check(aws: AwsCli, services: str) -> List[str]:
+    """What ``make deploy DEVOPS_AGENT_CONNECT=<services>`` asks before it deploys anything: the names are ones connect accepts
+    and the AWS CLI can connect a service. Connect would refuse either after a deploy of about two hours, so both are found
+    first. It reads nothing from the deployment, which does not exist yet on a first deploy, and it writes nothing. Returns the
+    names, as parse_service_names gives them."""
+    names = parse_service_names(services)
+    require_cli_support(aws)
+    return names
+
+
 def _settle(aws: AwsCli, env: Environment, states: Dict[str, Connection], arns: List[str], done: Callable[[Connection], bool],
             seconds: float, sleep: Callable[[float], None], clock: Callable[[], float]) -> Dict[str, Connection]:
     """Read the services until each one's state satisfies ``done``, or ``seconds`` have passed."""

tests/test_e2e_workflow.py

@@ -196,8 +196,13 @@ def test_no_step_connects_a_service_or_picks_an_agent_space_by_hand(self):
         for s in steps():
             run = s.get("run", "")
             assert "devops-agent-connect" not in run and "DEVOPS_AGENT_SERVICES" not in run, f"{s['name']} would start paid investigations"
+            assert "DEVOPS_AGENT_CONNECT" not in run, f"{s['name']} would connect services through make deploy, which starts paid investigations"
             assert "DEVOPS_AGENT_SPACE_ARN" not in run, f"{s['name']} would override the Agent Space of the stack the run just made"
 
+    def test_the_workflow_does_not_set_the_connect_variable_anywhere(self):
+        # not in a run line, and not in an env block either: make reads the environment too
+        assert "DEVOPS_AGENT_CONNECT" not in E2E.read_text()
+
     def test_the_teardown_deletes_it_through_destroy_all_which_deletes_the_ngrh_stack_first(self):
         assert 'make destroy-all "ENV=${ENV}"' in step(TEARDOWN)["run"]
         rule = next(line for line in MAKEFILE.read_text().splitlines() if line.startswith("destroy-all:"))

tests/test_ngrh_devops_agent.py

@@ -273,6 +273,26 @@ def test_it_does_not_claim_more_than_was_checked(self):
         assert "Nothing is billed" not in section and "nothing has been billed" not in section
         assert "not in the AWS documentation" not in section, "only five Resilience Hub pages were read"
 
+    def test_it_says_how_to_deploy_and_connect_in_one_command_and_what_that_does_and_costs(self):
+        # make deploy DEVOPS_AGENT_CONNECT="orders" (deployment/Makefile): the checks, then the deploy, then the Agent Space and
+        # the connection. Everything the paragraph says is a behaviour tests/test_ngrh_devops_agent_make.py pins.
+        paragraph = next(p for p in _section().split("\n\n") if p.startswith("**In one command.**"))
+        assert '`make deploy DEVOPS_AGENT_CONNECT="orders"`' in paragraph
+        assert "`make devops-agent`" in paragraph and "`make devops-agent-connect`" in paragraph
+        assert "same names as `DEVOPS_AGENT_SERVICES`" in paragraph and "`all`" in paragraph
+        assert "Without the variable `make deploy` does neither" in paragraph and "the `e2e` workflow does not set it" in paragraph
+        assert "Before it deploys anything" in paragraph and "AWS CLI is 2.38.0 or later" in paragraph
+        assert "`DEVOPS_AGENT_IAM_WAIT`" in paragraph and "a minute" in paragraph and "is not sent again" in paragraph
+        assert "`make deploy` runs no assessment" in paragraph and "name one service first" in paragraph
+        assert "`DEVOPS_AGENT_SPACE_ARN=<arn>`" in paragraph and "is not created" in paragraph
+        assert "the deploy is done" in paragraph and "run `make devops-agent-connect` as above" in paragraph
+        assert "\u2014" not in paragraph and " -- " not in paragraph
+
+    def test_the_readme_and_the_makefile_name_the_same_variables(self):
+        makefile = (DEPLOYMENT / "Makefile").read_text()
+        for name in ("DEVOPS_AGENT_CONNECT", "DEVOPS_AGENT_IAM_WAIT"):
+            assert name in makefile and name in _section(), name
+
     def test_it_gives_the_price_and_the_command_that_shows_the_usage(self):
         section = _section()
         assert "$0.0083 per agent-second" in section and "`aws devops-agent get-account-usage`" in section