diff --git a/.github/workflows/PMM_PDPGSQL.yaml b/.github/workflows/PMM_PDPGSQL.yaml index 47852bbec..2f2ba594b 100644 --- a/.github/workflows/PMM_PDPGSQL.yaml +++ b/.github/workflows/PMM_PDPGSQL.yaml @@ -66,12 +66,7 @@ jobs: working-directory: qa-integration/pmm_qa run: | mkdir -m 777 -p /tmp/backup_data - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install --force-reinstall -U setuptools - pip install -r requirements.txt - python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }} + ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }} - name: Setup npm modules for CodeceptJS working-directory: codeceptjs-e2e diff --git a/.github/workflows/PMM_PROXYSQL.yaml b/.github/workflows/PMM_PROXYSQL.yaml index 042ea28b2..b9063c597 100644 --- a/.github/workflows/PMM_PROXYSQL.yaml +++ b/.github/workflows/PMM_PROXYSQL.yaml @@ -99,15 +99,10 @@ jobs: working-directory: qa-integration/pmm_qa run: | mkdir -m 777 -p /tmp/backup_data - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install --force-reinstall -U setuptools - pip install -r requirements.txt if [[ "${PXC_VERSION}" != http* ]]; then - python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${PXC_VERSION} + ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${PXC_VERSION} else - python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC,TARBALL=${PXC_TARBALL_PATH} + ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC,TARBALL=${PXC_TARBALL_PATH} fi - name: Install npx dependencies diff --git a/.github/workflows/gssapi-psmdb-tests-matrix.yml b/.github/workflows/gssapi-psmdb-tests-matrix.yml index d524b523d..af81758f0 100644 --- a/.github/workflows/gssapi-psmdb-tests-matrix.yml +++ b/.github/workflows/gssapi-psmdb-tests-matrix.yml @@ -185,7 +185,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose' + services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=8' test_name: 'PSMDB Replica 6.x' psmdb_7_replica-tests_ol8: @@ -199,7 +199,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose' + services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=8' test_name: 'PSMDB Replica 7.x' psmdb_8_replica-tests_ol8: @@ -213,7 +213,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose' + services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=8' test_name: 'PSMDB Replica 8.x' psmdb_6_replica-tests_ol9: @@ -227,7 +227,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose' + services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=9' test_name: 'PSMDB Replica 6.x' psmdb_7_replica-tests_ol9: @@ -241,7 +241,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose' + services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=9' test_name: 'PSMDB Replica 7.x' psmdb_8_replica-tests_ol9: @@ -255,7 +255,7 @@ jobs: pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }} pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }} cli_tag: '@psmdb' - services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose' + services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=9' test_name: 'PSMDB Replica 8.x' mongo_package_tests_integration_ol8: diff --git a/.github/workflows/integration-cli-tests.yml b/.github/workflows/integration-cli-tests.yml index f5e2e0635..1ba809177 100644 --- a/.github/workflows/integration-cli-tests.yml +++ b/.github/workflows/integration-cli-tests.yml @@ -379,7 +379,7 @@ jobs: pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }} launchable_confidence: ${{ inputs.launchable_confidence || '100%' }} cli_tag: '@psmdb' - services_list: '--database psmdb=6.0,SETUP_TYPE=pss --verbose' + services_list: '--database psmdb=6.0,SETUP_TYPE=pss' test_name: 'PSMDB Replica 6.x' psmdb-7-replica-tests: @@ -395,7 +395,7 @@ jobs: pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }} launchable_confidence: ${{ inputs.launchable_confidence || '100%' }} cli_tag: '@psmdb' - services_list: '--database psmdb=7.0,SETUP_TYPE=pss --verbose' + services_list: '--database psmdb=7.0,SETUP_TYPE=pss' test_name: 'PSMDB Replica 7.x' psmdb-8-tests: @@ -411,7 +411,7 @@ jobs: pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }} launchable_confidence: ${{ inputs.launchable_confidence || '100%' }} cli_tag: '@psmdb' - services_list: '--database psmdb=8.0,SETUP_TYPE=pss --verbose' + services_list: '--database psmdb=8.0,SETUP_TYPE=pss' test_name: 'PSMDB Replica 8.x' psmdb-7-shard-tests: @@ -427,7 +427,7 @@ jobs: pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }} launchable_confidence: ${{ inputs.launchable_confidence || '100%' }} cli_tag: '@shard-psmdb' - services_list: '--database psmdb=7.0,SETUP_TYPE=sharding --verbose' + services_list: '--database psmdb=7.0,SETUP_TYPE=sharding' test_name: 'PSMDB Shard 7.x' psmdb-8-shard-tests: @@ -443,7 +443,7 @@ jobs: pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }} launchable_confidence: ${{ inputs.launchable_confidence || '100%' }} cli_tag: '@shard-psmdb' - services_list: '--database psmdb=8.0,SETUP_TYPE=sharding --verbose' + services_list: '--database psmdb=8.0,SETUP_TYPE=sharding' test_name: 'PSMDB Shard 8.x' haproxy-tests: diff --git a/.github/workflows/runner-e2e-tests-codeceptjs-remote-nightly-setup.yml b/.github/workflows/runner-e2e-tests-codeceptjs-remote-nightly-setup.yml index 0894b4298..2364190ea 100644 --- a/.github/workflows/runner-e2e-tests-codeceptjs-remote-nightly-setup.yml +++ b/.github/workflows/runner-e2e-tests-codeceptjs-remote-nightly-setup.yml @@ -127,16 +127,11 @@ jobs: command: | cd pmm-qa/qa-integration/pmm_qa mkdir -m 777 -p /tmp/backup_data || true - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install -r requirements.txt - pip install setuptools - python pmm-framework.py \ + ./pmm-framework/pmm-framework \ + --parallel \ --pmm-server-ip "${SERVER_IP}" \ --pmm-server-password "${{ env.ADMIN_PASSWORD }}" \ --client-version "${{ env.PMM_CLIENT_VERSION }}" \ - --verbose \ ${{ env.WIZARD_ARGS }} on_retry_command: | docker rm -f $(docker ps -a -q) || true diff --git a/.github/workflows/runner-e2e-tests-codeceptjs.yml b/.github/workflows/runner-e2e-tests-codeceptjs.yml index 06c1363f1..0c983c4ee 100644 --- a/.github/workflows/runner-e2e-tests-codeceptjs.yml +++ b/.github/workflows/runner-e2e-tests-codeceptjs.yml @@ -221,12 +221,7 @@ jobs: working-directory: pmm-qa/qa-integration/pmm_qa run: | mkdir -m 777 -p /tmp/backup_data - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install -r requirements.txt - pip install setuptools - python pmm-framework.py --pmm-server-password=${{ env.ADMIN_PASSWORD }} --verbose ${{ env.WIZARD_ARGS }} + ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }} - name: Execute e2e tests with tags ${{ env.TAGS_FOR_TESTS }} with launchable if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }} diff --git a/.github/workflows/runner-e2e-tests-playwright.yml b/.github/workflows/runner-e2e-tests-playwright.yml index 73645f36c..b9155ec7b 100644 --- a/.github/workflows/runner-e2e-tests-playwright.yml +++ b/.github/workflows/runner-e2e-tests-playwright.yml @@ -193,12 +193,7 @@ jobs: if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }} working-directory: ./pmm-qa/qa-integration/pmm_qa run: | - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install -r requirements.txt - pip install setuptools - python pmm-framework.py --verbosity-level=2 --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }} + ./pmm-framework/pmm-framework --parallel --verbosity-level=2 --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }} - name: 'Run UI tests: ${{ env.PMM_TEST_FLAG }} with launchable' if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }} diff --git a/.github/workflows/runner-e2e-tests-podman.yml b/.github/workflows/runner-e2e-tests-podman.yml index f033215fc..7fbc52ec7 100644 --- a/.github/workflows/runner-e2e-tests-podman.yml +++ b/.github/workflows/runner-e2e-tests-podman.yml @@ -222,12 +222,7 @@ jobs: working-directory: pmm-qa/qa-integration/pmm_qa run: | export HOST_IP=$(ip -4 addr show docker0 | awk '/inet / {print $2}' | cut -d/ -f1) - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install -r requirements.txt - pip install setuptools - python pmm-framework.py --pmm-server-ip=$HOST_IP --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }} + ./pmm-framework/pmm-framework --parallel --pmm-server-ip=$HOST_IP --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }} - name: Setup npm modules for e2e tests working-directory: ./pmm-qa/codeceptjs-e2e diff --git a/.github/workflows/runner-integration-cli-tests.yml b/.github/workflows/runner-integration-cli-tests.yml index 2eb404d1c..94e27e177 100644 --- a/.github/workflows/runner-integration-cli-tests.yml +++ b/.github/workflows/runner-integration-cli-tests.yml @@ -21,7 +21,7 @@ on: required: true type: string services_list: - description: "pmm-framework.py flags to perform setup" + description: "pmm-framework flags to perform setup" required: false type: string cli_tag: @@ -189,12 +189,7 @@ jobs: if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }} working-directory: pmm-qa/qa-integration/pmm_qa run: | - python3 -m venv virtenv - . virtenv/bin/activate - pip install --upgrade pip - pip install setuptools - pip install -r requirements.txt - python pmm-framework.py --client-debug ${{ env.WIZARD_ARGS }} + ./pmm-framework/pmm-framework --parallel --client-debug ${{ env.WIZARD_ARGS }} - name: 'Run CLI tests: ${{ inputs.cli_tag }}' if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }} diff --git a/qa-integration/pmm_qa/client_container_pxc_setup.sh b/qa-integration/pmm_qa/client_container_pxc_setup.sh index 9ad714556..ed3c9e870 100644 --- a/qa-integration/pmm_qa/client_container_pxc_setup.sh +++ b/qa-integration/pmm_qa/client_container_pxc_setup.sh @@ -53,6 +53,12 @@ tar -xzf Percona-XtraDB-Cluster.tar.gz sleep 10 rm -r Percona-XtraDB-Cluster.tar.gz mv Percona-XtraDB-Cluster* PXC + +# Docker Desktop's Rosetta emulation exposes /proc//exe as an +# unresolvable /run/rosetta/rosetta link. Do not let readlink's failure +# terminate the SST helper before it can fall back to mysqld from PATH. +sed -i 's#MYSQLD_PATH=$(readlink -f /proc/${WSREP_SST_OPT_PARENT}/exe)#MYSQLD_PATH=$(readlink -f /proc/${WSREP_SST_OPT_PARENT}/exe || true)#' PXC/bin/wsrep_sst_common + cd PXC ## start PXC diff --git a/qa-integration/pmm_qa/haproxy_setup.yml b/qa-integration/pmm_qa/haproxy_setup.yml index 1ba497b60..c0d6cf19d 100644 --- a/qa-integration/pmm_qa/haproxy_setup.yml +++ b/qa-integration/pmm_qa/haproxy_setup.yml @@ -44,7 +44,7 @@ - name: Setup haproxy for monitoring shell: "{{ item }}" with_items: - - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh > setup_haproxy.log + - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh - name: Install pmm3-client on the haproxy_container shell: "{{ item }}" diff --git a/qa-integration/pmm_qa/mlaunch_modb_setup.yml b/qa-integration/pmm_qa/mlaunch_modb_setup.yml index dfb749c00..aeab6a50b 100644 --- a/qa-integration/pmm_qa/mlaunch_modb_setup.yml +++ b/qa-integration/pmm_qa/mlaunch_modb_setup.yml @@ -1,8 +1,6 @@ --- - hosts: all - become: true - become_method: sudo vars: modb_version: "{{ lookup('vars', 'extra_modb_version', default=lookup('env','MODB_VERSION') | default('4.4', true) ) }}" modb_tarball: "{{ lookup('vars', 'extra_modb_tarball', default=lookup('env','MODB_TARBALL') | default('', true) ) }}" @@ -82,4 +80,4 @@ shell: "{{ item }}" with_items: - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log - when: modb_setup == "sharded" \ No newline at end of file + when: modb_setup == "sharded" diff --git a/qa-integration/pmm_qa/mlaunch_psmdb_setup.yml b/qa-integration/pmm_qa/mlaunch_psmdb_setup.yml index 7b6fa3543..dabf266ce 100644 --- a/qa-integration/pmm_qa/mlaunch_psmdb_setup.yml +++ b/qa-integration/pmm_qa/mlaunch_psmdb_setup.yml @@ -1,8 +1,6 @@ --- - hosts: all - become: true - become_method: sudo vars: psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}" psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}" @@ -82,4 +80,4 @@ shell: "{{ item }}" with_items: - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log - when: psmdb_setup == "sharded" \ No newline at end of file + when: psmdb_setup == "sharded" diff --git a/qa-integration/pmm_qa/ms_pmm_setup.yml b/qa-integration/pmm_qa/ms_pmm_setup.yml index 62915e549..8afa3da71 100644 --- a/qa-integration/pmm_qa/ms_pmm_setup.yml +++ b/qa-integration/pmm_qa/ms_pmm_setup.yml @@ -3,8 +3,6 @@ # enables Percona testing repository - hosts: all - become: true - become_method: sudo vars: ms_version: "{{ lookup('vars', 'extra_ms_version', default=lookup('env','MS_VERSION') | default('8.0', true) ) }}" ms_tarball: "{{ lookup('vars', 'extra_ms_tarball', default=lookup('env','MS_TARBALL') | default('', true) ) }}" diff --git a/qa-integration/pmm_qa/mysql/mysql-setup.yml b/qa-integration/pmm_qa/mysql/mysql-setup.yml index 077472d9f..5bff3b6ee 100644 --- a/qa-integration/pmm_qa/mysql/mysql-setup.yml +++ b/qa-integration/pmm_qa/mysql/mysql-setup.yml @@ -43,8 +43,12 @@ ignore_errors: true - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - loop: "{{ range(1, nodes_count | int + 1) | list }}" + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: @@ -54,7 +58,10 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Recursively change ownership of a directory - shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data" + shell: | + docker run --rm \ + -v "{{ data_dir }}/node{{ item }}/data:/data" \ + {{ busybox_image }} chown -R 1001:1001 /data loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Setup MySQL group replication diff --git a/qa-integration/pmm_qa/mysql/tasks/prepare_install_mysql.yml b/qa-integration/pmm_qa/mysql/tasks/prepare_install_mysql.yml index 595de6a7d..396ede200 100644 --- a/qa-integration/pmm_qa/mysql/tasks/prepare_install_mysql.yml +++ b/qa-integration/pmm_qa/mysql/tasks/prepare_install_mysql.yml @@ -5,9 +5,11 @@ - name: Prepare socket folders shell: | - sudo rm -fr /tmp/mysql-sockets/{{ item }} - sudo mkdir -p /tmp/mysql-sockets/{{ item }} - sudo chmod 0777 /tmp/mysql-sockets/{{ item }} + docker run --rm -v /tmp:/host-tmp {{ busybox_image }} sh -c ' + rm -rf /host-tmp/mysql-sockets/{{ item }} + mkdir -p /host-tmp/mysql-sockets/{{ item }} + chmod 0777 /host-tmp/mysql-sockets/{{ item }} + ' - name: Prepare Container for MySQL 8.0+ shell: | diff --git a/qa-integration/pmm_qa/pdpgsql_pgsm_setup.yml b/qa-integration/pmm_qa/pdpgsql_pgsm_setup.yml index 92668c692..0068ff58b 100644 --- a/qa-integration/pmm_qa/pdpgsql_pgsm_setup.yml +++ b/qa-integration/pmm_qa/pdpgsql_pgsm_setup.yml @@ -3,8 +3,6 @@ # enables Percona testing repository - hosts: all - become: true - become_method: sudo vars: pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('15', true) ) }}" pdpgsql_pgsm_container: "{{ lookup('vars', 'extra_pdpgsql_pgsm_container', default=lookup('env','PDPGSQL_PGSM_CONTAINER') | default('pdpgsql_pgsm', true) ) }}" @@ -32,7 +30,7 @@ - name: Prepare Container for PostgreSQL shell: > - sudo docker run -d --name={{ pdpgsql_pgsm_container }} + docker run -d --name={{ pdpgsql_pgsm_container }} -p {{ pdpgsql_pgsm_port }}:5432 phusion/baseimage:jammy-1.0.1 diff --git a/qa-integration/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml b/qa-integration/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml index 9dc9e84a7..758ac0b6c 100644 --- a/qa-integration/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml +++ b/qa-integration/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml @@ -34,7 +34,6 @@ - name: Create Docker network shell: docker network create {{ network_name }} ignore_errors: yes - become: true - name: Install Percona Distribution for Postgres include_tasks: tasks/install-percona-distribution-postgresql-single.yml diff --git a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml index 437834a7c..4effb4715 100644 --- a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml +++ b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-pdpgsql.yml @@ -1,3 +1,8 @@ +- name: Cleanup existing Percona Distribution for Postgresql containers + shell: docker rm -fv {{ container_prefix }}{{ item }} + loop: "{{ range(1, nodes_count | int + 1) | list }}" + ignore_errors: true + - name: Start containers for Percona Distribution for Postgresql shell: | docker run -d \ @@ -10,14 +15,12 @@ -v /var/lib/containerd \ -p {{ pdpgsql_port + (item - 1) }}:5432 \ antmelekhin/docker-systemd:ubuntu-24.04 - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install dependencies shell: | docker exec {{ container_prefix }}{{ item }} apt-get update docker exec {{ container_prefix }}{{ item }} apt-get -y install wget gnupg2 lsb-release curl - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install Percona Release @@ -25,25 +28,21 @@ docker exec {{ container_prefix }}{{ item }} wget https://repo.percona.com/apt/percona-release_latest.generic_all.deb docker exec {{ container_prefix }}{{ item }} dpkg -i ./percona-release_latest.generic_all.deb docker exec {{ container_prefix }}{{ item }} percona-release enable ppg-{{ pdpgsql_version }} - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install Percona Distribution for PostgreSQL shell: | docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y percona-postgresql-{{ pdpgsql_version }} percona-pgbackrest - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install PG Stat Monitor shell: docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y percona-pg-stat-monitor{{ pdpgsql_version }} - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" when: pgsm_branch | length == 0 - name: Prepare installation of PG Stat Monitor from sources shell: | docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y git clang-18 llvm-18 build-essential percona-postgresql-server-dev-{{ pdpgsql_version }} - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" when: pgsm_branch | length > 0 @@ -55,15 +54,15 @@ make USE_PGXS=1 && make USE_PGXS=1 install ' - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" when: pgsm_branch | length > 0 - name: Start Percona distribution for Postgresql - shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql - become: true + shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql@{{ pdpgsql_version }}-main loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Wait 5 seconds for Percona Distribution for postgresql to start - pause: - seconds: 5 +- name: Wait until PostgreSQL is ready after install + include_tasks: ../tasks/wait_pgsql_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" diff --git a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-percona-distribution-postgresql-single.yml b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-percona-distribution-postgresql-single.yml index b8295ee44..9ff61b098 100644 --- a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-percona-distribution-postgresql-single.yml +++ b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/install-percona-distribution-postgresql-single.yml @@ -1,27 +1,28 @@ - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - become: true + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: path: "{{ data_dir }}/node{{ item }}/data" state: directory mode: '0755' - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate pg_hba.conf for primary node template: src: data/pg_hba.conf.j2 dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate postgres.conf for primary node template: src: data/postgresql-single.conf.j2 dest: "{{ data_dir }}/node1/postgresql.conf" - become: true - name: Install Percona Distribution for Postgres include_tasks: tasks/install-pdpgsql.yml @@ -39,12 +40,20 @@ shell: | docker cp {{ data_dir }}/node{{ item }}/postgresql.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf docker cp {{ data_dir }}/node{{ item }}/pg_hba.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf - become: true + # docker cp preserves host UIDs; Debian/pg_ctlcluster rejects non-postgres/non-root owners. + docker exec -u root {{ container_prefix }}{{ item }} chown postgres:postgres \ + /etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf \ + /etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Restart Percona distribution for Postgresql - shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl restart postgresql - become: true + shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl restart postgresql@{{ pdpgsql_version }}-main + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Wait until PostgreSQL is ready + include_tasks: ../tasks/wait_pgsql_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password and grant pg_monitor @@ -59,38 +68,36 @@ END \$\$; " - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Add service to pmm server shell: | docker exec {{ container_prefix }}{{ item }} pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor {{ container_prefix }}{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 docker exec {{ container_prefix }}{{ item }} pmm-admin add postgresql --username=pmm --password=pmm --query-source=pgstatmonitor --socket=/var/run/postgresql socket_{{ container_prefix }}{{ item }}{{ random_service_name_value }} - become: true + loop: "{{ range(1, nodes_count | int + 1) | list }}" + +- name: Run queries for generating data + shell: docker exec {{ container_prefix }}{{ item }} bash ./data/pgsm_run_queries.sh & loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Copy query load scripts into the container shell: | docker cp ./data/pgsql_load.sql {{ container_prefix }}{{ item }}:/pgsql_load.sql docker cp ./data/pdpgsql_run_queries.sh {{ container_prefix }}{{ item }}:/pdpgsql_run_queries.sh - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Create database if it doesn't exist shell: docker exec {{ container_prefix }}{{ item }} psql -U postgres -c "CREATE DATABASE school;" - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run SQL script using docker exec shell: docker exec {{ container_prefix }}{{ item }} psql -U postgres -d school -f /pgsql_load.sql > /dev/null - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run queries for generating data shell: docker exec {{ container_prefix }}{{ item }} bash /pdpgsql_run_queries.sh & - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Change postgres user password shell: | - docker exec -u postgres {{ container_prefix }}1 psql -U postgres -c "ALTER USER postgres WITH PASSWORD 'pass+this';" \ No newline at end of file + docker exec -u postgres {{ container_prefix }}1 psql -U postgres -c "ALTER USER postgres WITH PASSWORD 'pass+this';" diff --git a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml index e5d794908..ac17de305 100644 --- a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml +++ b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-patroni-setup.yml @@ -8,48 +8,46 @@ pdpgsql_port: 6432 - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - become: true + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: path: "{{ data_dir }}/node{{ item }}/data" state: directory mode: '0755' - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate postgres.conf for primary node template: src: data/postgresql-primary.conf.j2 dest: "{{ data_dir }}/node1/postgresql.conf" - become: true - name: Generate etcd configuration template: src: data/etcd.conf.yaml.j2 dest: "{{ data_dir }}/node{{ item }}/etcd.conf.yaml" - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate patroni configuration template: src: data/patroni.yml.j2 dest: "{{ data_dir }}/node{{ item }}/patroni.yml" - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate postgres.conf for primary node template: src: data/postgresql-primary.conf.j2 dest: "{{ data_dir }}/node1/postgresql.conf" - become: true - name: Generate postgres.conf for replica nodes template: src: data/postgresql-replica.conf.j2 dest: "{{ data_dir }}/node{{ item }}/postgresql.conf" - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Install Percona distribution for Postgresql @@ -59,14 +57,12 @@ shell: | docker cp {{ data_dir }}/node{{ item }}/patroni.yml {{ container_prefix }}{{ item }}:/patroni.yml docker cp {{ data_dir }}/node{{ item }}/etcd.conf.yaml {{ container_prefix }}{{ item }}:/etcd.conf.yaml - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install dependencies for Percona Distribution for PostgresSQL shell: | docker exec -u root {{ container_prefix }}{{ item }} apt-get install -y percona-patroni etcd python3-etcd percona-pgbackrest docker exec -u root {{ container_prefix }}{{ item }} patroni --version - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Start etcd service @@ -83,22 +79,21 @@ chown -R postgres:postgres /pg_wal chown -R postgres:postgres /var/lib/postgresql/{{ pdpgsql_version }}/main/ ' - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Prepare replication user and config file for primary node shell: | docker cp {{ data_dir }}/node1/postgresql.conf {{ container_prefix }}1:/var/lib/postgresql/{{ pdpgsql_version }}/main + docker exec -u root {{ container_prefix }}1 chown postgres:postgres \ + /var/lib/postgresql/{{ pdpgsql_version }}/main/postgresql.conf docker exec -u postgres {{ container_prefix }}1 psql -c "CREATE ROLE replicator WITH LOGIN REPLICATION PASSWORD 'replPasswd';" docker exec -u postgres {{ container_prefix }}1 mkdir -p /var/lib/pgbackrest/archive/patroni_backup docker exec -u postgres {{ container_prefix }}1 chown -R postgres:postgres /var/lib/pgbackrest docker exec -u postgres {{ container_prefix }}1 pgbackrest --stanza=patroni_backup --pg1-path=/var/lib/postgresql/{{ pdpgsql_version }}/main stanza-create - become: true - name: Remove data folder on replica nodes. shell: | docker exec {{ container_prefix }}{{ item }} rm -fr /var/lib/postgresql/{{ pdpgsql_version }}/main - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Install pg stat monitor. @@ -109,27 +104,22 @@ docker exec -u postgres {{ container_prefix }}1 bash -c ' psql -d {{ db_name | default("postgres") }} -c "GRANT pg_monitor TO postgres;" ' - become: true - name: Change postgres user password shell: | docker exec -u postgres {{ container_prefix }}1 psql -U postgres -c "ALTER USER postgres WITH PASSWORD 'pass+this';" - become: true - name: Stop Percona distribution for Postgresql, due to being managed by patroni shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl stop postgresql - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Start patroni service in all nodes shell: > docker exec -u postgres {{ container_prefix }}{{ item }} /bin/sh -c "nohup patroni /patroni.yml > /data/db/logs/patroni.log 2>&1 &" - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Show logs from patroni for debug shell: docker exec -u postgres {{ container_prefix }}{{ item }} cat /data/db/logs/patroni.log - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install and add pmm client. @@ -140,7 +130,6 @@ - name: Get already connected services to pmm server shell: docker exec {{ container_prefix }}1 sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ "80" if pmm_server_ip is match('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$') else "8080" }}/v1/management/services" | jq -r ".services[].service_name"' - become: true register: pmm_server_services - name: Display already connected services to pmm server @@ -157,30 +146,23 @@ - name: Add service to pmm server shell: docker exec {{ container_prefix }}{{ item }} pmm-admin add postgresql --username=postgres --cluster=pdpgsql_patroni_cluster --environment=pdpgsql_patroni_environment --password=pass+this --query-source=pgstatmonitor {{ container_prefix }}{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Add patroni primary service to pmm server shell: docker exec {{ container_prefix }}1 pmm-admin add external --listen-port=8008 --service-name=patroni_service_1{{ random_service_name_value }} - become: true - name: Add patroni replication service to pmm server shell: docker exec {{ container_prefix }}{{ item }} pmm-admin add external --listen-port=8008 --cluster=pdpgsql_patroni_service_cluster --environment=pdpgsql_patroni_service_environment --service-name=patroni_service_{{ item }}{{ random_service_name_value }} - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Copy a file into the container shell: docker cp ./data/pgsql_load.sql {{ container_prefix }}1:/pgsql_load.sql - become: true - name: Create database if it doesn't exist shell: docker exec {{ container_prefix }}1 psql -U postgres -c "CREATE DATABASE school;" - become: true - name: Run SQL script using docker exec shell: docker exec {{ container_prefix }}1 psql -U postgres -d school -f /pgsql_load.sql > /dev/null - become: true - name: Log Patroni cluster shell: docker exec {{ container_prefix }}1 patronictl -c /patroni.yml list - become: true diff --git a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml index 9700dcf78..770f8ef10 100644 --- a/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml +++ b/qa-integration/pmm_qa/percona-distribution-postgresql/tasks/percona-distribution-postgres-replication-setup.yml @@ -4,50 +4,47 @@ when: nodes_count | int < 2 and setup_type == "replication" - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - become: true + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: path: "{{ data_dir }}/node{{ item }}/data" state: directory mode: '0755' - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Generate pg_hba.conf for primary node template: src: data/pg_hba.conf.j2 dest: "{{ data_dir }}/node1/pg_hba.conf" - become: true - name: Generate postgres.conf for primary node template: src: data/postgresql-primary.conf.j2 dest: "{{ data_dir }}/node1/postgresql.conf" - become: true - name: Generate postgres.conf for replica nodes template: src: data/postgresql-replica.conf.j2 dest: "{{ data_dir }}/node{{ item }}/postgresql.conf" - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Generate pg_hba.conf for replica node template: src: data/pg_hba_replica.conf.j2 dest: "{{ data_dir }}/node{{ item }}/pg_hba.conf" - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Fix permissions on data directory - become: true - file: - path: "{{ data_dir }}/node{{ item }}/data" - owner: 1001 - group: 1001 - recurse: yes + shell: | + docker run --rm \ + -v "{{ data_dir }}/node{{ item }}/data:/data" \ + {{ busybox_image }} chown -R 1001:1001 /data loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Prepare and install Percona Distribution for Postgresql @@ -60,30 +57,29 @@ - name: Stop Percona distribution for Postgresql shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl stop postgresql - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Copy config files to primary node shell: | docker cp {{ data_dir }}/node1/postgresql.conf {{ container_prefix }}1:/etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf docker cp {{ data_dir }}/node1/pg_hba.conf {{ container_prefix }}1:/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf - become: true + docker exec -u root {{ container_prefix }}1 chown postgres:postgres \ + /etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf \ + /etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf - name: Start Percona distribution for Postgresql primary node - shell: docker exec -u root {{ container_prefix }}1 systemctl start postgresql - become: true + shell: docker exec -u root {{ container_prefix }}1 systemctl start postgresql@{{ pdpgsql_version }}-main -- name: Wait 5 seconds for Percona Distribution for postgresql to start - pause: - seconds: 5 +- name: Wait until PostgreSQL primary is ready + include_tasks: ../tasks/wait_pgsql_ready.yml + vars: + container_name: "{{ container_prefix }}1" - name: Create replication user shell: docker exec {{ container_prefix }}1 psql -U postgres -c "CREATE ROLE {{ replication_user }} WITH REPLICATION LOGIN ENCRYPTED PASSWORD '{{ replication_password }}';" - become: true - name: Wipe replica data directory before basebackup shell: docker exec {{ container_prefix }}{{ item }} rm -rf /var/lib/postgresql/{{ pdpgsql_version }}/main/ - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Create PostgreSQL user 'pmm' with password and grant pg_monitor @@ -98,7 +94,6 @@ END \$\$; " - become: true - name: Create custom database shell: | @@ -113,7 +108,6 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO pmm; \" | psql -U postgres -v ON_ERROR_STOP=1 " - become: true - name: Run pg_basebackup from primary to replica shell: | @@ -123,25 +117,27 @@ pg_basebackup --pgdata=/var/lib/postgresql/{{ pdpgsql_version }}/main -R -v -Fp -Xs -P \ --host={{ container_prefix}}1 --port=5432 -U {{ replication_user }} " - become: true loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Copy config files to replica nodes shell: | docker cp {{ data_dir }}/node{{ item }}/postgresql.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf docker cp {{ data_dir }}/node{{ item }}/pg_hba.conf {{ container_prefix }}{{ item }}:/etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf - docker exec {{ container_prefix}}{{ item }} chown -R postgres:postgres /var/lib/postgresql/{{ pdpgsql_version }}/main - become: true + docker exec -u root {{ container_prefix }}{{ item }} chown postgres:postgres \ + /etc/postgresql/{{ pdpgsql_version }}/main/postgresql.conf \ + /etc/postgresql/{{ pdpgsql_version }}/main/pg_hba.conf + docker exec {{ container_prefix }}{{ item }} chown -R postgres:postgres /var/lib/postgresql/{{ pdpgsql_version }}/main loop: "{{ range(2, nodes_count | int + 1) | list }}" - name: Start Percona distribution for Postgresql - shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql - become: true + shell: docker exec -u root {{ container_prefix }}{{ item }} systemctl start postgresql@{{ pdpgsql_version }}-main loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Wait 5 seconds for Percona Distribution for postgresql to start - pause: - seconds: 5 +- name: Wait until PostgreSQL nodes are ready + include_tasks: ../tasks/wait_pgsql_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install and add pmm client. include_tasks: ../tasks/install_pmm_client.yml @@ -151,7 +147,6 @@ - name: Get already connected services to pmm server shell: docker exec {{ container_prefix }}1 sh -c 'curl --location --insecure -u"admin:{{ admin_password }}" -s --request GET "http://{{ pmm_server_ip }}:{{ "80" if pmm_server_ip is match('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(\.|$)){4}$') else "8080" }}/v1/management/services" | jq -r ".services[].service_name"' - become: true register: pmm_server_services - name: Display already connected services to pmm server @@ -168,7 +163,6 @@ - name: Add service to pmm server shell: docker exec {{ container_prefix }}{{ item }} pmm-admin add postgresql --username=pmm --password=pmm --cluster=pdpgsql_replication_cluster --environment=pdpgsql_replication_environment --query-source=pgstatmonitor {{ container_prefix}}{{ item }}{{ random_service_name_value }} --debug 127.0.0.1:5432 - become: true loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Create pg_custom_publication view using psql @@ -184,34 +178,26 @@ FROM pg_publication; SQL' - become: true - name: Copy sample load into container shell: docker cp ../data/load_pgsql.sql {{ container_prefix }}1:/load_pgsql.sql - become: true - name: Start permanent SQL load in background shell: | docker exec {{ container_prefix }}1 sh -c 'nohup bash -c "while true; do echo Starting insert at \$(date +\"%Y-%m-%d %H:%M:%S\"); psql -U postgres -d test_database -f /load_pgsql.sql; sleep 30; done" > /tmp/sql_loop.log 2>&1 &' - become: true - name: Create logical replication slots shell: docker exec {{ container_prefix }}1 psql -U postgres -d postgres -c "SELECT * FROM pg_create_logical_replication_slot('test_slot', 'test_decoding');" - become: true - name: Copy a file into the container shell: docker cp ./data/pgsql_load.sql {{ container_prefix }}1:/pgsql_load.sql - become: true - name: Create database if it doesn't exist shell: docker exec {{ container_prefix }}1 psql -U postgres -c "CREATE DATABASE school;" - become: true - name: Run SQL script using docker exec shell: docker exec {{ container_prefix }}1 psql -U postgres -d school -f /pgsql_load.sql > /dev/null - become: true - name: Change postgres user password shell: | docker exec -u postgres {{ container_prefix }}1 psql -U postgres -c "ALTER USER postgres WITH PASSWORD 'pass+this';" - become: true diff --git a/qa-integration/pmm_qa/percona_server_for_mysql/percona-server-setup.yml b/qa-integration/pmm_qa/percona_server_for_mysql/percona-server-setup.yml index 91c85a057..80d696c43 100644 --- a/qa-integration/pmm_qa/percona_server_for_mysql/percona-server-setup.yml +++ b/qa-integration/pmm_qa/percona_server_for_mysql/percona-server-setup.yml @@ -52,8 +52,12 @@ ignore_errors: true - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' - loop: "{{ range(1, nodes_count | int + 1) | list }}" + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: @@ -63,7 +67,10 @@ loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Recursively change ownership of a directory - shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data" + shell: | + docker run --rm \ + -v "{{ data_dir }}/node{{ item }}/data:/data" \ + {{ busybox_image }} chown -R 1001:1001 /data loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Setup Percona Server group replication @@ -78,9 +85,13 @@ include_tasks: tasks/percona-server-setup-single.yml when: setup_type != "gr" and setup_type != "replication" - - name: Wait 10 seconds for setup to finish - pause: - seconds: 10 + - name: Wait until MySQL is ready on all nodes + include_tasks: ../tasks/wait_mysql_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" + mysql_user: root + mysql_password: "{{ root_password }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Install Percona XtraBackup for MySQL backups include_tasks: tasks/backup_setup.yml @@ -155,21 +166,23 @@ when: setup_type == "gr" or setup_type == "replication" - name: Run load for sysbench inside of all percona server nodes - shell: docker exec {{ container_prefix }}{{ item }} sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=sbtest --mysql-password=password --mysql-db=sbtest --tables=10 --table-size=100000 --threads=16 --time=60 run + shell: docker exec {{ container_prefix }}{{ item }} sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=sbtest --mysql-password=password --mysql-db=sbtest --tables=10 --table-size=100000 --threads=16 --time=30 run loop: "{{ range(1, nodes_count | int + 1) | list }}" when: setup_type != "gr" and setup_type != "replication" - name: Run load for sysbench inside of primary percona server node - shell: docker exec {{ container_prefix }}1 sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=sbtest --mysql-password=password --mysql-db=sbtest --tables=10 --table-size=100000 --threads=16 --time=60 run + shell: docker exec {{ container_prefix }}1 sysbench /usr/share/sysbench/oltp_read_write.lua --mysql-host=127.0.0.1 --mysql-port=3306 --mysql-user=sbtest --mysql-password=password --mysql-db=sbtest --tables=10 --table-size=100000 --threads=16 --time=30 run when: setup_type == "gr" or setup_type == "replication" - name: Copy a load file into the container shell: docker cp ../data/mysql_load.sql {{ container_prefix }}{{ item }}:/ps_load.sql loop: "{{ range(1, nodes_count | int + 1) | list }}" - - name: Wait 10 seconds for node to be connected - pause: - seconds: 10 + - name: Wait until pmm-agent is connected on all nodes + include_tasks: ../tasks/wait_pmm_agent_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Run load inside of first percona server node shell: | @@ -187,4 +200,4 @@ - name: Enable MySQL MyRocks shell: docker exec {{ container_prefix }}{{ item }} ps-admin --enable-rocksdb -u root -p{{ root_password }} loop: "{{ range(1, nodes_count | int + 1) | list }}" - when: my_rocks | bool \ No newline at end of file + when: my_rocks | bool diff --git a/qa-integration/pmm_qa/percona_server_for_mysql/tasks/prepare_install_ps.yml b/qa-integration/pmm_qa/percona_server_for_mysql/tasks/prepare_install_ps.yml index 3ad0b1e5e..9e07084e1 100644 --- a/qa-integration/pmm_qa/percona_server_for_mysql/tasks/prepare_install_ps.yml +++ b/qa-integration/pmm_qa/percona_server_for_mysql/tasks/prepare_install_ps.yml @@ -63,9 +63,11 @@ shell: docker exec {{ container_prefix }}{{ item }} systemctl restart mysql loop: "{{ range(1, nodes_count | int + 1) | list }}" -- name: Wait 5 seconds for MySQL to start - pause: - seconds: 5 +- name: Wait until MySQL accepts connections after restart + include_tasks: ../tasks/wait_mysql_ready.yml + vars: + container_name: "{{ container_prefix }}{{ item }}" + loop: "{{ range(1, nodes_count | int + 1) | list }}" - name: Chance root password Percona Server for MySQL 5.7 shell: > diff --git a/qa-integration/pmm_qa/pgsql_pgss_setup.yml b/qa-integration/pmm_qa/pgsql_pgss_setup.yml index b1674c070..abfbc8cab 100644 --- a/qa-integration/pmm_qa/pgsql_pgss_setup.yml +++ b/qa-integration/pmm_qa/pgsql_pgss_setup.yml @@ -3,8 +3,6 @@ # install postgres with pgstatement enabled for QAN and PMM Integration testing - hosts: all - become: true - become_method: sudo vars: pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('14', true) ) }}" pgsql_pgss_container: "{{ lookup('vars', 'extra_pgsql_pgss_container', default=lookup('env','PGSQL_PGSS_CONTAINER') | default('pgsql_pgss', true) ) }}" @@ -28,7 +26,7 @@ - name: Prepare Container for PostgreSQL shell: > - sudo docker run -d --name={{ pgsql_pgss_container }} + docker run -d --name={{ pgsql_pgss_container }} -p {{ pgsql_pgss_port }}:5432 phusion/baseimage:jammy-1.0.1 diff --git a/qa-integration/pmm_qa/pmm-framework/ARCHITECTURE.md b/qa-integration/pmm_qa/pmm-framework/ARCHITECTURE.md new file mode 100644 index 000000000..8e19ea4b1 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/ARCHITECTURE.md @@ -0,0 +1,345 @@ +# pmm-framework (Bash) — architecture and contribution guide + +How the framework is put together, what happens on a run, and the exact steps +to extend it. For installing and *using* it, see [README.md](README.md). + +The framework does not provision anything itself. It is a **dispatcher**: it +turns a `--database` spec into a map of environment variables and hands that to +an existing Ansible playbook or shell script under `qa-integration/`. Almost +every question about behaviour is answered by asking *which env map was built, +and which playbook received it*. + +--- + +## 1. The big picture + +```mermaid +flowchart TB + subgraph CLI["Input"] + ARGS["--parallel --database ps=8.4,SETUP_TYPE=gr --database psmdb"] + end + + subgraph FW["pmm-framework (this directory)"] + direction TB + PARSE["lib/cli.sh
parse_args → DATABASE_SPECS"] + PRE["lib/execution.sh
preflight_database_setups"] + STRAT{"PARALLEL
still true?"} + SEQ["sequential
one spec at a time"] + PAR["parallel
all specs at once"] + SPEC["run_database_spec
parse_database_spec → DB_TYPE / DB_VERSION / DB_CONFIG"] + DISP["setups/dispatch.sh
dispatch_setup"] + SETUP["setups/*.sh
setup_NAME builds env_map"] + end + + subgraph BACK["Backends"] + PB["lib/ansible.sh
run_playbook"] + SC["lib/runners.sh
run_setup_script"] + end + + subgraph REAL["qa-integration/ (not part of this framework)"] + YML["Ansible playbooks"] + SH["docker-compose setup scripts"] + DOCKER[("Docker containers
+ PMM Client")] + end + + ARGS --> PARSE --> PRE --> STRAT + STRAT -- no --> SEQ --> SPEC + STRAT -- yes --> PAR --> SPEC + SPEC --> DISP --> SETUP + SETUP --> PB --> YML --> DOCKER + SETUP --> SC --> SH --> DOCKER +``` + +**The one rule to remember:** a setup function's only job is to build +`env_map`. That map is the contract with the playbook, which reads it via +`lookup('env', 'KEY')`. + +--- + +## 2. Module map + +| File | Responsibility | Depends on | +|---|---|---| +| `pmm-framework` | Bash version gate, path anchors, sources everything, calls `parse_args` then `run_database_setups` | — | +| `lib/common.sh` | `log_*`, `die`, `require_command`, `bool_string`, `normalize_client_version` | nothing | +| `lib/config.sh` | The catalogue: `register_database`, lookups, `resolve_value` | common | +| `lib/cli.sh` | `parse_args`, `parse_database_spec`, `print_help` | common, config | +| `lib/docker.sh` | `discover_pmm_server`, `resolve_pmm_server` | common | +| `lib/ansible.sh` | `run_playbook`, `print_env_map`, collection/interpreter setup | common | +| `lib/runners.sh` | `run_setup_script`, version/password/branch resolvers | common, config, ansible | +| `setups/*.sh` | One `setup_` per type, plus `dispatch_setup` | everything above | +| `lib/execution.sh` | `preflight_database_setups`, sequential and parallel strategies | everything above | + +Source order matters only because `lib/config.sh` runs `register_database` +calls and a validation loop at source time — both need `lib/common.sh`'s +`die()` already defined. + +--- + +## 3. What happens on a run + +```mermaid +sequenceDiagram + autonumber + participant U as User / CI + participant E as pmm-framework + participant C as lib/cli.sh + participant X as lib/execution.sh + participant D as setups/dispatch.sh + participant S as setup_NAME + participant B as run_playbook / run_setup_script + + U->>E: --parallel --database ps=8.4 --database psmdb + E->>C: parse_args + C-->>E: DATABASE_SPECS=(ps=8.4, psmdb) + E->>X: run_database_setups + X->>X: preflight — conflicts? server? curl? ansible? + Note over X: a conflict here turns --parallel off + loop each spec + X->>C: parse_database_spec + C-->>X: DB_TYPE, DB_VERSION, DB_CONFIG + X->>D: dispatch_setup + D->>S: setup_ps + S->>S: resolve version / client / options → env_map + S->>B: run_playbook 'percona-server-setup.yml' env_map + B-->>S: success or die + end + X-->>U: exit 0, or non-zero if any setup failed +``` + +### Preflight + +Every spec is parsed once before anything is provisioned, so a bad request +fails in seconds rather than halfway through. Preflight decides: + +- **does anything need a PMM Server?** — `BUCKET` and `DOCKERCLIENTS` do not, + so `--database bucket` works with no server running +- **does anything need `curl`?** — only the PSMDB patch lookup +- **do any two setups conflict?** — see below +- **warm up Ansible** before parallel jobs fork, so they cannot race to install + the same collection + +### The conflict rule + +Two setups of the **same type**, or any two of the **MySQL family** +(`PS`/`MYSQL`), reuse the same container names, host ports and data +directories. They cannot run at the same time. + +When `--parallel` is asked for and a conflict exists, the framework keeps every +setup and gives up only the concurrency: + +``` +WARNING: Running setups sequentially: two PS setups cannot run in parallel. +``` + +This matters because CI passes `--parallel` unconditionally; refusing the run +would fail jobs that are perfectly valid, just not parallelisable. + +### Sequential vs parallel + +| | sequential | parallel | +|---|---|---| +| Order | argument order | all at once, reported as they finish | +| Output | streams straight to the console | buffered per setup, printed whole | +| On failure | stops immediately | every setup still finishes, run exits non-zero | +| Successful logs | stream live (not buffered) | summary line only, unless `--verbose` | +| Failed logs | stream live | always dumped, plus the directory is kept | + +A **failed** parallel setup always dumps its buffered log — no flag needed, +since that is what makes a broken setup diagnosable. A **successful** one +prints just its summary line, so a green CI run stays short; add `--verbose` +to echo those too when you want to see what a passing setup actually did. + +The CI runners deliberately do **not** pass `--verbose`, so day-to-day runs +stay quiet. Individual callers can still opt in through `services_list` / +`setup_services`. + +Parallel mode enables job control (`set -m`) so each setup gets its own process +group. That way an interrupt takes down `ansible-playbook` and its children +too, not just the wrapper subshell. It is also why each job gets +`SETUP_TYPE=gr ./pmm-framework ..."] --> B["2. global flag
--client-version
(CLIENT_VERSION only)"] + B --> C["3. spec option
--database ps,SETUP_TYPE=gr"] + C --> D["4. registered default
lib/config.sh"] +``` + +Two resolvers implement this, and they differ on purpose: + +| Helper | Used for | Empty env var | +|---|---|---| +| `resolve_value TYPE KEY MAP` | spec options | **wins** — yields `''`, mirroring Python's `os.environ.get` | +| `resolved_version ENV TYPE REQ` | versions | **skipped** — mirrors Python's `os.getenv(X) or ...` | + +> ⚠️ `resolve_value` looks variables up by name, and bash sees non-exported +> shell variables too. Never name a global in `lib/cli.sh` after a registered +> option key, or it will silently win over the spec. + +Versions have their own rule: the **order of the version list carries no +meaning**. The default comes from the explicit `DEFAULT_VERSION=` entry, and a +versioned type that omits it fails at startup. + +--- + +## 5. How to extend + +### Add a new database type + +Three files, in order. Say you are adding `FOODB`: + +**1 — register it** in `lib/config.sh`: + +```bash +register_database FOODB \ + '1.0 2.0' \ + 'CLIENT_VERSION SETUP_TYPE TARBALL' \ + 'DEFAULT_VERSION=2.0' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=' 'TARBALL=' +``` + +Every option key needs a matching default — an option without one silently +resolves to `''`. + +**2 — write the setup function** in the matching `setups/*.sh` (or a new file, +sourced from `pmm-framework`): + +```bash +# FooDB, monitored by PMM. +setup_foodb() { + local version setup_type client + version=$(resolved_version FOODB_VERSION FOODB "$DB_VERSION") + setup_type=$(resolve_value FOODB SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + client=$(resolved_client_version FOODB DB_CONFIG) + + declare -A env_map=( + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [FOODB_VERSION]="$version" + [SETUP_TYPE]="$setup_type" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'foodb/foodb-setup.yml' env_map +} +``` + +Keys must match exactly what the playbook reads. A key the playbook ignores is +dead weight; a key it expects but you omit falls back to the playbook's own +`default(...)` with **no error**, which is the most common way to get a setup +that "succeeds" but is misconfigured. + +**3 — wire up dispatch** in `setups/dispatch.sh`: + +```bash +FOODB) setup_foodb ;; +``` + +Then check the two capability predicates: + +- `setup_requires_server` (`setups/dispatch.sh`) — add it if it needs **no** + PMM Server +- `setup_uses_ansible` (`lib/execution.sh`) — add it if it is **script**-backed + +**4 — add a test** in `tests/dispatch.bats`. The suite stubs the backends and +asserts on the captured env map, so no containers are involved: + +```bash +@test "FooDB selects its playbook and environment" { + parse_database_spec 'foodb=2.0,SETUP_TYPE=cluster' + dispatch_setup + + [[ $CAPTURE_KIND == playbook ]] + [[ $CAPTURE_TARGET == foodb/foodb-setup.yml ]] + [[ ${CAPTURE_ENV[FOODB_VERSION]} == 2.0 ]] + [[ ${CAPTURE_ENV[SETUP_TYPE]} == cluster ]] +} +``` + +### Add a global flag + +1. default it in the block at the top of `lib/cli.sh` +2. add a `case` arm in `parse_args` — value-taking flags join the shared arm so + the "next argument looks like a flag" rule stays in one place +3. document it in `print_help` +4. read it where it applies, usually a key in one or more env maps + +Do not name it after a registered option key (see the warning above). + +### Add a new backend + +`run_playbook` and `run_setup_script` are the only two. Both take +`(target, env_map_name)`, pass variables through `env` rather than exporting, +and `die` on failure. A third backend should follow the same shape and be +reflected in `setup_uses_ansible`. + +--- + +## 6. Testing + +```bash +make check # bash -n, shellcheck -x, and the full bats suite +make test # bats only +``` + +Three suites, none of which start a container: + +| Suite | Covers | +|---|---| +| `tests/cli.bats` | parsing, precedence, the catalogue, server discovery, log formatting | +| `tests/dispatch.bats` | each type selects the right playbook/script and env map | +| `tests/integration.bats` | the real entrypoint with stubbed `docker`/`ansible-playbook`/`curl` | + +`tests/helpers/test_helper.bash` sources the modules and **replaces** +`run_playbook` and `run_setup_script` with capture stubs, so a test can assert +on `CAPTURE_KIND`, `CAPTURE_TARGET` and `CAPTURE_ENV` without provisioning +anything. `reset_framework_state` runs before each test. + +`tests/integration.bats` takes the opposite approach: it puts fake `docker`, +`ansible-playbook` and `curl` executables on `PATH` and runs the real +entrypoint end to end. + +When you change behaviour, make the test fail first. A test that passes both +before and after a fix is not testing the fix. + +--- + +## 7. Conventions and gotchas + +**Bash 5.1 or newer.** The version gate in `pmm-framework` currently says 4.4, +but `wait -n -p` in `run_parallel_setups` needs 5.1 — worth tightening. + +**`set -euo pipefail` plus `inherit_errexit`.** A failing `$(...)` aborts the +run instead of yielding an empty string. Beware `local x=$(...)`: `local` +masks the exit status, so always split the declaration from the assignment. + +**Value helpers print, predicates return.** `helper` that yields a value writes +it to stdout with no trailing newline; one that answers a question returns 0/1. + +**Arrays are passed by name.** Bash cannot pass an associative array by value, +so `run_playbook 'x.yml' env_map` takes the *name* and re-binds it with +`local -n`. That is why a caller must never name a local `env_ref` or +`map_ref` — it would collide with the nameref and error. + +**Env maps are written out in full.** The repetition across setup functions is +deliberate; the differences between them are real (`setup_external` omits +`CLIENT_DEBUG`, the PSMDB setups use `PMM_CLIENT_VERSION`). Factoring out the +common keys would hide those asymmetries. + +**Unknown versions and options are not fatal.** They are noted under +`--verbose` and the default is used, matching the Python framework so a typo +degrades instead of failing a long CI job. Unknown *database names* are fatal. + +**`die` exits the current shell.** At top level that ends the run; inside +`$(...)` or a parallel job it ends only that subshell, and `set -e` propagates +the failure outward. diff --git a/qa-integration/pmm_qa/pmm-framework/Makefile b/qa-integration/pmm_qa/pmm-framework/Makefile new file mode 100644 index 000000000..b7f6f3233 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/Makefile @@ -0,0 +1,16 @@ +SHELL := /usr/bin/env bash + +.PHONY: check syntax lint test + +check: syntax lint test + +syntax: + bash -n pmm-framework *.sh lib/*.sh setups/*.sh tests/*.bats tests/helpers/*.bash + +lint: + @command -v shellcheck >/dev/null 2>&1 || { echo "shellcheck is required"; exit 1; } + shellcheck -x --exclude=SC2030,SC2031,SC2034 pmm-framework *.sh lib/*.sh setups/*.sh tests/*.bats tests/helpers/*.bash + +test: + @command -v bats >/dev/null 2>&1 || { echo "bats is required"; exit 1; } + bats tests diff --git a/qa-integration/pmm_qa/pmm-framework/README.md b/qa-integration/pmm_qa/pmm-framework/README.md new file mode 100644 index 000000000..b7630b5b6 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/README.md @@ -0,0 +1,180 @@ +# Bash PMM Framework + +This directory contains an isolated Bash implementation of +[`../pmm-framework.py`](../pmm-framework.py). It preserves the live command-line +interface and reuses the existing Ansible playbooks and setup scripts. The +Python implementation and CI workflows remain unchanged while both versions +are validated side by side. + +> **Contributing or reading the code?** See +> [ARCHITECTURE.md](ARCHITECTURE.md) for the module map, diagrams of the run +> flow, how value resolution works, and step-by-step instructions for adding a +> database type, a CLI flag, or a backend. + +## Requirements + +- Bash 4.4 or newer (associative arrays, name references, `inherit_errexit`, + and safe expansion of empty arrays under `set -u`) +- Docker with the Compose v2 plugin +- Ansible, including `ansible-playbook` and `ansible-galaxy` +- `curl` +- A running PMM Server container, or an explicit `--pmm-server-ip` + +The framework installs `community.docker` with `ansible-galaxy` if the +collection is missing. On macOS, ensure Homebrew Bash precedes `/bin/bash` in +`PATH`; Apple's bundled Bash 3.2 is unsupported. + +When the existing `pmm_framework` virtual environment is available, its Python +interpreter is automatically used for Ansible modules that require `requests`. +Set `ANSIBLE_PYTHON_INTERPRETER` explicitly to override this selection. + +The `SSL_PSMDB` setup uses Compose's `!reset` override tag and therefore +requires Docker Compose 2.24 or newer. It generates temporary script and +override files instead of modifying the tracked source compose or shell files. + +## Usage + +Run from any working directory: + +```bash +qa-integration/pmm_qa/pmm-framework/pmm-framework \ + --database ps=8.4 +``` + +Use an external PMM Server: + +```bash +qa-integration/pmm_qa/pmm-framework/pmm-framework \ + --pmm-server-ip 192.0.2.10 \ + --pmm-server-password admin \ + --client-version 3-dev-latest \ + --database pgsql=17,SETUP_TYPE=replication +``` + +Set up multiple databases sequentially: + +```bash +qa-integration/pmm_qa/pmm-framework/pmm-framework \ + --database ps=8.4,SETUP_TYPE=gr \ + --database psmdb=8.0,SETUP_TYPE=sharding,GSSAPI=true \ + --database valkey=8,SETUP_TYPE=sentinel +``` + +Run independent setups concurrently: + +```bash +qa-integration/pmm_qa/pmm-framework/pmm-framework \ + --parallel \ + --database ps=8.4 \ + --database pgsql=17,SETUP_TYPE=replication \ + --database valkey=8 +``` + +`--parallel` writes each setup's stdout/stderr to its own log file. Successful +setups print only a one-line summary with the log path as soon as they finish; +failed setups dump their buffered log immediately, with no flag required. Add +`--verbose` to echo the logs of successful setups as well. All setups are +allowed to finish, and the framework returns nonzero if any setup fails. On +failure the log directory is kept for inspection; on a fully successful run it +is removed. + +Each setup runs in its own process group, so interrupting the framework also +terminates the `ansible-playbook` processes it started. + +Setups that cannot run concurrently — two of the same database type, or any two +of the MySQL family (PS/MySQL), which share `mysql_cluster_data` and host +ports — are detected during preflight. The run is not rejected: it falls back +to sequential execution with a warning, so every requested setup still runs. + +To rerun the representative four-database setup and print its wall-clock time +to milliseconds: + +```bash +qa-integration/pmm_qa/pmm-framework/run_parallel_timing.sh +``` + +`--database` values use this grammar: + +```text +NAME[=VERSION][,OPTION=VALUE...] +``` + +Names and option keys are case-insensitive. Supported setup keys are: + +- `PS`, `MYSQL`, `SSL_MYSQL` +- `PGSQL`, `PDPGSQL`, `SSL_PDPGSQL` +- `PSMDB`, `MLAUNCH_PSMDB`, `MLAUNCH_MODB`, `SSL_MLAUNCH`, `SSL_PSMDB` +- `PXC`, `HAPROXY`, `EXTERNAL` +- `DOCKERCLIENTS`, `BUCKET`, `VALKEY` + +Aliases from the Python framework remain valid: `shards`/`sharding` for PSMDB +and `sentinel`/`sentinels` for Valkey. + +Configuration precedence is: + +1. Environment variable +2. Global `--client-version` (for `CLIENT_VERSION`) +3. Per-database option +4. Registered default + +Each registration in `lib/config.sh` pins the version used when a spec omits +one with an explicit `DEFAULT_VERSION=` entry. It is not a user-settable +option, and the order of the version list carries no meaning. Registering +versions without a valid `DEFAULT_VERSION` fails immediately at startup. + +A value passed as `--pmm-server-password` stays in the process command line for +the whole run and is readable by other users on the host via `ps`. On shared +runners, export `ADMIN_PASSWORD` instead; it takes precedence over the flag and +is not exposed in the command line. + +`latest-tarball` is normalized to the current PMM Client build-cache URL. +PSMDB patch lookup uses `curl`; it does not require Python or `requests`. + +## Side-by-side validation + +From `qa-integration/pmm_qa`, run equivalent commands: + +```bash +python3 pmm-framework.py \ + --pmm-server-ip 192.0.2.10 \ + --database ps=8.4,SETUP_TYPE=gr + +./pmm-framework/pmm-framework \ + --pmm-server-ip 192.0.2.10 \ + --database ps=8.4,SETUP_TYPE=gr +``` + +Use `--verbose` on either command to compare the selected setup and resolved +environment. The Bash implementation passes a fresh environment map to each +playbook or script, so repeated `--database` values cannot leak setup-specific +variables into subsequent runs. + +The old argparse-generated database subcommands and obsolete PMM 2 flags are +intentionally not implemented; current workflows use repeatable `--database`. + +## Development + +The entrypoint is intentionally thin. Shared behavior is under `lib/`, while +product-specific environment maps and dispatch live under `setups/`. +[ARCHITECTURE.md](ARCHITECTURE.md) walks through the call flow and the +extension points; every function also carries a header comment describing its +arguments, the globals it reads and writes, and how it reports failure. + +Install development tools (`bats-core` and `shellcheck`), then run: + +```bash +make check +``` + +Individual targets are available: + +```bash +make syntax +make lint +make test +``` + +Contract tests stub Docker, curl, `ansible-galaxy`, and `ansible-playbook`. +They verify parsing, defaults and precedence, aliases, playbook/script +selection, representative environment maps, failure behavior, and +multi-database ordering without provisioning containers. diff --git a/qa-integration/pmm_qa/pmm-framework/lib/ansible.sh b/qa-integration/pmm_qa/pmm-framework/lib/ansible.sh new file mode 100644 index 000000000..d76876e79 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/ansible.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# lib/ansible.sh -- running the existing Ansible playbooks. +# +# The framework owns no provisioning logic of its own. Each setup function +# builds an associative array of environment variables and hands it here; the +# playbooks under qa-integration/pmm_qa/ read those with `lookup('env', ...)` +# and do the actual work. +# +# That env map is the contract between a setup function and its playbook. If a +# playbook reads a variable the setup never sets, the playbook's own +# `| default(...)` decides -- there is no error, so a typo in a key name fails +# silently. Check both sides when adding one. + +# Guard so the collection check runs at most once per process. +ANSIBLE_COLLECTION_CHECKED=false + +# Point Ansible modules at a Python that has `requests`, if one is available. +# +# community.docker modules need `requests`; the interpreter Ansible discovers by +# default may not have it. Prefers an explicit ANSIBLE_PYTHON_INTERPRETER, then +# the repo's pmm_framework virtualenv. Silently does nothing when neither is +# usable, leaving Ansible's own discovery in charge. +# +# Reads: ANSIBLE_PYTHON_INTERPRETER, PMM_FRAMEWORK_ANSIBLE_PYTHON_FALLBACK, +# PMM_QA_ROOT +# Writes: exports ANSIBLE_PYTHON_INTERPRETER when a suitable interpreter is found +configure_ansible_python() { + [[ -n ${ANSIBLE_PYTHON_INTERPRETER:-} ]] && return + + local candidate=${PMM_FRAMEWORK_ANSIBLE_PYTHON_FALLBACK:-$PMM_QA_ROOT/pmm_framework/bin/python} + if [[ -x $candidate ]] && "$candidate" -c 'import requests' >/dev/null 2>&1; then + export ANSIBLE_PYTHON_INTERPRETER=$candidate + log_verbose "Using Ansible module interpreter: $ANSIBLE_PYTHON_INTERPRETER" + fi +} + +# Install the community.docker collection unless it is already present. +# +# Checked once per process, and pre-warmed by preflight before parallel setups +# so several concurrent jobs cannot race to install the same collection. +# +# Writes: ANSIBLE_COLLECTION_CHECKED +# Exits: via die() when the install fails +ensure_docker_collection() { + [[ $ANSIBLE_COLLECTION_CHECKED == true ]] && return + if ! ansible-galaxy collection list community.docker >/dev/null 2>&1; then + log_info "Installing Ansible collection community.docker..." + ansible-galaxy collection install community.docker || + die "Failed to install Ansible collection community.docker." + fi + ANSIBLE_COLLECTION_CHECKED=true +} + +# Print an env map as sorted ` KEY=value` lines, shell-quoted. +# +# Takes the array *by name* because bash cannot pass an associative array by +# value. Used by --verbose to show exactly what a playbook will receive. +# +# Usage: print_env_map env_map +# Stdout: one sorted ` KEY=value` line per entry +print_env_map() { + local map_name=$1 + local -n map_ref=$map_name + local key + for key in "${!map_ref[@]}"; do + printf ' %s=%q\n' "$key" "${map_ref[$key]}" + done | sort +} + +# Run a playbook with an env map, from the qa-integration/pmm_qa directory. +# +# Usage: run_playbook 'valkey/valkey-cluster.yml' env_map +# PLAYBOOK path relative to PMM_QA_ROOT +# MAP_NAME *name* of an associative array of KEY=value pairs +# +# Variables are passed with `env KEY=value ...` rather than exported, so nothing +# leaks between setups: each playbook sees exactly the map it was given. `env` +# execs ansible-playbook, so the values do not linger in the process command +# line either. +# +# VERBOSITY_LEVEL becomes that many -v flags (default 1). +# +# Every playbook is given vars/pinned_images.yml as --extra-vars, so pinned +# third-party image versions (e.g. busybox_image) have one place to bump +# instead of a literal repeated in each playbook that needs one. +# +# Reads: PMM_QA_ROOT, VERBOSE, VERBOSITY_LEVEL +# Exits: via die() when the playbook fails +run_playbook() { + local playbook=$1 map_name=$2 + local -n env_ref=$map_name + configure_ansible_python + ensure_docker_collection + + local -a env_args=() + local key + for key in "${!env_ref[@]}"; do + env_args+=("$key=${env_ref[$key]}") + done + + local -a verbosity_args=() + local index + for ((index = 0; index < VERBOSITY_LEVEL; index++)); do + verbosity_args+=(-v) + done + + log_verbose "Running playbook $playbook with:" + [[ $VERBOSE == true ]] && print_env_map "$map_name" + + ( + cd "$PMM_QA_ROOT" + env "${env_args[@]}" ansible-playbook \ + -i 'localhost,' \ + --connection=local \ + --extra-vars '@vars/pinned_images.yml' \ + "${verbosity_args[@]}" \ + "$playbook" + ) || die "$playbook playbook execution failed." + log_info "$playbook playbook execution successful" +} diff --git a/qa-integration/pmm_qa/pmm-framework/lib/cli.sh b/qa-integration/pmm_qa/pmm-framework/lib/cli.sh new file mode 100644 index 000000000..ff3ef8cca --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/cli.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# +# lib/cli.sh -- command-line parsing and the --database spec grammar. +# +# Turns argv into two things the rest of the framework reads: +# * global switches (PMM Server address, verbosity, --parallel, ...) +# * DATABASE_SPECS -- the raw spec strings, still unparsed +# +# Specs stay raw on purpose. preflight_database_setups() parses each one to +# inspect the run, then each setup parses it again just before running, so a +# spec is always expanded against freshly reset state. See lib/execution.sh. +# +# TO ADD A GLOBAL FLAG: +# 1. give it a default in the block below +# 2. add a case arm in parse_args (value-taking flags join the shared arm) +# 3. document it in print_help +# 4. read it wherever it applies -- most flags end up in a setup's env map + +# Raw `--database` values, in the order given on the command line. +declare -ag DATABASE_SPECS=() + +# Options parsed from the spec currently being expanded, e.g. DB_CONFIG[SETUP_TYPE]. +# parse_database_spec() clears and repopulates this for every spec. +declare -Ag DB_CONFIG=() + +# --- global switch defaults ------------------------------------------------- +# Do not name any of these after a registered option key: resolve_value() looks +# up shell variables by name and would pick one of these up by accident. +PMM_SERVER_IP_ARG='' # --pmm-server-ip; empty means discover via Docker +PMM_SERVER_PASSWORD='' # --pmm-server-password; ADMIN_PASSWORD env wins +GLOBAL_CLIENT_VERSION='' # --client-version; applies to every setup +VERBOSE=false # --verbose/--v +VERBOSITY_LEVEL=1 # --verbosity-level; becomes that many -v for ansible +CLIENT_DEBUG=false # --client-debug +PARALLEL=false # --parallel; preflight may turn this back off + +# Print the user-facing help text. Keep in sync with parse_args. +print_help() { + cat <<'EOF' +PMM Framework (Bash) + +Usage: + pmm-framework [options] --database SPEC [--database SPEC ...] + +Options: + --database SPEC Database setup, repeatable. + Example: ps=8.4,SETUP_TYPE=gr + --pmm-server-ip VALUE PMM Server address (otherwise Docker discovery). + --pmm-server-password VALUE PMM Server admin password (default: admin). + --client-version VALUE Global PMM Client version/tarball override. + --verbose, --v Print resolved setup details, and with --parallel + also echo the logs of successful setups. + --verbosity-level N Ansible verbosity level (numeric, default: 1). + --client-debug Enable PMM Client debug mode. + --parallel Run setups concurrently; dump logs only on failure. + -h, --help Show this help. + +Database SPEC: + NAME[=VERSION][,OPTION=VALUE...] + +Examples: + pmm-framework --database ps=8.4 + pmm-framework --database ps=8.4,SETUP_TYPE=gr --database psmdb + pmm-framework --pmm-server-ip 10.0.0.5 --database valkey=8 +EOF +} + +# Parse argv into the global switches and DATABASE_SPECS. +# +# Both `--flag value` and `--flag=value` spellings are accepted. Value-taking +# flags treat a following argument that starts with '-' as "no value given" +# rather than swallowing the next flag, which keeps `--client-version +# --database ps` from silently eating the --database. +# +# Writes: DATABASE_SPECS and every global switch above +# Exits: 0 via --help; via die() on an unknown flag, a non-numeric +# --verbosity-level, or when no --database was supplied +parse_args() { + DATABASE_SPECS=() + while (($#)); do + local arg=$1 inline='' consumed=1 has_inline=false + # Split the --flag=value spelling before matching. + if [[ $arg == --*=* ]]; then + inline=${arg#*=} + arg=${arg%%=*} + has_inline=true + fi + case "$arg" in + --database) + if [[ $has_inline == true ]]; then + DATABASE_SPECS+=("$inline") + else + (($# >= 2)) || die "--database requires a value." + DATABASE_SPECS+=("$2") + consumed=2 + fi + ;; + # Flags that take a value. Collected here so the "next arg looks like a + # flag" rule lives in exactly one place, then dispatched below. + --pmm-server-ip|--pmm-server-password|--client-version|--verbosity-level) + local value + if [[ $has_inline == true ]]; then + value=$inline + elif (($# >= 2)) && [[ ${2:0:1} != '-' ]]; then + value=$2 + consumed=2 + else + value='' + fi + case "$arg" in + --pmm-server-ip) PMM_SERVER_IP_ARG=$value ;; + --pmm-server-password) PMM_SERVER_PASSWORD=$value ;; + --client-version) GLOBAL_CLIENT_VERSION=$value ;; + --verbosity-level) + # A bare --verbosity-level keeps the default rather than blanking it. + if [[ $has_inline == true || -n $value ]]; then + VERBOSITY_LEVEL=$value + fi + ;; + esac + ;; + --verbose|--v) VERBOSE=true ;; + --client-debug) CLIENT_DEBUG=true ;; + --parallel) PARALLEL=true ;; + -h|--help) + print_help + exit 0 + ;; + *) die "Unknown option '$1'. Run with --help for supported options." ;; + esac + shift "$consumed" + done + + [[ $VERBOSITY_LEVEL =~ ^[0-9]+$ ]] || + die "Invalid verbosity level '$VERBOSITY_LEVEL'; provide a number." + ((${#DATABASE_SPECS[@]} > 0)) || + die "At least one --database SPEC is required." +} + +# Expand one spec string into DB_TYPE, DB_VERSION and DB_CONFIG. +# +# Grammar: NAME[=VERSION][,OPTION=VALUE...] +# ps=8.4,SETUP_TYPE=gr,QUERY_SOURCE=slowlog +# +# Names and option keys are case-insensitive and upper-cased here, so the +# catalogue in lib/config.sh only ever deals in uppercase. +# +# Unknown *versions* and unknown *options* are not fatal: they are noted under +# --verbose and the registered default is used instead. This matches the Python +# framework, where a typo degrades to a default rather than failing a long CI +# job. An unknown database *name* is fatal, since there is nothing to fall back +# to. +# +# Writes: DB_TYPE, DB_VERSION, DB_CONFIG (all reset on entry) +# Exits: via die() on an empty spec or unknown database name +parse_database_spec() { + local spec=$1 + DB_TYPE='' + DB_VERSION='' + DB_CONFIG=() + + local -a tokens=() + IFS=',' read -r -a tokens <<< "$spec" + ((${#tokens[@]} > 0)) || die "Empty --database specification." + + # First token is NAME or NAME=VERSION. + local first=${tokens[0]} + local type_token=${first%%=*} + DB_TYPE=${type_token^^} + database_exists "$DB_TYPE" || + die "Database type '$type_token' is not recognized." + + if [[ $first == *=* ]]; then + local candidate=${first#*=} + if database_version_exists "$DB_TYPE" "$candidate"; then + DB_VERSION=$candidate + else + log_verbose "Value '$candidate' is not recognized for $DB_TYPE; using its default version." + fi + fi + + # Remaining tokens are OPTION=VALUE pairs. + local token key value + for token in "${tokens[@]:1}"; do + if [[ $token != *=* ]]; then + log_verbose "Option '$token' is not recognized for $DB_TYPE; using defaults." + continue + fi + key=${token%%=*} + key=${key^^} + value=${token#*=} + if database_option_exists "$DB_TYPE" "$key"; then + DB_CONFIG["$key"]=$value + else + log_verbose "Option '$key' is not recognized for $DB_TYPE; using defaults." + fi + done +} diff --git a/qa-integration/pmm_qa/pmm-framework/lib/common.sh b/qa-integration/pmm_qa/pmm-framework/lib/common.sh new file mode 100644 index 000000000..c35316044 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/common.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# lib/common.sh -- logging, fatal errors and small value helpers. +# +# The lowest layer of the framework: it depends on nothing else and is sourced +# first, so every other module may call these helpers. Keep it free of Docker, +# Ansible and database knowledge. +# +# Convention used throughout the framework: helpers that produce a *value* +# print it to stdout with no trailing newline, so callers can capture it with +# `value=$(helper ...)`. Helpers that report a *fact* return 0 or 1 instead. + +# Print a line to stdout. Always shown, regardless of --verbose. +# Usage: log_info "PS setup finished" +log_info() { + printf '%s\n' "$*" +} + +# Print a line to stdout only when --verbose was passed. +# Use for detail that helps when debugging a setup but is noise otherwise. +# Reads: VERBOSE +# Usage: log_verbose "Using interpreter: $path" +log_verbose() { + if [[ ${VERBOSE:-false} == true ]]; then + printf '%s\n' "$*" + fi +} + +# Print a WARNING line to stderr and carry on. +# Use when the run can still succeed but the user should know something was +# assumed or degraded (for example falling back to sequential setups). +# Usage: log_warn "Found 2 PMM Server containers; using the first" +log_warn() { + printf 'WARNING: %s\n' "$*" >&2 +} + +# Print an ERROR line to stderr and terminate. +# +# Note this exits the *current shell*: at top level it aborts the whole run, +# but inside `$(...)` or a background parallel setup it only ends that +# subshell. The framework relies on `set -e` (plus `inherit_errexit`) to +# propagate that failure outward -- see the entrypoint. +# Usage: die "Database type '$name' is not recognized." +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +# Abort unless an executable is on PATH. +# Usage: require_command docker +require_command() { + command -v "$1" >/dev/null 2>&1 || die "Required command '$1' was not found in PATH." +} + +# Normalize a loosely-typed truthy value to exactly 'true' or 'false'. +# +# Ansible playbooks read these as `| bool`, which accepts both spellings, but +# emitting one canonical form keeps --verbose output and tests predictable. +# Anything unrecognized (including empty) becomes 'false'. +# Stdout: 'true' or 'false' +# Usage: env_map[CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" +bool_string() { + case "${1:-false}" in + true|TRUE|True|1|yes|YES|Yes) printf 'true' ;; + *) printf 'false' ;; + esac +} + +# Expand the 'latest-tarball' alias into the PMM Client build-cache URL. +# +# Any other value -- a version like '3-dev-latest', or an explicit URL -- is +# passed through untouched, so callers can always pipe CLIENT_VERSION through +# this before handing it to a playbook. +# Stdout: the resolved client version or URL +# Usage: client=$(normalize_client_version "$raw") +normalize_client_version() { + if [[ ${1:-} == latest-tarball ]]; then + printf '%s' 'https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client/pmm-client-latest.tar.gz' + else + printf '%s' "${1:-}" + fi +} diff --git a/qa-integration/pmm_qa/pmm-framework/lib/config.sh b/qa-integration/pmm_qa/pmm-framework/lib/config.sh new file mode 100644 index 000000000..10f901c51 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/config.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# +# lib/config.sh -- the database catalogue and value resolution. +# +# Everything the framework knows about a product declaratively lives here: the +# versions it accepts, which spec options are valid, and the default for each. +# Nothing in this file runs a setup; it only answers questions about them. +# +# TO ADD A NEW DATABASE TYPE you normally touch three files: +# 1. this one -- register_database ... (validation + defaults) +# 2. setups/.sh -- a setup_ function that builds the env map +# 3. setups/dispatch.sh -- one case arm pointing DB_TYPE at that function +# See ARCHITECTURE.md for the walkthrough. +# +# Data model (four parallel associative arrays, all keyed by uppercase type): +# DB_VERSIONS[TYPE] space-separated list of accepted versions +# DB_OPTIONS[TYPE] space-separated list of accepted option keys +# DB_DEFAULTS[TYPE:KEY] default value for one option +# DB_DEFAULT_VERSIONS[TYPE] version used when the spec omits one + +declare -Ag DB_VERSIONS=() +declare -Ag DB_OPTIONS=() +declare -Ag DB_DEFAULTS=() +declare -Ag DB_DEFAULT_VERSIONS=() + +# Declare one database type. +# +# Usage: register_database TYPE 'versions...' 'OPTION_KEYS...' 'KEY=default'... +# +# TYPE uppercase identifier, matched case-insensitively on the CLI +# versions accepted versions, space separated; '' for versionless types +# OPTION_KEYS accepted `KEY=value` option names inside a --database spec +# KEY=default one pair per option; unlisted options resolve to '' +# +# DEFAULT_VERSION is not a user-settable option. It is pulled out of the +# defaults list into DB_DEFAULT_VERSIONS and names the version used when the +# spec omits one, so the *order* of the version list carries no meaning. The +# guard at the bottom of this file rejects a versioned type that forgets it. +# +# Writes: DB_VERSIONS, DB_OPTIONS, DB_DEFAULTS, DB_DEFAULT_VERSIONS +register_database() { + local type=$1 versions=$2 options=$3 + shift 3 + DB_VERSIONS["$type"]=$versions + DB_OPTIONS["$type"]=$options + local pair key + for pair in "$@"; do + key=${pair%%=*} + if [[ $key == DEFAULT_VERSION ]]; then + DB_DEFAULT_VERSIONS["$type"]=${pair#*=} + else + DB_DEFAULTS["$type:$key"]=${pair#*=} + fi + done +} + +# -------------------------------------------------------------------------- +# The catalogue. Keep each entry's option list and defaults in sync: an option +# without a default silently resolves to the empty string. +# -------------------------------------------------------------------------- + +register_database PSMDB \ + '4.4 5.0 6.0 7.0 8.0 latest' \ + 'CLIENT_VERSION SETUP_TYPE COMPOSE_PROFILES TARBALL OL_VERSION GSSAPI STORAGE_ENGINE' \ + 'DEFAULT_VERSION=latest' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=pss' 'COMPOSE_PROFILES=classic' \ + 'TARBALL=' 'OL_VERSION=9' 'GSSAPI=false' 'STORAGE_ENGINE=wiredTiger' + +register_database MLAUNCH_PSMDB \ + '4.4 5.0 6.0 7.0 8.0' \ + 'CLIENT_VERSION SETUP_TYPE TARBALL' \ + 'DEFAULT_VERSION=8.0' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=pss' 'TARBALL=' + +register_database MLAUNCH_MODB \ + '4.4 5.0 6.0 7.0 8.0' \ + 'CLIENT_VERSION SETUP_TYPE TARBALL' \ + 'DEFAULT_VERSION=8.0' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=pss' 'TARBALL=' + +register_database SSL_MLAUNCH \ + '4.4 5.0 6.0 7.0 8.0' \ + 'CLIENT_VERSION SETUP_TYPE COMPOSE_PROFILES TARBALL' \ + 'DEFAULT_VERSION=8.0' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=pss' 'COMPOSE_PROFILES=classic' 'TARBALL=' + +register_database SSL_PSMDB \ + '4.4 5.0 6.0 7.0 8.0 latest' \ + 'CLIENT_VERSION SETUP_TYPE COMPOSE_PROFILES TARBALL' \ + 'DEFAULT_VERSION=latest' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=pss' 'COMPOSE_PROFILES=classic' 'TARBALL=' + +register_database MYSQL \ + '5.7 8.0 8.4 9.7' \ + 'QUERY_SOURCE SETUP_TYPE CLIENT_VERSION TARBALL ENCRYPTED_CLIENT_CONFIG' \ + 'DEFAULT_VERSION=9.7' \ + 'QUERY_SOURCE=perfschema' 'SETUP_TYPE=' 'CLIENT_VERSION=3-dev-latest' \ + 'TARBALL=' 'ENCRYPTED_CLIENT_CONFIG=false' + +register_database PS \ + '5.7 8.0 8.4' \ + 'QUERY_SOURCE SETUP_TYPE CLIENT_VERSION TARBALL NODES_COUNT MY_ROCKS ENCRYPTED_CLIENT_CONFIG BACKUP' \ + 'DEFAULT_VERSION=8.0' \ + 'QUERY_SOURCE=perfschema' 'SETUP_TYPE=' 'CLIENT_VERSION=3-dev-latest' \ + 'TARBALL=' 'NODES_COUNT=1' 'MY_ROCKS=false' 'ENCRYPTED_CLIENT_CONFIG=false' 'BACKUP=false' + +register_database SSL_MYSQL \ + '5.7 8.0 8.4' \ + 'QUERY_SOURCE SETUP_TYPE CLIENT_VERSION TARBALL' \ + 'DEFAULT_VERSION=8.0' \ + 'QUERY_SOURCE=perfschema' 'SETUP_TYPE=' 'CLIENT_VERSION=3-dev-latest' 'TARBALL=' + +register_database PGSQL \ + '11 12 13 14 15 16 17 18' \ + 'QUERY_SOURCE CLIENT_VERSION USE_SOCKET SETUP_TYPE ENCRYPTED_CLIENT_CONFIG' \ + 'DEFAULT_VERSION=17' \ + 'QUERY_SOURCE=pgstatements' 'CLIENT_VERSION=3-dev-latest' 'USE_SOCKET=' \ + 'SETUP_TYPE=' 'ENCRYPTED_CLIENT_CONFIG=false' + +register_database PDPGSQL \ + '11 12 13 14 15 16 17 18' \ + 'CLIENT_VERSION USE_SOCKET SETUP_TYPE PGSM_BRANCH ENCRYPTED_CLIENT_CONFIG' \ + 'DEFAULT_VERSION=17' \ + 'CLIENT_VERSION=3-dev-latest' 'USE_SOCKET=' 'SETUP_TYPE=' 'PGSM_BRANCH=' \ + 'ENCRYPTED_CLIENT_CONFIG=false' + +register_database SSL_PDPGSQL \ + '11 12 13 14 15 16 17' \ + 'CLIENT_VERSION USE_SOCKET' \ + 'DEFAULT_VERSION=17' \ + 'CLIENT_VERSION=3-dev-latest' 'USE_SOCKET=' + +register_database PXC \ + '5.7 8.0' \ + 'CLIENT_VERSION QUERY_SOURCE TARBALL' \ + 'DEFAULT_VERSION=8.0' \ + 'CLIENT_VERSION=3-dev-latest' 'QUERY_SOURCE=perfschema' 'TARBALL=' + +# PROXYSQL is not independently setup-able: it only supplies defaults that the +# PXC setup reads (see setups/mysql.sh). dispatch_setup rejects it explicitly. +register_database PROXYSQL '2' 'PACKAGE' 'DEFAULT_VERSION=2' 'PACKAGE=' + +# Versionless types: '' means "no version accepted", so `--database haproxy=1` +# logs a note under --verbose and falls back to the (empty) default. +register_database HAPROXY '' 'CLIENT_VERSION' 'CLIENT_VERSION=3-dev-latest' +register_database EXTERNAL '' 'CLIENT_VERSION' 'CLIENT_VERSION=3-dev-latest' +register_database DOCKERCLIENTS '' '' +register_database BUCKET '' 'BUCKET_NAMES' 'BUCKET_NAMES=bcp' + +register_database VALKEY \ + '7 8' \ + 'CLIENT_VERSION SETUP_TYPE TARBALL ENCRYPTED_CLIENT_CONFIG' \ + 'DEFAULT_VERSION=8' \ + 'CLIENT_VERSION=3-dev-latest' 'SETUP_TYPE=' 'TARBALL=' 'ENCRYPTED_CLIENT_CONFIG=false' + +# -------------------------------------------------------------------------- +# Catalogue queries. All take an already-uppercased TYPE. +# -------------------------------------------------------------------------- + +# Is TYPE a registered database? +# Returns: 0 if registered, 1 otherwise +database_exists() { + [[ -v "DB_OPTIONS[$1]" ]] +} + +# Does TYPE accept the spec option KEY? +# Usage: database_option_exists PS SETUP_TYPE +# Returns: 0 if accepted, 1 otherwise +database_option_exists() { + local type=$1 key=$2 option + for option in ${DB_OPTIONS[$type]-}; do + [[ $option == "$key" ]] && return 0 + done + return 1 +} + +# Does TYPE offer the given version? +# Usage: database_version_exists PGSQL 17 +# Returns: 0 if offered, 1 otherwise (including versionless types) +database_version_exists() { + local type=$1 wanted=$2 version + [[ -z ${DB_VERSIONS[$type]-} ]] && return 1 + for version in ${DB_VERSIONS[$type]}; do + [[ $version == "$wanted" ]] && return 0 + done + return 1 +} + +# The version used when a spec omits one. +# Stdout: the registered DEFAULT_VERSION, or '' for versionless types +database_default_version() { + printf '%s' "${DB_DEFAULT_VERSIONS[$1]-}" +} + +# The registered default for one option. +# Usage: database_default_value PS QUERY_SOURCE # -> perfschema +# Stdout: the default, or '' when the option has none +database_default_value() { + printf '%s' "${DB_DEFAULTS["$1:$2"]-}" +} + +# Resolve one option value using the framework's precedence rules. +# +# Usage: resolve_value TYPE KEY CONFIG_ARRAY_NAME +# e.g. setup_type=$(resolve_value PS SETUP_TYPE DB_CONFIG) +# +# Precedence, highest first: +# 1. an existing shell/environment variable named KEY +# 2. the global --client-version, for KEY == CLIENT_VERSION only +# 3. the per-database option parsed from the --database spec +# 4. the default registered above +# +# Step 1 mirrors the Python framework's `os.environ.get(KEY)`, so an exported +# but *empty* variable deliberately wins and yields ''. Contrast with +# resolved_version() in lib/runners.sh, which mirrors `os.getenv(...) or ...` +# and therefore skips empty values -- the two rules are intentionally +# different. Because `-v` also sees non-exported shell variables, avoid naming +# any global in lib/cli.sh after a registered option key. +# +# Reads: the named config array (normally DB_CONFIG), GLOBAL_CLIENT_VERSION +# Stdout: the resolved value (possibly empty) +resolve_value() { + local type=$1 key=$2 config_name=$3 + local -n config_ref=$config_name + if [[ -v $key ]]; then + printf '%s' "${!key}" + elif [[ $key == CLIENT_VERSION && -n ${GLOBAL_CLIENT_VERSION:-} ]]; then + printf '%s' "$GLOBAL_CLIENT_VERSION" + elif [[ -v "config_ref[$key]" ]]; then + printf '%s' "${config_ref[$key]}" + else + database_default_value "$type" "$key" + fi +} + +# Catch a registration that adds versions but forgets DEFAULT_VERSION, or names +# one that is not offered, at source time rather than mid-setup. +for _db_type in "${!DB_VERSIONS[@]}"; do + [[ -n ${DB_VERSIONS[$_db_type]} ]] || continue + database_version_exists "$_db_type" "${DB_DEFAULT_VERSIONS[$_db_type]-}" || + die "register_database $_db_type: DEFAULT_VERSION must be one of:" \ + "${DB_VERSIONS[$_db_type]}" +done +unset _db_type diff --git a/qa-integration/pmm_qa/pmm-framework/lib/docker.sh b/qa-integration/pmm_qa/pmm-framework/lib/docker.sh new file mode 100644 index 000000000..f1ea2e9d9 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/docker.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# lib/docker.sh -- locating the PMM Server the setups should report to. +# +# Every setup needs an address to point pmm-agent at. It comes from one of two +# places, and the choice also decides the port: +# +# --pmm-server-ip 10.0.0.5 an external/remote server, reached on 443 +# (omitted) a local container on the pmm-qa Docker network, +# reached by container name on 8443 +# +# Both are published as PMM_SERVER_HOST / PMM_SERVER_PORT for the setup +# functions to drop into their env maps. + +# Name of the discovered PMM Server container; empty until discovery runs. +PMM_SERVER_CONTAINER='' + +# Find a running PMM Server container and attach it to the pmm-qa network. +# +# Matches any running container whose *image* contains 'pmm-server'. When +# several match, the first is used and a warning names them all -- picking +# silently would let a run monitor the wrong server, which is confusing to +# debug. Pass --pmm-server-ip to be explicit. +# +# Creates the pmm-qa network if missing and connects the server to it, so the +# database containers the setups create can reach it by name. +# +# Writes: PMM_SERVER_CONTAINER +# Returns: 0 when a server was found, 1 when none is running +discover_pmm_server() { + local image name + local -a candidates=() + while IFS=$'\t' read -r image name; do + if [[ $image == *pmm-server* ]]; then + candidates+=("$name") + fi + done < <(docker ps --format '{{.Image}}{{"\t"}}{{.Names}}') + + ((${#candidates[@]} > 0)) || return 1 + PMM_SERVER_CONTAINER=${candidates[0]} + if ((${#candidates[@]} > 1)); then + log_warn "Found ${#candidates[@]} PMM Server containers (${candidates[*]});" \ + "using '$PMM_SERVER_CONTAINER'. Pass --pmm-server-ip to select one explicitly." + fi + + if ! docker network inspect pmm-qa >/dev/null 2>&1; then + docker network create pmm-qa >/dev/null + fi + + if ! docker network inspect pmm-qa \ + --format '{{range .Containers}}{{.Name}}{{"\n"}}{{end}}' | + grep -Fxq "$PMM_SERVER_CONTAINER"; then + docker network connect pmm-qa "$PMM_SERVER_CONTAINER" + fi +} + +# Decide the PMM Server address for this run. +# +# Called once from preflight, and only when at least one requested setup +# actually needs a server (BUCKET and DOCKERCLIENTS do not). +# +# Reads: PMM_SERVER_IP_ARG +# Writes: PMM_SERVER_HOST, PMM_SERVER_PORT +# Exits: via die() when no server is running and no address was given +resolve_pmm_server() { + if [[ -n ${PMM_SERVER_IP_ARG:-} ]]; then + PMM_SERVER_HOST=$PMM_SERVER_IP_ARG + PMM_SERVER_PORT=443 + return + fi + + discover_pmm_server || + die "PMM Server is not running and --pmm-server-ip was not provided." + PMM_SERVER_HOST=$PMM_SERVER_CONTAINER + PMM_SERVER_PORT=8443 +} diff --git a/qa-integration/pmm_qa/pmm-framework/lib/execution.sh b/qa-integration/pmm_qa/pmm-framework/lib/execution.sh new file mode 100644 index 000000000..19d960e7d --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/execution.sh @@ -0,0 +1,309 @@ +#!/usr/bin/env bash +# +# lib/execution.sh -- orchestration: preflight checks, then run the setups. +# +# This is the top of the call graph. run_database_setups() is the only function +# the entrypoint calls, and everything below is either preflight or one of the +# two execution strategies: +# +# sequential each spec in argument order, output streaming straight to the +# console. The default, and where a failure stops the run. +# parallel every spec at once, each with its output buffered to its own +# log file so concurrent runs cannot interleave. All setups are +# allowed to finish and the run fails if any of them did. +# +# Preflight may downgrade parallel to sequential -- see the conflict rules in +# preflight_database_setups(). + +# Does this database type provision through Ansible rather than a shell script? +# +# Used only to decide whether preflight needs to warm up the Ansible bits +# before parallel setups start. The actual choice of backend lives in each +# setup function, which calls run_playbook() or run_setup_script() directly. +# +# Returns: 0 for playbook-backed types, 1 for script-backed ones +setup_uses_ansible() { + case "$1" in + PSMDB|SSL_PSMDB|DOCKERCLIENTS) return 1 ;; + *) return 0 ;; + esac +} + +# Validate the whole run before provisioning anything. +# +# Walks every spec once to answer three questions, so that a bad request fails +# in seconds rather than halfway through a ten-minute provisioning run: +# +# * does anything need a PMM Server address, curl, or Ansible? +# * do any two setups conflict, so parallel is unsafe? +# * are the shared Ansible prerequisites ready before jobs fork? +# +# Conflict rule: two setups of the same type, any two of the MySQL family +# (PS/MYSQL), or PDPGSQL with a PGSQL setup that uses replication, reuse the +# same container names, host ports and/or data directories. Rather than +# refusing the run, the framework keeps every setup and gives up only the +# concurrency -- the caller asked for something valid that merely cannot +# happen at the same time. +# +# The PDPGSQL/PGSQL rule is narrower than the MySQL one: only PGSQL's +# replication playbook (postgresql/postgresql-setup.yml) shares PDPGSQL's +# fixed $HOME/pgsql_cluster_data and port 6432 -- PGSQL's default, +# non-replication path is fully containerized and does not conflict. +# +# Reads: DATABASE_SPECS, PARALLEL +# Writes: PARALLEL (may be turned off), PMM_SERVER_HOST/PORT via resolve_pmm_server +# Exits: via die() from the helpers it calls (unknown type, missing server, ...) +preflight_database_setups() { + local spec needs_server=false needs_curl=false needs_ansible=false + local mysql_data_owner='' conflict='' + local pdpgsql_seen=false pgsql_replication_seen=false + declare -A seen_types=() + + for spec in "${DATABASE_SPECS[@]}"; do + parse_database_spec "$spec" + setup_requires_server "$DB_TYPE" && needs_server=true + [[ $DB_TYPE == PSMDB || $DB_TYPE == SSL_PSMDB ]] && needs_curl=true + setup_uses_ansible "$DB_TYPE" && needs_ansible=true + + # Two setups of the same product, or any two of the MySQL family, reuse the + # same container names, host ports and data directories, so they cannot run + # at the same time. + if [[ -v "seen_types[$DB_TYPE]" ]]; then + conflict="two $DB_TYPE setups" + elif [[ $DB_TYPE == PS || $DB_TYPE == MYSQL ]]; then + if [[ -n $mysql_data_owner ]]; then + conflict="$mysql_data_owner and $DB_TYPE setups (shared mysql_cluster_data and host ports)" + fi + mysql_data_owner=$DB_TYPE + elif [[ $DB_TYPE == PDPGSQL ]]; then + pdpgsql_seen=true + [[ $pgsql_replication_seen == true ]] && + conflict="PGSQL (replication) and PDPGSQL setups (shared pgsql_cluster_data and host port 6432)" + elif [[ $DB_TYPE == PGSQL ]]; then + local pgsql_setup_type + pgsql_setup_type=$(resolve_value PGSQL SETUP_TYPE DB_CONFIG) + if [[ ${pgsql_setup_type,,} == replication ]]; then + pgsql_replication_seen=true + [[ $pdpgsql_seen == true ]] && + conflict="PDPGSQL and PGSQL (replication) setups (shared pgsql_cluster_data and host port 6432)" + fi + fi + seen_types["$DB_TYPE"]=1 + done + + # Fall back to sequential rather than refusing to run: the caller asked for a + # valid set of setups, they just cannot be provisioned concurrently. + if [[ $PARALLEL == true && -n $conflict ]]; then + log_warn "Running setups sequentially: $conflict cannot run in parallel." + PARALLEL=false + fi + + [[ $needs_server == true ]] && resolve_pmm_server + [[ $needs_curl == true ]] && require_command curl + # Warm these up before forking so parallel jobs cannot race to install the + # same Ansible collection. + if [[ $PARALLEL == true && $needs_ansible == true ]]; then + configure_ansible_python + ensure_docker_collection + fi +} + +# Expand one spec and provision it. +# +# The single unit of work, shared by both strategies: sequential calls it in a +# loop, parallel calls it once per background job. Re-parsing here (preflight +# already parsed every spec) is deliberate -- it guarantees DB_TYPE, DB_VERSION +# and DB_CONFIG describe *this* spec and nothing has leaked from the previous +# one. +# +# Reads: VERBOSE +# Writes: DB_TYPE, DB_VERSION, DB_CONFIG via parse_database_spec +# Exits: via die() when the setup fails +run_database_spec() { + local spec=$1 + parse_database_spec "$spec" + + if [[ $VERBOSE == true ]]; then + if [[ -n $DB_VERSION ]]; then + log_info "Setting up $DB_TYPE version $DB_VERSION" + else + log_info "Setting up $DB_TYPE" + fi + fi + dispatch_setup +} + +# Should a *successful* setup echo its buffered log to the console? +# +# No by default: a green run prints one summary line per setup and keeps the +# full transcript on disk, so CI logs stay readable. `--verbose` opts in and +# echoes them too, for when you want to see what a passing setup actually did. +# +# A setup that FAILED always dumps its log regardless -- that is not optional. +# +# Reads: VERBOSE +# Returns: 0 to echo successful logs, 1 to keep them on disk only +should_dump_successful_logs() { + [[ ${VERBOSE:-false} == true ]] +} + +# Report one finished parallel setup. +# +# Usage: print_setup_log INDEX TOTAL SPEC STATUS LOG_FILE +# +# A failed setup always dumps its buffered log; a successful one prints just a +# summary line unless --verbose asked for more. Only ever used by the parallel +# path -- sequential setups write straight to the console and need no +# buffering. Because a log is emitted in one go, concurrent setups never +# interleave mid-line. +# +# Stdout: a summary line, plus the buffered log when the setup failed or when +# --verbose was given +print_setup_log() { + local index=$1 total=$2 spec=$3 status=$4 log_file=$5 + + if ((status == 0)); then + printf '[%d/%d] %s: OK (log: %s)\n' "$index" "$total" "$spec" "$log_file" + if should_dump_successful_logs; then + printf '\n===== [%d/%d] %s setup log =====\n' "$index" "$total" "$spec" + cat "$log_file" + if [[ -s $log_file ]] && (($(tail -c 1 "$log_file" | wc -l) == 0)); then + printf '\n' + fi + printf '===== END [%d/%d] %s =====\n' "$index" "$total" "$spec" + fi + return + fi + + printf '\n===== [%d/%d] %s FAILED (exit=%d) =====\n' \ + "$index" "$total" "$spec" "$status" + printf 'log: %s\n' "$log_file" + cat "$log_file" + # Keep the END marker on its own line when the log has no trailing newline. + if [[ -s $log_file ]] && (($(tail -c 1 "$log_file" | wc -l) == 0)); then + printf '\n' + fi + printf '===== END [%d/%d] %s =====\n' "$index" "$total" "$spec" +} + +# Provision every spec concurrently, reporting each as it finishes. +# +# Each setup runs in its own background subshell with stdout and stderr +# redirected to a per-setup log file, so output never interleaves. Results are +# printed in completion order, not argument order, so a fast setup is visible +# immediately instead of appearing stuck behind a slow neighbour. +# +# Every setup is allowed to finish even after one fails, because tearing down +# half-provisioned containers mid-run leaves more mess than it saves. +# +# On success the log directory is removed; on failure it is kept and its path +# printed, so the full transcripts survive for inspection. +# +# Requires: bash 5.1+ for `wait -n -p` +# Reads: DATABASE_SPECS +# Returns: 0 when every setup succeeded, 1 when any failed +# Exits: 130 from the INT/TERM trap +run_parallel_setups() { + local log_dir total index spec status overall_status=0 + local -a pids=() logs=() + log_dir=$(mktemp -d "${TMPDIR:-/tmp}/pmm-framework-parallel.XXXXXX") + total=${#DATABASE_SPECS[@]} + + # Job control puts each background setup in its own process group, so an + # interrupt can take down ansible-playbook and its children too. Without it + # `kill $pid` would only reap the wrapper subshell and leave the real + # provisioning work running. + set -m + + # shellcheck disable=SC2329 # Invoked by the INT/TERM trap. + cleanup_parallel_jobs() { + local pid + for pid in "${pids[@]}"; do + # Negative PID targets the whole process group; fall back to the single + # process if the group is already gone. + kill -- -"$pid" >/dev/null 2>&1 || kill "$pid" >/dev/null 2>&1 || true + done + wait >/dev/null 2>&1 || true + rm -rf "$log_dir" + exit 130 + } + trap cleanup_parallel_jobs INT TERM + + for ((index = 0; index < total; index++)); do + spec=${DATABASE_SPECS[index]} + logs[index]=$log_dir/setup-$index.log + printf 'Starting [%d/%d] %s\n' "$((index + 1))" "$total" "$spec" + # stdin must come from /dev/null: job control puts each setup in a + # background process group, where reading the terminal raises SIGTTIN and + # stops the job forever. Parallel setups have no usable stdin anyway. + ( + run_database_spec "$spec" + ) >"${logs[index]}" 2>&1 0)); do + status=0 + finished_pid= + wait -n -p finished_pid "${active_pids[@]}" || status=$? + [[ -n $finished_pid ]] || die "Parallel wait lost track of setup processes." + + # Map the reaped pid back to its slot so the report names the right spec. + matched=false + for ((index = 0; index < total; index++)); do + if [[ ${pids[index]} == "$finished_pid" ]]; then + ((status == 0)) || overall_status=1 + print_setup_log \ + "$((index + 1))" "$total" "${DATABASE_SPECS[index]}" \ + "$status" "${logs[index]}" + pids[index]= + matched=true + break + fi + done + [[ $matched == true ]] || die "Parallel wait reaped unknown pid $finished_pid." + + # Rebuild the still-running set; cleared slots drop out. + active_pids=() + for ((index = 0; index < total; index++)); do + [[ -n ${pids[index]} ]] && active_pids+=("${pids[index]}") + done + done + + trap - INT TERM + set +m + if ((overall_status == 0)); then + rm -rf "$log_dir" + else + printf '\nParallel setup logs kept at: %s\n' "$log_dir" + fi + return "$overall_status" +} + +# Entry point for provisioning: preflight, then run every requested setup. +# +# The only function the entrypoint calls. Preflight decides which strategy is +# safe, so the PARALLEL check below happens *after* any downgrade. +# +# Unlike the parallel path, a sequential run stops at the first failure: die() +# in a setup aborts the whole script. +# +# Reads: DATABASE_SPECS, PARALLEL +# Returns: 0 when every setup succeeded, non-zero otherwise +run_database_setups() { + preflight_database_setups + if [[ $PARALLEL == true ]]; then + run_parallel_setups + return + fi + + local spec + for spec in "${DATABASE_SPECS[@]}"; do + run_database_spec "$spec" + done +} diff --git a/qa-integration/pmm_qa/pmm-framework/lib/runners.sh b/qa-integration/pmm_qa/pmm-framework/lib/runners.sh new file mode 100644 index 000000000..04bf3c253 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/lib/runners.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# lib/runners.sh -- the shell-script backend, plus value helpers used by setups. +# +# Two unrelated groups of helpers share this file: +# +# * run_setup_script() -- the alternative to run_playbook() for the handful of +# products provisioned by a shell script instead of a playbook (PSMDB, +# SSL_PSMDB, DOCKERCLIENTS). See setup_uses_ansible() in lib/execution.sh. +# +# * the small resolvers every setup function calls when filling in its env +# map: resolved_version, resolved_client_version, admin_password, +# git_branch, and the PSMDB patch lookup. + +# Run a setup shell script with an env map, from a given directory. +# +# Usage: run_setup_script DIRECTORY SCRIPT MAP_NAME +# DIRECTORY working directory for the script (scripts use relative paths) +# SCRIPT script name inside DIRECTORY, or an absolute path +# MAP_NAME *name* of an associative array of KEY=value pairs +# +# The absolute-path form exists for setups that must rewrite a tracked script +# before running it -- setup_ssl_psmdb() generates a patched copy in a temp +# directory but still runs it with the original directory as cwd. +# +# Mirrors run_playbook(): variables go through `env`, never exported. +# +# Reads: VERBOSE +# Exits: via die() when the script is missing or exits non-zero +run_setup_script() { + local directory=$1 script=$2 map_name=$3 + local -n env_ref=$map_name + local -a env_args=() + local key + for key in "${!env_ref[@]}"; do + env_args+=("$key=${env_ref[$key]}") + done + + local script_path=$directory/$script + if [[ $script == /* ]]; then + script_path=$script + fi + [[ -f $script_path ]] || + die "Setup script '$script_path' does not exist." + log_verbose "Running setup script $directory/$script with:" + [[ $VERBOSE == true ]] && print_env_map "$map_name" + + ( + cd "$directory" + env "${env_args[@]}" bash "$script_path" + ) || die "Setup script '$script' failed." +} + +# Compare two dotted version strings numerically. +# +# Missing components count as 0, so 8.0 < 8.0.1. Components are forced to base +# 10 (10#) so a zero-padded part such as 08 is not read as octal. +# +# Usage: version_is_greater 12.1 8.1 +# Returns: 0 when LEFT > RIGHT, 1 when LEFT <= RIGHT +version_is_greater() { + local left=$1 right=$2 index left_part right_part + local -a left_parts=() right_parts=() + IFS='.' read -r -a left_parts <<<"$left" + IFS='.' read -r -a right_parts <<<"$right" + local length=${#left_parts[@]} + ((${#right_parts[@]} > length)) && length=${#right_parts[@]} + for ((index = 0; index < length; index++)); do + left_part=${left_parts[index]:-0} + right_part=${right_parts[index]:-0} + ((10#$left_part > 10#$right_part)) && return 0 + ((10#$left_part < 10#$right_part)) && return 1 + done + return 1 +} + +# Expand a PSMDB major version into its newest full version. +# +# The PSMDB setup scripts want a complete version such as '8.0-12.1', but specs +# name the major series ('8.0'). This asks the Percona products API which +# patches exist and picks the highest. +# +# 'latest' and '' pass through untouched -- they are meaningful to the setup +# script as-is. This is the only network call the framework makes, and the only +# reason `curl` is required (preflight checks for it only when a PSMDB setup is +# requested). +# +# Usage: version=$(latest_psmdb_version 8.0) # -> 8.0-12.1 +# Stdout: the resolved version +# Exits: via die() when the API call fails or no patch matches +latest_psmdb_version() { + local requested=$1 + if [[ $requested == latest || -z $requested ]]; then + printf '%s' "$requested" + return + fi + + local response latest='' latest_patch='' candidate patch + response=$(curl --fail --silent --show-error \ + --data-urlencode "version=percona-server-mongodb-$requested" \ + 'https://www.percona.com/products-api.php') || + die "Failed to query the Percona products API for PSMDB $requested." + + # The API answers with an HTML form: pull out the value="..." entries, strip + # the product prefix, and keep only entries for the requested series. + while IFS= read -r candidate; do + patch=${candidate#"$requested-"} + if [[ -z $latest ]] || version_is_greater "$patch" "$latest_patch"; then + latest=$candidate + latest_patch=$patch + fi + done < <( + printf '%s' "$response" | + grep -Eo 'value="[^"]+"' | + cut -d'"' -f2 | + cut -d'|' -f1 | + sed 's/^percona-server-mongodb-//' | + grep -E "^${requested//./\\.}-[0-9]+([.][0-9]+)*$" + ) + [[ -n $latest ]] || + die "Could not resolve the latest PSMDB patch for '$requested'." + printf '%s' "$latest" +} + +# The PMM Server admin password for this run. +# Precedence: ADMIN_PASSWORD env > --pmm-server-password > 'admin'. +# Stdout: the password +admin_password() { + printf '%s' "${ADMIN_PASSWORD:-${PMM_SERVER_PASSWORD:-admin}}" +} + +# The pmm-qa branch playbooks should use when fetching helper assets. +# Stdout: PMM_QA_GIT_BRANCH, or 'v3' +git_branch() { + printf '%s' "${PMM_QA_GIT_BRANCH:-v3}" +} + +# Resolve the version for a setup. +# +# Usage: resolved_version PS_VERSION PS "$DB_VERSION" +# ENV_NAME product-specific override variable, e.g. PS_VERSION +# TYPE registered database type, for the default lookup +# REQUESTED version from the spec, or '' when none was given +# +# Precedence: $ENV_NAME > spec version > registered DEFAULT_VERSION. +# +# Note this skips an *empty* $ENV_NAME, mirroring Python's `os.getenv(X) or ...`. +# resolve_value() in lib/config.sh deliberately does the opposite for options -- +# see the note there. +# +# Stdout: the resolved version +resolved_version() { + local env_name=$1 type=$2 requested=$3 + if [[ -n ${!env_name:-} ]]; then + printf '%s' "${!env_name}" + elif [[ -n $requested ]]; then + printf '%s' "$requested" + else + database_default_version "$type" + fi +} + +# Resolve CLIENT_VERSION for a setup and expand the latest-tarball alias. +# +# Usage: client=$(resolved_client_version PS DB_CONFIG) +# Stdout: a PMM Client version or tarball URL +resolved_client_version() { + local type=$1 config_name=$2 + normalize_client_version "$(resolve_value "$type" CLIENT_VERSION "$config_name")" +} diff --git a/qa-integration/pmm_qa/pmm-framework/pmm-framework b/qa-integration/pmm_qa/pmm-framework/pmm-framework new file mode 100755 index 000000000..e5fcc52ee --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/pmm-framework @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# pmm-framework -- provision databases and register them with a PMM Server. +# +# ./pmm-framework --database ps=8.4 --database psmdb,SETUP_TYPE=sharding +# ./pmm-framework --parallel --pmm-server-ip 10.0.0.5 --database valkey=8 +# ./pmm-framework --help +# +# This entrypoint is intentionally thin: it checks the Bash version, works out +# where the repo lives, sources every module, then hands over to parse_args() +# and run_database_setups(). All real behaviour lives in lib/ and setups/. +# +# Layout: +# lib/common.sh logging, die(), small value helpers +# lib/config.sh the database catalogue and value resolution +# lib/cli.sh argument parsing and the --database spec grammar +# lib/docker.sh finding the PMM Server +# lib/ansible.sh running playbooks +# lib/runners.sh running setup scripts, plus version/password helpers +# setups/*.sh one function per database type, and the dispatch table +# lib/execution.sh preflight and the sequential/parallel strategies +# +# See ARCHITECTURE.md for the call flow, a diagram, and how to add a database. + +set -euo pipefail + +# Bash 4.4 is the floor: expanding an empty array under `set -u` (setups with +# no environment variables, such as DOCKERCLIENTS) errors on 4.3 and older, +# and `inherit_errexit` below does not exist there either. +if ((BASH_VERSINFO[0] < 4 || (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 4))); then + printf 'ERROR: pmm-framework requires Bash 4.4 or newer (found %s).\n' \ + "$BASH_VERSION" >&2 + exit 1 +fi + +# Let command substitutions inherit `set -e`, so a failing $(...) aborts the +# run instead of silently yielding an empty value. +shopt -s inherit_errexit + +# Anchor every path to this script rather than the caller's cwd, so the +# framework can be invoked from anywhere: +# FRAMEWORK_DIR .../qa-integration/pmm_qa/pmm-framework +# PMM_QA_ROOT .../qa-integration/pmm_qa (playbooks live here) +# QA_INTEGRATION_ROOT .../qa-integration (setup scripts here) +FRAMEWORK_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PMM_QA_ROOT=$(cd "$FRAMEWORK_DIR/.." && pwd) +QA_INTEGRATION_ROOT=$(cd "$PMM_QA_ROOT/.." && pwd) +export FRAMEWORK_DIR PMM_QA_ROOT QA_INTEGRATION_ROOT + +# shellcheck source=lib/common.sh +source "$FRAMEWORK_DIR/lib/common.sh" +# shellcheck source=lib/config.sh +source "$FRAMEWORK_DIR/lib/config.sh" +# shellcheck source=lib/cli.sh +source "$FRAMEWORK_DIR/lib/cli.sh" +# shellcheck source=lib/docker.sh +source "$FRAMEWORK_DIR/lib/docker.sh" +# shellcheck source=lib/ansible.sh +source "$FRAMEWORK_DIR/lib/ansible.sh" +# shellcheck source=lib/runners.sh +source "$FRAMEWORK_DIR/lib/runners.sh" +# shellcheck source=setups/mysql.sh +source "$FRAMEWORK_DIR/setups/mysql.sh" +# shellcheck source=setups/postgresql.sh +source "$FRAMEWORK_DIR/setups/postgresql.sh" +# shellcheck source=setups/mongodb.sh +source "$FRAMEWORK_DIR/setups/mongodb.sh" +# shellcheck source=setups/services.sh +source "$FRAMEWORK_DIR/setups/services.sh" +# shellcheck source=setups/dispatch.sh +source "$FRAMEWORK_DIR/setups/dispatch.sh" +# shellcheck source=lib/execution.sh +source "$FRAMEWORK_DIR/lib/execution.sh" + +parse_args "$@" + +# Checked up front so a missing tool fails in milliseconds. curl is not here: +# only the PSMDB setups need it, so preflight checks for it conditionally. +require_command docker +require_command bash +require_command ansible-playbook +require_command ansible-galaxy + +run_database_setups diff --git a/qa-integration/pmm_qa/pmm-framework/setups/dispatch.sh b/qa-integration/pmm_qa/pmm-framework/setups/dispatch.sh new file mode 100644 index 000000000..a09bb6c12 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/setups/dispatch.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# setups/dispatch.sh -- the map from a database type to its setup function. +# +# Deliberately tiny: it is the one place to look to answer "what actually runs +# for --database X", and the one place to edit when adding a type. +# +# TO ADD A DATABASE TYPE, after registering it in lib/config.sh and writing its +# setup_ function in the matching setups/ file, add one case arm here. +# Also check setup_requires_server() below, and setup_uses_ansible() in +# lib/execution.sh, if the new type is unusual in either respect. + +# Does this type need a PMM Server to report to? +# +# Almost everything does. BUCKET only creates MinIO storage and DOCKERCLIENTS +# only builds images, so neither needs a server -- and preflight skips server +# discovery entirely when those are the only setups requested, which means +# `--database bucket` works with no PMM Server running at all. +# +# Returns: 0 when a server is required, 1 when not +setup_requires_server() { + case "$1" in + DOCKERCLIENTS|BUCKET) return 1 ;; + *) return 0 ;; + esac +} + +# Call the setup function for the parsed database type. +# +# Reads the DB_TYPE global rather than taking an argument, because it always +# runs directly after parse_database_spec() has populated DB_TYPE, DB_VERSION +# and DB_CONFIG together -- the setup functions read all three. +# +# Reads: DB_TYPE (and, through the setup functions, DB_VERSION and DB_CONFIG) +# Exits: via die() for PROXYSQL or an unmapped type +dispatch_setup() { + case "$DB_TYPE" in + PS) setup_ps ;; + MYSQL) setup_mysql ;; + SSL_MYSQL) setup_ssl_mysql ;; + PXC) setup_pxc ;; + PGSQL) setup_pgsql ;; + PDPGSQL) setup_pdpgsql ;; + SSL_PDPGSQL) setup_ssl_pdpgsql ;; + PSMDB) setup_psmdb ;; + SSL_PSMDB) setup_ssl_psmdb ;; + MLAUNCH_PSMDB) setup_mlaunch_psmdb ;; + MLAUNCH_MODB) setup_mlaunch_modb ;; + SSL_MLAUNCH) setup_ssl_mlaunch ;; + HAPROXY) setup_haproxy ;; + EXTERNAL) setup_external ;; + VALKEY) setup_valkey ;; + BUCKET) setup_bucket ;; + DOCKERCLIENTS) setup_dockerclients ;; + # Registered so PXC can read its defaults, but never runnable on its own. + PROXYSQL) + die "PROXYSQL is a PXC option source and cannot be set up independently." + ;; + # Reached only when a type is registered in lib/config.sh but has no arm + # here -- parse_database_spec would have rejected a genuinely unknown name. + *) die "Database type '$DB_TYPE' is not recognized." ;; + esac +} diff --git a/qa-integration/pmm_qa/pmm-framework/setups/mongodb.sh b/qa-integration/pmm_qa/pmm-framework/setups/mongodb.sh new file mode 100644 index 000000000..ef9f86337 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/setups/mongodb.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# +# setups/mongodb.sh -- MongoDB-family setups. +# +# Two different provisioning styles live here: +# +# PSMDB / SSL_PSMDB docker-compose scripts under qa-integration/, driven +# through run_setup_script(). These want a full patch +# version and the PMM Server *address* (host:port). +# MLAUNCH_* / SSL_MLAUNCH ordinary Ansible playbooks, like every other setup. +# +# See setups/mysql.sh for the general pattern the env maps follow. + +# Resolve the PSMDB version for the current spec. +# +# Precedence: PSMDB_VERSION env > spec version expanded to its newest patch > +# registered default. The middle case is why this helper exists at all: the +# compose scripts need a full version such as '8.0-12.1', so a spec version is +# sent through latest_psmdb_version() (a network lookup) rather than used +# as-is. +# +# Shared by setup_psmdb and setup_ssl_psmdb. +# +# Reads: PSMDB_VERSION, DB_VERSION, DB_TYPE +# Stdout: the resolved version +psmdb_version() { + if [[ -n ${PSMDB_VERSION:-} ]]; then + printf '%s' "$PSMDB_VERSION" + elif [[ -n $DB_VERSION ]]; then + latest_psmdb_version "$DB_VERSION" + else + database_default_version "$DB_TYPE" + fi +} + +# Percona Server for MongoDB as a replica set or a sharded cluster. +# +# Script-backed rather than playbook-backed, so it needs +# PMM_SERVER_CONTAINER_ADDRESS (host:port) instead of PMM_SERVER_IP, and names +# the client version PMM_CLIENT_VERSION. TESTS/CLEANUP are pinned to 'no' -- +# the scripts can run their own tests and tear down afterwards, which the +# framework never wants. +# +# SETUP_TYPE picks the script; 'shards' and 'sharding' are aliases. +setup_psmdb() { + local version client setup_type script + version=$(psmdb_version) + client=$(resolved_client_version PSMDB DB_CONFIG) + setup_type=$(resolve_value PSMDB SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + + declare -A env_map=( + [PSMDB_VERSION]="$version" + [PMM_SERVER_CONTAINER_ADDRESS]="$PMM_SERVER_HOST:$PMM_SERVER_PORT" + [PSMDB_CONTAINER]="psmdb_pmm_$version" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_CLIENT_VERSION]="$client" + [COMPOSE_PROFILES]="$(resolve_value PSMDB COMPOSE_PROFILES DB_CONFIG)" + [MONGO_SETUP_TYPE]="$setup_type" + [MONGO_STORAGE_ENGINE]="$(resolve_value PSMDB STORAGE_ENGINE DB_CONFIG)" + [OL_VERSION]="$(resolve_value PSMDB OL_VERSION DB_CONFIG)" + [GSSAPI]="$(resolve_value PSMDB GSSAPI DB_CONFIG)" + [TESTS]=no + [CLEANUP]=no + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + + case "$setup_type" in + pss|psa) script='start-rs-only.sh' ;; + shards|sharding) script='start-sharded.sh' ;; + *) die "Unsupported PSMDB SETUP_TYPE '$setup_type'." ;; + esac + run_setup_script "$QA_INTEGRATION_ROOT/pmm_psmdb-pbm_setup" "$script" env_map +} + +# PSMDB launched with mlaunch instead of docker-compose. +# +# Playbook-backed, so unlike setup_psmdb it takes a plain major version and +# uses the usual PMM_SERVER_IP / CLIENT_VERSION key names. +setup_mlaunch_psmdb() { + local version client + version=$(resolved_version PSMDB_VERSION MLAUNCH_PSMDB "$DB_VERSION") + client=$(resolved_client_version MLAUNCH_PSMDB DB_CONFIG) + declare -A env_map=( + [PSMDB_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [PSMDB_CONTAINER]="psmdb_pmm_$version" + [PSMDB_SETUP]="$(resolve_value MLAUNCH_PSMDB SETUP_TYPE DB_CONFIG)" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + ) + run_playbook 'mlaunch_psmdb_setup.yml' env_map +} + +# Upstream MongoDB launched with mlaunch. +# +# Same as setup_mlaunch_psmdb but for MongoDB Community; note the MODB_* key +# names its playbook expects. +setup_mlaunch_modb() { + local version client + version=$(resolved_version MODB_VERSION MLAUNCH_MODB "$DB_VERSION") + client=$(resolved_client_version MLAUNCH_MODB DB_CONFIG) + declare -A env_map=( + [MODB_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [MODB_CONTAINER]="modb_pmm_$version" + [MODB_SETUP]="$(resolve_value MLAUNCH_MODB SETUP_TYPE DB_CONFIG)" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + ) + run_playbook 'mlaunch_modb_setup.yml' env_map +} + +# mlaunch-based MongoDB with TLS. +setup_ssl_mlaunch() { + local version client + version=$(resolved_version PSMDB_VERSION SSL_MLAUNCH "$DB_VERSION") + client=$(resolved_client_version SSL_MLAUNCH DB_CONFIG) + declare -A env_map=( + [MONGODB_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [MONGODB_SSL_CONTAINER]="psmdb_ssl_pmm_$version" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + ) + run_playbook 'tls-ssl-setup/mlaunch_tls_setup.yml' env_map +} + +# PSMDB with the different-authentication (TLS/Kerberos) compose stack. +# +# The most involved setup in the framework, because the tracked assets it reuses +# assume they own the whole environment. Rather than editing files under version +# control, it builds a throwaway copy in a temp directory: +# +# * a compose override that disables the stack's own pmm-server and kerberos +# services (the framework supplies the server), resets psmdb-server's +# depends_on, and joins the external pmm-qa network +# * a patched copy of test-auth.sh that loads the override and points +# --server-address at the resolved PMM Server +# +# The override references ${ADMIN_PASSWORD} and ${PMM_SERVER_CONTAINER_ADDRESS} +# as literals -- escaped in the heredoc -- so compose expands them from the env +# map at run time and no secret is written to disk. +# +# The EXIT trap plus the SSL_PSMDB_TEMP_DIR global ensure the temp directory is +# removed even when the script fails; both are cleared on the success path. +setup_ssl_psmdb() { + local version client directory base_script temp_dir override temp_script + version=$(psmdb_version) + client=$(resolved_client_version SSL_PSMDB DB_CONFIG) + directory=$QA_INTEGRATION_ROOT/pmm_psmdb_diffauth_setup + base_script=$directory/test-auth.sh + temp_dir=$(mktemp -d "${TMPDIR:-/tmp}/pmm-framework-ssl-psmdb.XXXXXX") + override=$temp_dir/compose.yml + temp_script=$temp_dir/test-auth.sh + SSL_PSMDB_TEMP_DIR=$temp_dir + trap 'rm -rf "$SSL_PSMDB_TEMP_DIR"' EXIT + + cat >"$override" <"$temp_script" + chmod +x "$temp_script" + + declare -A env_map=( + [PSMDB_VERSION]="$version" + [PMM_SERVER_CONTAINER_ADDRESS]="$PMM_SERVER_HOST:$PMM_SERVER_PORT" + [PSMDB_CONTAINER]="psmdb_pmm_$version" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_CLIENT_VERSION]="$client" + [COMPOSE_PROFILES]="$(resolve_value SSL_PSMDB COMPOSE_PROFILES DB_CONFIG)" + [MONGO_SETUP_TYPE]="$(resolve_value SSL_PSMDB SETUP_TYPE DB_CONFIG)" + [TESTS]=no + [CLEANUP]=no + ) + + # Absolute script path, but the original directory as cwd: the patched script + # still resolves the stack's compose files relative to where they live. + run_setup_script "$directory" "$temp_script" env_map + rm -rf "$temp_dir" + SSL_PSMDB_TEMP_DIR='' + trap - EXIT +} diff --git a/qa-integration/pmm_qa/pmm-framework/setups/mysql.sh b/qa-integration/pmm_qa/pmm-framework/setups/mysql.sh new file mode 100644 index 000000000..2c205be9b --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/setups/mysql.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# +# setups/mysql.sh -- MySQL-family setups: PS, MYSQL, SSL_MYSQL, PXC. +# +# Every setup function here follows the same shape, which is the pattern to +# copy when adding a new one: +# +# 1. resolve the version, setup type and client version +# 2. build `declare -A env_map=(...)` -- the contract with the playbook +# 3. hand it to run_playbook() (or run_setup_script()) +# +# The env maps are written out in full rather than shared through a helper. The +# repetition is deliberate: each map mirrors exactly what its playbook reads, +# and the differences between them are real (setup_external omits CLIENT_DEBUG, +# the PSMDB setups use PMM_CLIENT_VERSION instead of CLIENT_VERSION). Factoring +# out the common keys would hide those asymmetries. +# +# Reads, in every function: DB_VERSION and DB_CONFIG (set by +# parse_database_spec), PMM_SERVER_HOST, CLIENT_DEBUG. + +# Percona Server for MySQL. +# +# SETUP_TYPE selects the topology inside the playbook ('' single, gr, replication). +# NODES_COUNT, MY_ROCKS and BACKUP are passed through for the playbook to act on. +setup_ps() { + local version setup_type client + version=$(resolved_version PS_VERSION PS "$DB_VERSION") + setup_type=$(resolve_value PS SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + client=$(resolved_client_version PS DB_CONFIG) + + declare -A env_map=( + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [SETUP_TYPE]="$setup_type" + [NODES_COUNT]="$(resolve_value PS NODES_COUNT DB_CONFIG)" + [QUERY_SOURCE]="$(resolve_value PS QUERY_SOURCE DB_CONFIG)" + [PS_VERSION]="$version" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [MY_ROCKS]="$(resolve_value PS MY_ROCKS DB_CONFIG)" + [ENCRYPTED_CLIENT_CONFIG]="$(resolve_value PS ENCRYPTED_CLIENT_CONFIG DB_CONFIG)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + [BACKUP]="$(resolve_value PS BACKUP DB_CONFIG)" + ) + run_playbook 'percona_server_for_mysql/percona-server-setup.yml' env_map +} + +# Upstream MySQL. +# +# Unlike PS, this playbook wants the topology pre-translated: SETUP_TYPE=gr sets +# GROUP_REPLICATION=1, and SETUP_TYPE=replication asks for two nodes. Both are +# still passed alongside the raw SETUP_TYPE. +setup_mysql() { + local version setup_type client group_replication='' nodes=1 + version=$(resolved_version MS_VERSION MYSQL "$DB_VERSION") + setup_type=$(resolve_value MYSQL SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + client=$(resolved_client_version MYSQL DB_CONFIG) + if [[ $setup_type == gr ]]; then + group_replication=1 + elif [[ $setup_type == replication ]]; then + nodes=2 + fi + + declare -A env_map=( + [GROUP_REPLICATION]="$group_replication" + [MS_NODES]="$nodes" + [MS_VERSION]="$version" + [SETUP_TYPE]="$setup_type" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [MS_CONTAINER]="mysql_pmm_$version" + [CLIENT_VERSION]="$client" + [QUERY_SOURCE]="$(resolve_value MYSQL QUERY_SOURCE DB_CONFIG)" + [MS_TARBALL]="$(resolve_value MYSQL TARBALL DB_CONFIG)" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [ENCRYPTED_CLIENT_CONFIG]="$(resolve_value MYSQL ENCRYPTED_CLIENT_CONFIG DB_CONFIG)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'mysql/mysql-setup.yml' env_map +} + +# MySQL with TLS, monitored over an encrypted connection. +setup_ssl_mysql() { + local version client + version=$(resolved_version MS_VERSION SSL_MYSQL "$DB_VERSION") + client=$(resolved_client_version SSL_MYSQL DB_CONFIG) + declare -A env_map=( + [MYSQL_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [MYSQL_SSL_CONTAINER]="mysql_ssl_$version" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'tls-ssl-setup/mysql_tls_setup.yml' env_map +} + +# Percona XtraDB Cluster, always three nodes, fronted by ProxySQL. +# +# ProxySQL is not separately requestable (dispatch_setup rejects it), so its +# version and package come from the PROXYSQL registration instead of a spec. +# The empty proxysql_config array exists only to satisfy resolve_value's +# signature -- there is no PROXYSQL spec to read options from. +setup_pxc() { + local version proxysql_version client + version=$(resolved_version PXC_VERSION PXC "$DB_VERSION") + proxysql_version=${PROXYSQL_VERSION:-$(database_default_version PROXYSQL)} + client=$(resolved_client_version PXC DB_CONFIG) + declare -A proxysql_config=() + declare -A env_map=( + [PXC_NODES]=3 + [PXC_VERSION]="$version" + [PROXYSQL_VERSION]="$proxysql_version" + [PXC_TARBALL]="$(resolve_value PXC TARBALL DB_CONFIG)" + [PROXYSQL_PACKAGE]="$(resolve_value PROXYSQL PACKAGE proxysql_config)" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [PXC_CONTAINER]="pxc_proxysql_pmm_$version" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [QUERY_SOURCE]="$(resolve_value PXC QUERY_SOURCE DB_CONFIG)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'pxc_proxysql_setup.yml' env_map +} diff --git a/qa-integration/pmm_qa/pmm-framework/setups/postgresql.sh b/qa-integration/pmm_qa/pmm-framework/setups/postgresql.sh new file mode 100644 index 000000000..4e9e7b2f8 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/setups/postgresql.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# setups/postgresql.sh -- PostgreSQL-family setups: PDPGSQL, SSL_PDPGSQL, PGSQL. +# +# Same shape as setups/mysql.sh -- see that file's header for the pattern and +# for why the env maps are spelled out rather than shared. +# +# The distinction between the two products: PDPGSQL is the Percona Distribution +# with pg_stat_monitor (PGSM), PGSQL is upstream PostgreSQL with +# pg_stat_statements (PGSS). They use different playbooks, ports and container +# names. + +# Percona Distribution for PostgreSQL, monitored through pg_stat_monitor. +# +# PGSTAT_MONITOR_BRANCH is pinned to 'main' while PGSM_BRANCH stays +# spec-controlled: the playbook uses them for different things, so the +# similar-looking names are not a duplicate. DISTRIBUTION is intentionally +# empty -- the playbook fills in its own default. +setup_pdpgsql() { + local version setup_type pgsm_branch client + version=$(resolved_version PDPGSQL_VERSION PDPGSQL "$DB_VERSION") + setup_type=$(resolve_value PDPGSQL SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + pgsm_branch=$(resolve_value PDPGSQL PGSM_BRANCH DB_CONFIG) + pgsm_branch=${pgsm_branch,,} + client=$(resolved_client_version PDPGSQL DB_CONFIG) + + declare -A env_map=( + [PGSTAT_MONITOR_BRANCH]=main + [PDPGSQL_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [PDPGSQL_PGSM_CONTAINER]="pdpgsql_pgsm_pmm_$version" + [CLIENT_VERSION]="$client" + [USE_SOCKET]="$(resolve_value PDPGSQL USE_SOCKET DB_CONFIG)" + [ADMIN_PASSWORD]="$(admin_password)" + [PDPGSQL_PGSM_PORT]=5447 + [DISTRIBUTION]='' + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [SETUP_TYPE]="$setup_type" + [PGSM_BRANCH]="$pgsm_branch" + [ENCRYPTED_CLIENT_CONFIG]="$(resolve_value PDPGSQL ENCRYPTED_CLIENT_CONFIG DB_CONFIG)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook \ + 'percona-distribution-postgresql/percona-distribution-postgres-setup.yml' \ + env_map +} + +# Percona Distribution for PostgreSQL with TLS. +# +# Note the env key is PGSQL_VERSION even though the type is SSL_PDPGSQL -- that +# is what the TLS playbook reads. Match the playbook, not the type name. +setup_ssl_pdpgsql() { + local version client + version=$(resolved_version PDPGSQL_VERSION SSL_PDPGSQL "$DB_VERSION") + client=$(resolved_client_version SSL_PDPGSQL DB_CONFIG) + declare -A env_map=( + [PGSTAT_MONITOR_BRANCH]=main + [PGSQL_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [PGSQL_SSL_CONTAINER]="pdpgsql_pgsm_ssl_$version" + [CLIENT_VERSION]="$client" + [USE_SOCKET]="$(resolve_value SSL_PDPGSQL USE_SOCKET DB_CONFIG)" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'tls-ssl-setup/postgresql_tls_setup.yml' env_map +} + +# Upstream PostgreSQL, monitored through pg_stat_statements. +# +# One of the two setups that pick their playbook at runtime (setup_valkey is +# the other). SETUP_TYPE=replication switches to the replication playbook and +# adds two keys that only it reads -- which is why they are appended after the +# map literal rather than always being present. +setup_pgsql() { + local version setup_type client playbook + version=$(resolved_version PGSQL_VERSION PGSQL "$DB_VERSION") + setup_type=$(resolve_value PGSQL SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + client=$(resolved_client_version PGSQL DB_CONFIG) + + declare -A env_map=( + [PGSQL_VERSION]="$version" + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [PGSQL_PGSS_CONTAINER]="pgsql_pgss_pmm_$version" + [CLIENT_VERSION]="$client" + [USE_SOCKET]="$(resolve_value PGSQL USE_SOCKET DB_CONFIG)" + [ADMIN_PASSWORD]="$(admin_password)" + [PGSQL_PGSS_PORT]=5448 + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + + if [[ $setup_type == replication ]]; then + env_map[SETUP_TYPE]=$setup_type + env_map[ENCRYPTED_CLIENT_CONFIG]=$(resolve_value PGSQL ENCRYPTED_CLIENT_CONFIG DB_CONFIG) + playbook='postgresql/postgresql-setup.yml' + else + playbook='pgsql_pgss_setup.yml' + fi + run_playbook "$playbook" env_map +} diff --git a/qa-integration/pmm_qa/pmm-framework/setups/services.sh b/qa-integration/pmm_qa/pmm-framework/setups/services.sh new file mode 100644 index 000000000..05d6ab8b9 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/setups/services.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# setups/services.sh -- non-database targets and Valkey. +# +# Same shape as the other setups files (see setups/mysql.sh for the pattern), +# but these cover things that are not a relational database: proxies, exporters, +# object storage and the pre-built client images. + +# HAProxy with the PMM Client attached, for the HAProxy dashboards. +setup_haproxy() { + local client + client=$(resolved_client_version HAPROXY DB_CONFIG) + declare -A env_map=( + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [HAPROXY_CONTAINER]=haproxy_pmm + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + run_playbook 'haproxy_setup.yml' env_map +} + +# External exporters (redis_exporter and process_exporter) registered with PMM. +# +# Their versions are not spec options -- override them with the REDIS_VERSION +# and NODE_PROCESS_VERSION environment variables. Note this setup intentionally +# does not pass CLIENT_DEBUG; its playbook does not read it. +setup_external() { + local client redis_version node_version + client=$(resolved_client_version EXTERNAL DB_CONFIG) + redis_version=${REDIS_VERSION:-1.58.0} + node_version=${NODE_PROCESS_VERSION:-0.7.10} + declare -A env_map=( + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [REDIS_EXPORTER_VERSION]="$redis_version" + [NODE_PROCESS_EXPORTER_VERSION]="$node_version" + [EXTERNAL_CONTAINER]=external_pmm + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + ) + run_playbook 'external_setup.yml' env_map +} + +# Valkey, as either a cluster or a sentinel topology. +# +# One of the two setups that pick their playbook at runtime (setup_pgsql is the +# other). Cluster is the default; sentinel must be asked for explicitly, and +# both the singular and plural spellings are accepted as aliases. +setup_valkey() { + local version setup_type client playbook + version=$(resolved_version VALKEY_VERSION VALKEY "$DB_VERSION") + setup_type=$(resolve_value VALKEY SETUP_TYPE DB_CONFIG) + setup_type=${setup_type,,} + client=$(resolved_client_version VALKEY DB_CONFIG) + declare -A env_map=( + [PMM_SERVER_IP]="$PMM_SERVER_HOST" + [VALKEY_VERSION]="$version" + [CLIENT_VERSION]="$client" + [ADMIN_PASSWORD]="$(admin_password)" + [PMM_QA_GIT_BRANCH]="$(git_branch)" + [SETUP_TYPE]="$setup_type" + [ENCRYPTED_CLIENT_CONFIG]="$(resolve_value VALKEY ENCRYPTED_CLIENT_CONFIG DB_CONFIG)" + [CLIENT_DEBUG]="$(bool_string "$CLIENT_DEBUG")" + ) + case "$setup_type" in + sentinel|sentinels) playbook='valkey/valkey-sentinel.yml' ;; + *) playbook='valkey/valkey-cluster.yml' ;; + esac + run_playbook "$playbook" env_map +} + +# A MinIO container holding S3 buckets, used as a backup location. +# +# BUCKET_NAMES is normalised before the playbook sees it: quotes stripped, +# lower-cased, and ';' separators turned into ',' -- the playbook splits on +# commas. So `BUCKET_NAMES=one;two` and `BUCKET_NAMES=one,two` are equivalent. +# Needs no PMM Server (see setup_requires_server). +setup_bucket() { + local buckets + buckets=$(resolve_value BUCKET BUCKET_NAMES DB_CONFIG) + buckets=${buckets//\"/} + buckets=${buckets,,} + buckets=${buckets//;/,} + declare -A env_map=([BUCKETS]="$buckets") + run_playbook 'tasks/create_minio_container.yml' env_map +} + +# Build the pre-baked client Docker images used by other suites. +# +# The only setup with an empty env map: the script takes no parameters. It also +# needs no PMM Server, and is script-backed rather than playbook-backed. +setup_dockerclients() { + declare -A env_map=() + run_setup_script "$PMM_QA_ROOT" 'setup_docker_client_images.sh' env_map +} diff --git a/qa-integration/pmm_qa/pmm-framework/tests/cli.bats b/qa-integration/pmm_qa/pmm-framework/tests/cli.bats new file mode 100644 index 000000000..a43889692 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/tests/cli.bats @@ -0,0 +1,243 @@ +#!/usr/bin/env bats + +load helpers/test_helper + +@test "parses repeatable database specs and global flags" { + parse_args \ + --verbose \ + --parallel \ + --verbosity-level=2 \ + --client-version latest-tarball \ + --pmm-server-ip=10.0.0.5 \ + --database ps=8.4,SETUP_TYPE=gr \ + --database=psmdb,SETUP_TYPE=sharding + + [[ $VERBOSE == true ]] + [[ $PARALLEL == true ]] + [[ $VERBOSITY_LEVEL == 2 ]] + [[ $GLOBAL_CLIENT_VERSION == latest-tarball ]] + [[ $PMM_SERVER_IP_ARG == 10.0.0.5 ]] + [[ ${#DATABASE_SPECS[@]} == 2 ]] + [[ ${DATABASE_SPECS[0]} == ps=8.4,SETUP_TYPE=gr ]] + [[ ${DATABASE_SPECS[1]} == psmdb,SETUP_TYPE=sharding ]] +} + +@test "parses a database spec case-insensitively" { + parse_database_spec 'Ps=8.4,setup_type=gr,query_source=slowlog' + + [[ $DB_TYPE == PS ]] + [[ $DB_VERSION == 8.4 ]] + [[ ${DB_CONFIG[SETUP_TYPE]} == gr ]] + [[ ${DB_CONFIG[QUERY_SOURCE]} == slowlog ]] +} + +@test "invalid version falls back to configured default" { + parse_database_spec 'ps=99' + + [[ -z $DB_VERSION ]] + [[ $(resolved_version PS_VERSION PS "$DB_VERSION") == 8.0 ]] +} + +@test "value precedence is environment then global then database then default" { + parse_database_spec 'ps,CLIENT_VERSION=from-spec,QUERY_SOURCE=slowlog' + GLOBAL_CLIENT_VERSION=from-global + [[ $(resolve_value PS CLIENT_VERSION DB_CONFIG) == from-global ]] + [[ $(resolve_value PS QUERY_SOURCE DB_CONFIG) == slowlog ]] + + CLIENT_VERSION=from-env + [[ $(resolve_value PS CLIENT_VERSION DB_CONFIG) == from-env ]] + unset CLIENT_VERSION + + GLOBAL_CLIENT_VERSION='' + unset 'DB_CONFIG[QUERY_SOURCE]' + [[ $(resolve_value PS QUERY_SOURCE DB_CONFIG) == perfschema ]] +} + +@test "empty exported versions fall back like Python getenv-or" { + PS_VERSION='' + parse_database_spec 'ps=8.4' + [[ $(resolved_version PS_VERSION PS "$DB_VERSION") == 8.4 ]] + + DB_VERSION='' + [[ $(resolved_version PS_VERSION PS "$DB_VERSION") == 8.0 ]] +} + +@test "optional-value flags do not consume the following option" { + parse_args \ + --pmm-server-ip \ + --pmm-server-password \ + --client-version \ + --verbosity-level \ + --database ps=8.4 + + [[ -z $PMM_SERVER_IP_ARG ]] + [[ -z $PMM_SERVER_PASSWORD ]] + [[ -z $GLOBAL_CLIENT_VERSION ]] + [[ $VERBOSITY_LEVEL == 1 ]] + [[ ${DATABASE_SPECS[0]} == ps=8.4 ]] +} + +@test "prebaked PS options are no longer accepted" { + run parse_args --use-prebaked-ps --database ps=8.4 + [[ $status -ne 0 ]] + [[ $output == *"Unknown option '--use-prebaked-ps'"* ]] + + run parse_args --prebaked-ps-image pmm-qa/ps:8.4 --database ps=8.4 + [[ $status -ne 0 ]] + [[ $output == *"Unknown option '--prebaked-ps-image'"* ]] +} + +@test "normalizes latest-tarball client version" { + [[ $(normalize_client_version latest-tarball) == \ + 'https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client/pmm-client-latest.tar.gz' ]] +} + +@test "resolves latest PSMDB patch without Python" { + curl() { + printf '%s\n' \ + '' \ + '' + } + + [[ $(latest_psmdb_version 8.0) == 8.0-12.1 ]] +} + +@test "selects the existing requests-capable interpreter for Ansible modules" { + local fake_python=$BATS_TEST_TMPDIR/python + printf '#!/usr/bin/env bash\nexit 0\n' >"$fake_python" + chmod +x "$fake_python" + PMM_FRAMEWORK_ANSIBLE_PYTHON_FALLBACK=$fake_python + + configure_ansible_python + + [[ $ANSIBLE_PYTHON_INTERPRETER == "$fake_python" ]] +} + +@test "requires at least one database" { + run parse_args --verbose + [[ $status -ne 0 ]] + [[ $output == *'At least one --database SPEC is required'* ]] +} + +@test "rejects unknown global options" { + run parse_args --not-a-real-option + [[ $status -ne 0 ]] + [[ $output == *"Unknown option '--not-a-real-option'"* ]] +} + +@test "every versioned database pins an explicit default version" { + local -A expected=( + [PSMDB]=latest [SSL_PSMDB]=latest + [MLAUNCH_PSMDB]=8.0 [MLAUNCH_MODB]=8.0 [SSL_MLAUNCH]=8.0 + [MYSQL]=9.7 [PS]=8.0 [SSL_MYSQL]=8.0 + [PGSQL]=17 [PDPGSQL]=17 [SSL_PDPGSQL]=17 + [PXC]=8.0 [PROXYSQL]=2 [VALKEY]=8 + ) + local type actual + for type in "${!expected[@]}"; do + actual=$(database_default_version "$type") + if [[ $actual != "${expected[$type]}" ]]; then + echo "$type default is '$actual', expected '${expected[$type]}'" + return 1 + fi + done +} + +@test "default version is independent of version list order" { + register_database ORDERTEST '9.9 1.1 5.5' 'CLIENT_VERSION' \ + 'DEFAULT_VERSION=1.1' 'CLIENT_VERSION=3-dev-latest' + + [[ $(database_default_version ORDERTEST) == 1.1 ]] +} + +@test "DEFAULT_VERSION is not a user-settable database option" { + parse_database_spec 'pgsql=16,DEFAULT_VERSION=11' + [[ $DB_VERSION == 16 ]] + [[ -z ${DB_CONFIG[DEFAULT_VERSION]-} ]] + + DB_VERSION='' + [[ $(resolved_version PGSQL_VERSION PGSQL "$DB_VERSION") == 17 ]] +} + +# Stubs `docker ps` with the given "imagename" lines and makes the +# pmm-qa network already exist with $1 connected, so no network calls are made. +stub_docker_ps() { + local connected=$1 + shift + local -a rows=("$@") + eval "docker() { + if [[ \$1 == ps ]]; then printf '%s\n' ${rows[*]@Q}; return 0; fi + if [[ \$1 == network && \$2 == inspect ]]; then + [[ \$* == *--format* ]] && printf '%s\n' ${connected@Q} + return 0 + fi + return 0 + }" +} + +@test "discovers a single PMM Server container without warning" { + stub_docker_ps pmm-server-a $'percona/pmm-server:3\tpmm-server-a' + + run discover_pmm_server + [[ $status -eq 0 ]] + [[ $output != *'PMM Server containers'* ]] + + discover_pmm_server + [[ $PMM_SERVER_CONTAINER == pmm-server-a ]] +} + +@test "warns and stays deterministic when several PMM Servers are running" { + stub_docker_ps pmm-server-a \ + $'percona/pmm-server:3\tpmm-server-a' \ + $'percona/pmm-server:2\tpmm-server-b' + + run discover_pmm_server + [[ $status -eq 0 ]] + [[ $output == *'Found 2 PMM Server containers'* ]] + [[ $output == *pmm-server-a* && $output == *pmm-server-b* ]] + [[ $output == *--pmm-server-ip* ]] + + discover_pmm_server + [[ $PMM_SERVER_CONTAINER == pmm-server-a ]] +} + +@test "reports no PMM Server when none is running" { + stub_docker_ps '' $'mysql:8.4\tsome-db' + + run discover_pmm_server + [[ $status -ne 0 ]] +} + +@test "successful setup logs stay on disk and print only a summary" { + local log=$BATS_TEST_TMPDIR/setup.log + printf 'first line\nno trailing newline' >"$log" + + run print_setup_log 1 2 'ps=8.4' 0 "$log" + + [[ $status -eq 0 ]] + [[ $output == *"[1/2] ps=8.4: OK (log: $log)"* ]] + [[ $output != *'first line'* ]] +} + +@test "verbose echoes the logs of setups that succeeded" { + local log=$BATS_TEST_TMPDIR/setup.log + printf 'first line\nno trailing newline' >"$log" + VERBOSE=true + + run print_setup_log 1 2 'ps=8.4' 0 "$log" + + [[ $status -eq 0 ]] + [[ $output == *"[1/2] ps=8.4: OK (log: $log)"* ]] + [[ $output == *$'no trailing newline\n===== END [1/2] ps=8.4 ====='* ]] +} + +@test "failed setup logs dump to the console with END on its own line" { + local log=$BATS_TEST_TMPDIR/setup.log + printf 'first line\nno trailing newline' >"$log" + + run print_setup_log 1 2 'ps=8.4' 1 "$log" + + [[ $status -eq 0 ]] + [[ $output == *'===== [1/2] ps=8.4 FAILED (exit=1) ====='* ]] + [[ $output == *$'no trailing newline\n===== END [1/2] ps=8.4 ====='* ]] +} diff --git a/qa-integration/pmm_qa/pmm-framework/tests/dispatch.bats b/qa-integration/pmm_qa/pmm-framework/tests/dispatch.bats new file mode 100644 index 000000000..097ec01df --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/tests/dispatch.bats @@ -0,0 +1,186 @@ +#!/usr/bin/env bats + +load helpers/test_helper + +@test "PS GR selects the existing playbook and exact environment" { + parse_database_spec 'ps=8.4,SETUP_TYPE=gr,QUERY_SOURCE=slowlog' + GLOBAL_CLIENT_VERSION=latest-tarball + CLIENT_DEBUG=true + dispatch_setup + + [[ $CAPTURE_KIND == playbook ]] + [[ $CAPTURE_TARGET == percona_server_for_mysql/percona-server-setup.yml ]] + [[ ${CAPTURE_ENV[PS_VERSION]} == 8.4 ]] + [[ ${CAPTURE_ENV[SETUP_TYPE]} == gr ]] + [[ ${CAPTURE_ENV[QUERY_SOURCE]} == slowlog ]] + [[ ${CAPTURE_ENV[NODES_COUNT]} == 1 ]] + [[ ${CAPTURE_ENV[CLIENT_DEBUG]} == true ]] + [[ ${CAPTURE_ENV[CLIENT_VERSION]} == https://pmm-build-cache.s3.us-east-2.amazonaws.com/PR-BUILDS/pmm-client/pmm-client-latest.tar.gz ]] + # Prebaked-image plumbing was a POC and is no longer part of the framework. + [[ -z ${CAPTURE_ENV[USE_PREBAKED_PS]-} ]] + [[ -z ${CAPTURE_ENV[PREBAKED_PS_IMAGE]-} ]] +} + +@test "MySQL GR maps group replication and node values" { + parse_database_spec 'mysql=8.4,SETUP_TYPE=gr' + dispatch_setup + + [[ $CAPTURE_TARGET == mysql/mysql-setup.yml ]] + [[ ${CAPTURE_ENV[MS_VERSION]} == 8.4 ]] + [[ ${CAPTURE_ENV[GROUP_REPLICATION]} == 1 ]] + [[ ${CAPTURE_ENV[MS_NODES]} == 1 ]] + [[ ${CAPTURE_ENV[MS_CONTAINER]} == mysql_pmm_8.4 ]] +} + +@test "PGSQL replication selects replication playbook" { + parse_database_spec 'pgsql=16,SETUP_TYPE=replication,ENCRYPTED_CLIENT_CONFIG=true' + dispatch_setup + + [[ $CAPTURE_TARGET == postgresql/postgresql-setup.yml ]] + [[ ${CAPTURE_ENV[PGSQL_VERSION]} == 16 ]] + [[ ${CAPTURE_ENV[SETUP_TYPE]} == replication ]] + [[ ${CAPTURE_ENV[ENCRYPTED_CLIENT_CONFIG]} == true ]] +} + +@test "PDPGSQL maps patroni and PGSM values" { + parse_database_spec 'pdpgsql=17,SETUP_TYPE=patroni,PGSM_BRANCH=feature' + dispatch_setup + + [[ $CAPTURE_TARGET == percona-distribution-postgresql/percona-distribution-postgres-setup.yml ]] + [[ ${CAPTURE_ENV[PDPGSQL_VERSION]} == 17 ]] + [[ ${CAPTURE_ENV[SETUP_TYPE]} == patroni ]] + [[ ${CAPTURE_ENV[PGSM_BRANCH]} == feature ]] + [[ ${CAPTURE_ENV[PDPGSQL_PGSM_PORT]} == 5447 ]] +} + +@test "PSMDB sharding alias selects sharded script and PMM client env name" { + parse_database_spec 'psmdb=latest,SETUP_TYPE=shards,COMPOSE_PROFILES=extra,OL_VERSION=8,GSSAPI=true' + GLOBAL_CLIENT_VERSION=3-dev-latest + dispatch_setup + + [[ $CAPTURE_KIND == script ]] + [[ $CAPTURE_TARGET == start-sharded.sh ]] + [[ $CAPTURE_DIRECTORY == "$QA_INTEGRATION_ROOT/pmm_psmdb-pbm_setup" ]] + [[ ${CAPTURE_ENV[PSMDB_VERSION]} == latest ]] + [[ ${CAPTURE_ENV[MONGO_SETUP_TYPE]} == shards ]] + [[ ${CAPTURE_ENV[COMPOSE_PROFILES]} == extra ]] + [[ ${CAPTURE_ENV[OL_VERSION]} == 8 ]] + [[ ${CAPTURE_ENV[GSSAPI]} == true ]] + [[ ${CAPTURE_ENV[PMM_CLIENT_VERSION]} == 3-dev-latest ]] +} + +@test "PXC tarball selects PXC and ProxySQL playbook" { + parse_database_spec 'PXC=8.0,TARBALL=/tmp/pxc.tar.gz' + dispatch_setup + + [[ $CAPTURE_TARGET == pxc_proxysql_setup.yml ]] + [[ ${CAPTURE_ENV[PXC_VERSION]} == 8.0 ]] + [[ ${CAPTURE_ENV[PXC_TARBALL]} == /tmp/pxc.tar.gz ]] + [[ ${CAPTURE_ENV[PROXYSQL_VERSION]} == 2 ]] + [[ ${CAPTURE_ENV[PXC_NODES]} == 3 ]] +} + +@test "Valkey sentinel alias selects sentinel playbook" { + parse_database_spec 'valkey=8,SETUP_TYPE=sentinels' + dispatch_setup + + [[ $CAPTURE_TARGET == valkey/valkey-sentinel.yml ]] + [[ ${CAPTURE_ENV[VALKEY_VERSION]} == 8 ]] + [[ ${CAPTURE_ENV[SETUP_TYPE]} == sentinels ]] +} + +@test "multiple specs dispatch sequentially without leaking environment maps" { + local -a targets=() + local spec + for spec in 'ps=8.4' 'external' 'haproxy'; do + parse_database_spec "$spec" + dispatch_setup + targets+=("$CAPTURE_TARGET") + done + + [[ ${targets[0]} == percona_server_for_mysql/percona-server-setup.yml ]] + [[ ${targets[1]} == external_setup.yml ]] + [[ ${targets[2]} == haproxy_setup.yml ]] + [[ -z ${CAPTURE_ENV[PS_VERSION]-} ]] + [[ -z ${CAPTURE_ENV[REDIS_EXPORTER_VERSION]-} ]] +} + +@test "SSL variants select their existing playbooks" { + parse_database_spec 'ssl_mysql=8.4' + dispatch_setup + [[ $CAPTURE_TARGET == tls-ssl-setup/mysql_tls_setup.yml ]] + + parse_database_spec 'ssl_pdpgsql=16' + dispatch_setup + [[ $CAPTURE_TARGET == tls-ssl-setup/postgresql_tls_setup.yml ]] + + parse_database_spec 'ssl_mlaunch=8.0' + dispatch_setup + [[ $CAPTURE_TARGET == tls-ssl-setup/mlaunch_tls_setup.yml ]] +} + +@test "mlaunch variants retain their playbook and variable names" { + parse_database_spec 'mlaunch_psmdb=8.0,SETUP_TYPE=sharding' + dispatch_setup + [[ $CAPTURE_TARGET == mlaunch_psmdb_setup.yml ]] + [[ ${CAPTURE_ENV[PSMDB_SETUP]} == sharding ]] + + parse_database_spec 'mlaunch_modb=7.0,SETUP_TYPE=pss' + dispatch_setup + [[ $CAPTURE_TARGET == mlaunch_modb_setup.yml ]] + [[ ${CAPTURE_ENV[MODB_VERSION]} == 7.0 ]] + [[ ${CAPTURE_ENV[MODB_SETUP]} == pss ]] +} + +@test "SSL PSMDB uses a temporary script without editing tracked setup files" { + PMM_SERVER_HOST=192.0.2.10 + PMM_SERVER_PORT=443 + PMM_SERVER_PASSWORD=$'quote" slash\\ newline\nvalue' + parse_database_spec 'ssl_psmdb=latest,SETUP_TYPE=pss' + dispatch_setup + + [[ $CAPTURE_KIND == script ]] + [[ $CAPTURE_DIRECTORY == "$QA_INTEGRATION_ROOT/pmm_psmdb_diffauth_setup" ]] + [[ $CAPTURE_TARGET == */pmm-framework-ssl-psmdb.*/test-auth.sh ]] + [[ ! -e $CAPTURE_TARGET ]] + [[ $CAPTURE_SCRIPT_CONTENT == *'--server-address=192.0.2.10:443'* ]] + # shellcheck disable=SC2016 + [[ $CAPTURE_OVERRIDE_CONTENT == *'PMM_AGENT_SERVER_PASSWORD: "${ADMIN_PASSWORD}"'* ]] + [[ $CAPTURE_OVERRIDE_CONTENT != *"$PMM_SERVER_PASSWORD"* ]] + [[ ${CAPTURE_ENV[PSMDB_VERSION]} == latest ]] + [[ ${CAPTURE_ENV[MONGO_SETUP_TYPE]} == pss ]] + + first_target=$CAPTURE_TARGET + dispatch_setup + [[ $CAPTURE_TARGET != "$first_target" ]] + [[ ! -e $CAPTURE_TARGET ]] +} + +@test "service handlers map exporters and client debug" { + REDIS_VERSION=1.14.0 + NODE_PROCESS_VERSION=0.7.5 + parse_database_spec external + dispatch_setup + [[ $CAPTURE_TARGET == external_setup.yml ]] + [[ ${CAPTURE_ENV[REDIS_EXPORTER_VERSION]} == 1.14.0 ]] + [[ ${CAPTURE_ENV[NODE_PROCESS_EXPORTER_VERSION]} == 0.7.5 ]] + + CLIENT_DEBUG=true + parse_database_spec haproxy + dispatch_setup + [[ $CAPTURE_TARGET == haproxy_setup.yml ]] + [[ ${CAPTURE_ENV[CLIENT_DEBUG]} == true ]] +} + +@test "bucket and Docker client setups reuse current targets" { + parse_database_spec 'bucket,BUCKET_NAMES=one;two' + dispatch_setup + [[ $CAPTURE_TARGET == tasks/create_minio_container.yml ]] + [[ ${CAPTURE_ENV[BUCKETS]} == one,two ]] + + parse_database_spec dockerclients + dispatch_setup + [[ $CAPTURE_KIND == script ]] + [[ $CAPTURE_DIRECTORY == "$PMM_QA_ROOT" ]] + [[ $CAPTURE_TARGET == setup_docker_client_images.sh ]] +} diff --git a/qa-integration/pmm_qa/pmm-framework/tests/helpers/test_helper.bash b/qa-integration/pmm_qa/pmm-framework/tests/helpers/test_helper.bash new file mode 100644 index 000000000..650b706c0 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/tests/helpers/test_helper.bash @@ -0,0 +1,77 @@ +FRAMEWORK_DIR=$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." && pwd) +PMM_QA_ROOT=$(cd "$FRAMEWORK_DIR/.." && pwd) +QA_INTEGRATION_ROOT=$(cd "$PMM_QA_ROOT/.." && pwd) + +source "$FRAMEWORK_DIR/lib/common.sh" +source "$FRAMEWORK_DIR/lib/config.sh" +source "$FRAMEWORK_DIR/lib/cli.sh" +source "$FRAMEWORK_DIR/lib/docker.sh" +source "$FRAMEWORK_DIR/lib/ansible.sh" +source "$FRAMEWORK_DIR/lib/runners.sh" +source "$FRAMEWORK_DIR/setups/mysql.sh" +source "$FRAMEWORK_DIR/setups/postgresql.sh" +source "$FRAMEWORK_DIR/setups/mongodb.sh" +source "$FRAMEWORK_DIR/setups/services.sh" +source "$FRAMEWORK_DIR/setups/dispatch.sh" +source "$FRAMEWORK_DIR/lib/execution.sh" + +reset_framework_state() { + unset PS_VERSION MS_VERSION PSMDB_VERSION PDPGSQL_VERSION PGSQL_VERSION + unset PXC_VERSION PROXYSQL_VERSION VALKEY_VERSION CLIENT_VERSION + unset REDIS_VERSION NODE_PROCESS_VERSION ADMIN_PASSWORD PMM_QA_GIT_BRANCH + unset ANSIBLE_PYTHON_INTERPRETER PMM_FRAMEWORK_ANSIBLE_PYTHON_FALLBACK + DATABASE_SPECS=() + DB_CONFIG=() + DB_TYPE='' + DB_VERSION='' + PMM_SERVER_IP_ARG='' + PMM_SERVER_PASSWORD='' + GLOBAL_CLIENT_VERSION='' + PMM_SERVER_HOST=pmm-server + PMM_SERVER_PORT=8443 + PMM_SERVER_CONTAINER='' + VERBOSE=false + VERBOSITY_LEVEL=1 + CLIENT_DEBUG=false + PARALLEL=false + CAPTURE_KIND='' + CAPTURE_TARGET='' + CAPTURE_DIRECTORY='' + CAPTURE_SCRIPT_CONTENT='' + CAPTURE_OVERRIDE_CONTENT='' + declare -gA CAPTURE_ENV=() +} + +run_playbook() { + local target=$1 map_name=$2 key + local -n source_map=$map_name + CAPTURE_KIND=playbook + CAPTURE_TARGET=$target + CAPTURE_ENV=() + for key in "${!source_map[@]}"; do + CAPTURE_ENV["$key"]=${source_map[$key]} + done +} + +run_setup_script() { + local directory=$1 target=$2 map_name=$3 key + local -n source_map=$map_name + CAPTURE_KIND=script + CAPTURE_DIRECTORY=$directory + CAPTURE_TARGET=$target + if [[ $target == /* && -f $target ]]; then + CAPTURE_SCRIPT_CONTENT=$(<"$target") + if [[ $CAPTURE_SCRIPT_CONTENT =~ -f[[:space:]]+([^[:space:]]+/compose.yml) ]] && + [[ -f ${BASH_REMATCH[1]} ]]; then + CAPTURE_OVERRIDE_CONTENT=$(<"${BASH_REMATCH[1]}") + fi + fi + CAPTURE_ENV=() + for key in "${!source_map[@]}"; do + CAPTURE_ENV["$key"]=${source_map[$key]} + done +} + +setup() { + reset_framework_state +} diff --git a/qa-integration/pmm_qa/pmm-framework/tests/integration.bats b/qa-integration/pmm_qa/pmm-framework/tests/integration.bats new file mode 100644 index 000000000..371b2ef31 --- /dev/null +++ b/qa-integration/pmm_qa/pmm-framework/tests/integration.bats @@ -0,0 +1,286 @@ +#!/usr/bin/env bats + +setup() { + FRAMEWORK_DIR=$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd) + TEST_BIN="$BATS_TEST_TMPDIR/bin" + RECORD_FILE="$BATS_TEST_TMPDIR/calls.log" + mkdir -p "$TEST_BIN" + + cat >"$TEST_BIN/docker" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + cat >"$TEST_BIN/ansible-galaxy" <<'EOF' +#!/usr/bin/env bash +exit 0 +EOF + cat >"$TEST_BIN/ansible-playbook" <<'EOF' +#!/usr/bin/env bash +{ + echo '--- call ---' + printf 'args=' + printf '%q ' "$@" + echo + env | grep -E '^(PS_.*|PGSQL_.*|SETUP_TYPE|QUERY_SOURCE|CLIENT_VERSION|PMM_SERVER_IP|ADMIN_PASSWORD)=' | sort +} >>"$RECORD_FILE" +if [[ ${PARALLEL_TEST:-false} == true ]]; then + if [[ -n ${PS_VERSION:-} ]]; then + sleep 1 + echo 'PS parallel log' + elif [[ -n ${PGSQL_VERSION:-} ]]; then + echo 'PGSQL parallel log' + fi +fi +if [[ ${FAIL_PS:-false} == true && -n ${PS_VERSION:-} ]]; then + echo 'PS failed as requested' + exit 9 +fi +EOF + cat >"$TEST_BIN/curl" <<'EOF' +#!/usr/bin/env bash +printf '\n' +EOF + chmod +x "$TEST_BIN"/* +} + +@test "entrypoint dispatches multiple databases in order through ansible-playbook" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --pmm-server-ip 10.0.0.5 \ + --pmm-server-password secret \ + --client-version latest-tarball \ + --database ps=8.4,SETUP_TYPE=gr,QUERY_SOURCE=slowlog \ + --database pgsql=16 + + [[ $status -eq 0 ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] + + first_call=$(awk '/--- call ---/{n++} n==1{print}' "$RECORD_FILE") + second_call=$(awk '/--- call ---/{n++} n==2{print}' "$RECORD_FILE") + [[ $first_call == *'percona_server_for_mysql/percona-server-setup.yml'* ]] + [[ $first_call == *'PS_VERSION=8.4'* ]] + [[ $first_call == *'SETUP_TYPE=gr'* ]] + [[ $first_call == *'QUERY_SOURCE=slowlog'* ]] + [[ $first_call == *'PMM_SERVER_IP=10.0.0.5'* ]] + [[ $second_call == *'pgsql_pgss_setup.yml'* ]] + [[ $second_call == *'PGSQL_VERSION=16'* ]] + [[ $second_call != *'PS_VERSION='* ]] +} + +@test "entrypoint reports invalid database without calling backends" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --pmm-server-ip 10.0.0.5 \ + --database no-such-db + + [[ $status -ne 0 ]] + [[ $output == *"Database type 'no-such-db' is not recognized"* ]] + [[ ! -f $RECORD_FILE ]] +} + +@test "parallel mode reports successes as they finish without dumping logs" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + PARALLEL_TEST=true \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 + + [[ $status -eq 0 ]] + [[ $output == *'Starting [1/2] ps=8.4'* ]] + [[ $output == *'Starting [2/2] pgsql=16'* ]] + [[ $output == *'[1/2] ps=8.4: OK (log:'* ]] + [[ $output == *'[2/2] pgsql=16: OK (log:'* ]] + [[ $output != *'PS parallel log'* ]] + [[ $output != *'PGSQL parallel log'* ]] + + # pgsql has no artificial delay, so it should finish before sleeping ps. + pgsql_ok_line=$(printf '%s\n' "$output" | awk '/\[2\/2\] pgsql=16: OK/{print NR; exit}') + ps_ok_line=$(printf '%s\n' "$output" | awk '/\[1\/2\] ps=8\.4: OK/{print NR; exit}') + [[ $pgsql_ok_line -lt $ps_ok_line ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "parallel mode waits for all setups and dumps only failed logs" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + FAIL_PS=true \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 + + [[ $status -ne 0 ]] + [[ $output == *'===== [1/2] ps=8.4 FAILED (exit=1) ====='* ]] + [[ $output == *'PS failed as requested'* ]] + [[ $output == *'[2/2] pgsql=16: OK (log:'* ]] + [[ $output == *'Parallel setup logs kept at:'* ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "parallel mode job control emits no job-status noise" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 + + [[ $status -eq 0 ]] + # `set -m` in run_parallel_setups must not leak "[1]+ Done ..." lines. + if grep -qE '^\[[0-9]+\][-+]?[[:space:]]' <<<"$output"; then + echo "job-control notifications leaked into parallel output" + return 1 + fi +} + +@test "parallel setups run with stdin detached" { + # Under job control a background setup that reads the terminal is stopped by + # SIGTTIN and never finishes, so each job must get /dev/null on stdin. + # Successful parallel runs no longer dump setup stdout, so record the probe + # result outside the buffered console log. + cat >"$TEST_BIN/ansible-playbook" <<'EOF' +#!/usr/bin/env bash +if read -r line; then + echo "STDIN_READABLE:$line" >>"$RECORD_FILE" +else + echo "STDIN_EOF" >>"$RECORD_FILE" +fi +EOF + chmod +x "$TEST_BIN/ansible-playbook" + + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 <<<'framework-stdin-payload' + + [[ $status -eq 0 ]] + [[ $(grep -c 'STDIN_EOF' "$RECORD_FILE") -eq 2 ]] + if grep -q 'STDIN_READABLE' "$RECORD_FILE"; then + echo 'background setup unexpectedly read framework stdin' + return 1 + fi +} + +@test "parallel mode falls back to sequential for PS and MySQL" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database mysql=8.4 + + # Both setups must still run; only their concurrency is given up. + [[ $status -eq 0 ]] + [[ $output == *'Running setups sequentially'* ]] + [[ $output == *'shared mysql_cluster_data and host ports'* ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "parallel mode falls back to sequential for duplicate database types" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database ps,SETUP_TYPE=replication \ + --database ps,SETUP_TYPE=gr + + [[ $status -eq 0 ]] + [[ $output == *'Running setups sequentially'* ]] + [[ $output == *'two PS setups'* ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "parallel mode falls back to sequential for PDPGSQL and PGSQL replication" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database pdpgsql \ + --database pgsql,SETUP_TYPE=replication + + # Both setups must still run; only their concurrency is given up. + [[ $status -eq 0 ]] + [[ $output == *'Running setups sequentially'* ]] + [[ $output == *'shared pgsql_cluster_data and host port 6432'* ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "parallel mode stays parallel for PDPGSQL and non-replication PGSQL" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + PARALLEL_TEST=true \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --pmm-server-ip 10.0.0.5 \ + --database pdpgsql \ + --database pgsql + + # No shared data_dir or port when PGSQL doesn't use replication, so the + # framework must not give up concurrency for this pair. + [[ $status -eq 0 ]] + [[ $output != *'Running setups sequentially'* ]] + [[ $(grep -c -- '--- call ---' "$RECORD_FILE") -eq 2 ]] +} + +@test "verbose parallel runs echo the logs of successful setups" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + PARALLEL_TEST=true \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --verbose \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 + + [[ $status -eq 0 ]] + [[ $output == *'PS parallel log'* ]] + [[ $output == *'PGSQL parallel log'* ]] + [[ $output == *'setup log ====='* ]] +} + +@test "verbose parallel runs still dump the log of a setup that failed" { + run env \ + PATH="$TEST_BIN:$PATH" \ + RECORD_FILE="$RECORD_FILE" \ + PARALLEL_TEST=true \ + FAIL_PS=true \ + "$FRAMEWORK_DIR/pmm-framework" \ + --parallel \ + --verbose \ + --pmm-server-ip 10.0.0.5 \ + --database ps=8.4 \ + --database pgsql=16 + + [[ $status -ne 0 ]] + # --verbose echoes both, but the failure keeps its own FAILED banner so it is + # still findable among the successful logs. + [[ $output == *'PS failed as requested'* ]] + [[ $output == *'FAILED (exit=1)'* ]] + [[ $output == *'PGSQL parallel log'* ]] + [[ $output == *'Parallel setup logs kept at:'* ]] +} diff --git a/qa-integration/pmm_qa/postgresql/postgresql-setup.yml b/qa-integration/pmm_qa/postgresql/postgresql-setup.yml index ef28180d1..a04456395 100644 --- a/qa-integration/pmm_qa/postgresql/postgresql-setup.yml +++ b/qa-integration/pmm_qa/postgresql/postgresql-setup.yml @@ -30,7 +30,12 @@ ignore_errors: yes - name: Remove old data folders - shell: 'rm -fr {{ data_dir }}' + shell: | + if [ -d "{{ data_dir }}" ]; then + docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \ + sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*' + rm -rf "{{ data_dir }}" + fi - name: Create data directories file: @@ -65,7 +70,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node1/data:/var/lib/postgresql/data" - - "./data/postgresql-primary.conf.j2:/etc/postgresql/postgresql.conf:ro" + - "./data/postgresql-primary.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" command: -c config_file=/etc/postgresql/postgresql.conf ports: @@ -107,7 +112,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgresql-replica.conf:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" loop: "{{ range(2, nodes_count | int + 1) | list }}" @@ -173,7 +178,7 @@ POSTGRES_PASSWORD: "{{ root_password }}" volumes: - "{{ data_dir }}/node{{ item }}/data:/var/lib/postgresql/data" - - "./data/postgresql-replica.conf.j2:/etc/postgresql/postgresql.conf:ro" + - "./data/postgres-replica.conf:/etc/postgresql/postgresql.conf:ro" - "{{ data_dir }}/node1/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro" ports: - "{{ pgsql_port + item - 1 }}:5432" diff --git a/qa-integration/pmm_qa/ps_pmm_setup.yml b/qa-integration/pmm_qa/ps_pmm_setup.yml index 2e28ac779..17ffad437 100644 --- a/qa-integration/pmm_qa/ps_pmm_setup.yml +++ b/qa-integration/pmm_qa/ps_pmm_setup.yml @@ -3,8 +3,6 @@ # enables Percona testing repository - hosts: all - become: true - become_method: sudo vars: ps_version: "{{ lookup('vars', 'extra_ps_version', default=lookup('env','PS_VERSION') | default('8.0', true) ) }}" ps_tarball: "{{ lookup('vars', 'extra_ps_tarball', default=lookup('env','PS_TARBALL') | default('', true) ) }}" diff --git a/qa-integration/pmm_qa/psmdb_setup.yml b/qa-integration/pmm_qa/psmdb_setup.yml index c27e60c57..5cff44978 100644 --- a/qa-integration/pmm_qa/psmdb_setup.yml +++ b/qa-integration/pmm_qa/psmdb_setup.yml @@ -1,8 +1,6 @@ --- - hosts: all - become: true - become_method: sudo vars: psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}" psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}" diff --git a/qa-integration/pmm_qa/pxc_proxysql_setup.yml b/qa-integration/pmm_qa/pxc_proxysql_setup.yml index 83dd168bd..7e5b1896f 100644 --- a/qa-integration/pmm_qa/pxc_proxysql_setup.yml +++ b/qa-integration/pmm_qa/pxc_proxysql_setup.yml @@ -2,8 +2,6 @@ # enables Percona testing repository - hosts: all - become: true - become_method: sudo vars: pxc_version: "{{ lookup('vars', 'extra_pxc_version', default=lookup('env','PXC_VERSION') | default('8.0', true) ) }}" pxc_tarball: "{{ lookup('vars', 'extra_pxc_tarball', default=lookup('env','PXC_TARBALL') | default('', true) ) }}" @@ -102,7 +100,10 @@ shell: "{{ item }}" with_items: - docker exec {{ pxc_container }} chmod 1777 /tmp - - docker exec --user pxc {{ pxc_container }} bash -xe ./client_container_pxc_setup.sh --pxc_version {{ pxc_version }} --pxc_tarball {{ pxc_tarball }} --number_of_nodes {{ number_of_nodes }} --pxc_dev_cluster {{ pxc_dev_cluster }} --query_source {{query_source}} > setup_pxc_{{ pxc_version }}.log + # Under Docker Desktop's Rosetta emulation /proc//exe points to + # /run/rosetta/rosetta, so the PXC SST script needs a PATH fallback. + - docker exec {{ pxc_container }} ln -sf /home/pxc/PXC/bin/mysqld /usr/local/bin/mysqld + - docker exec --user pxc {{ pxc_container }} bash -xe ./client_container_pxc_setup.sh --pxc_version {{ pxc_version }} --pxc_tarball {{ pxc_tarball }} --number_of_nodes {{ number_of_nodes }} --pxc_dev_cluster {{ pxc_dev_cluster }} --query_source {{query_source}} - name: Set Random Number Fact set_fact: @@ -121,9 +122,18 @@ - docker exec {{ pxc_container }} mysql -h 127.0.0.1 -P 6032 -u admin -padmin -e "LOAD ADMIN VARIABLES TO RUNTIME;" - docker exec {{ pxc_container }} mysql -h 127.0.0.1 -P 6032 -u admin -padmin -e "SAVE ADMIN VARIABLES TO DISK;" - docker exec {{ pxc_container }} pmm-admin add proxysql --username=admin --password=admin --service-name=my-new-proxysql_{{ pxc_container }}_{{ random_number }} --host=127.0.0.1 --port=6032 - + register: proxysql_start + # The proxysql2 package's postinst can auto-start the daemon during + # install (observed non-deterministically), so the explicit `proxysql -c + # ...` start above sometimes finds it already running -- that is the + # desired end state, not a failure. Any other non-zero exit in this loop + # still fails the play. + failed_when: >- + proxysql_start.rc != 0 and + 'Daemon already running' not in (proxysql_start.stderr | default('')) + - name: Execute Service, Sysbench Script inside the PXC pxc_container shell: "{{ item }}" with_items: - - docker exec {{ pxc_container }} bash -xe ./client_container_proxysql_setup.sh > setup_proxy_{{ pxc_version }}.log + - docker exec {{ pxc_container }} bash -xe ./client_container_proxysql_setup.sh diff --git a/qa-integration/pmm_qa/tasks/install_pmm_client.yml b/qa-integration/pmm_qa/tasks/install_pmm_client.yml index 83f51a570..2f6028873 100644 --- a/qa-integration/pmm_qa/tasks/install_pmm_client.yml +++ b/qa-integration/pmm_qa/tasks/install_pmm_client.yml @@ -274,9 +274,13 @@ - metrics_mode | length > 0 - use_encrypted_client_config | default(false) | bool -- name: Wait 5 seconds for connection to complete - pause: - seconds: 5 +- name: Wait until pmm-agent config is written + shell: docker exec --user root {{ container_name }} test -f /usr/local/percona/pmm/config/pmm-agent.yaml + register: pmm_agent_config_ready + retries: 15 + delay: 1 + until: pmm_agent_config_ready.rc == 0 + changed_when: false - name: Start pmm client without encrypted client config shell: | @@ -290,6 +294,14 @@ sh -c 'nohup pmm-agent --config-file=/usr/local/percona/pmm/config/pmm-agent.yaml --config-file-key-file="/usr/local/percona/pmm/config/pmm-key.pem" --config-file-key-password="testpass" > /var/log/pmm-agent.log 2>&1 &' when: use_encrypted_client_config | default(false) | bool -- name: Wait 5 seconds for start to complete - pause: - seconds: 5 +- name: Wait until pmm-agent is running and connected + shell: | + docker exec --user root {{ container_name }} sh -c ' + pmm-admin status 2>/dev/null | + grep -Eq "Connected[[:space:]]*: true" + ' + register: pmm_agent_ready + retries: 30 + delay: 1 + until: pmm_agent_ready.rc == 0 + changed_when: false diff --git a/qa-integration/pmm_qa/tasks/set_unique_service_name.yml b/qa-integration/pmm_qa/tasks/set_unique_service_name.yml index e2492dff5..5f34d42c6 100644 --- a/qa-integration/pmm_qa/tasks/set_unique_service_name.yml +++ b/qa-integration/pmm_qa/tasks/set_unique_service_name.yml @@ -1,17 +1,15 @@ -- name: Install jq via appropriate package manager - become: true - block: - - name: Install jq on Debian/Ubuntu - apt: - name: jq - state: present - when: ansible_facts['os_family'] == 'Debian' +- name: Ensure jq is available without host sudo + command: jq --version + changed_when: false + register: jq_check + failed_when: false - - name: Install jq on RHEL/CentOS/Alma/Rocky - dnf: - name: jq - state: present - when: ansible_facts['os_family'] == 'RedHat' +- name: Fail when jq is missing + fail: + msg: >- + jq is required for unique service-name checks. Install it on the host + (for example brew install jq / apt install jq) without relying on Ansible sudo. + when: jq_check.rc != 0 - name: Verify that service with expected name is not connected to pmm server shell: | diff --git a/qa-integration/pmm_qa/tasks/wait_mysql_ready.yml b/qa-integration/pmm_qa/tasks/wait_mysql_ready.yml new file mode 100644 index 000000000..89f485245 --- /dev/null +++ b/qa-integration/pmm_qa/tasks/wait_mysql_ready.yml @@ -0,0 +1,21 @@ +# Wait until mysqld accepts connections inside a container. +# Expects: container_name +# Optional: mysql_user (default root), mysql_password (omit for socket/default auth) +- name: Wait until MySQL is ready in {{ container_name }} + vars: + _mysql_user: "{{ mysql_user | default('root') }}" + _mysql_password: "{{ mysql_password | default('') }}" + shell: | + if [ -n "{{ _mysql_password }}" ]; then + docker exec {{ container_name }} mysqladmin ping \ + -u{{ _mysql_user }} \ + -p{{ _mysql_password }} \ + --silent + else + docker exec {{ container_name }} mysqladmin ping --silent + fi + register: mysql_ready + retries: "{{ mysql_ready_retries | default(30) }}" + delay: "{{ mysql_ready_delay | default(1) }}" + until: mysql_ready.rc == 0 + changed_when: false diff --git a/qa-integration/pmm_qa/tasks/wait_pgsql_ready.yml b/qa-integration/pmm_qa/tasks/wait_pgsql_ready.yml new file mode 100644 index 000000000..28fc54556 --- /dev/null +++ b/qa-integration/pmm_qa/tasks/wait_pgsql_ready.yml @@ -0,0 +1,10 @@ +# Wait until PostgreSQL accepts connections inside a container. +# Expects: container_name +# Optional: pgsql_ready_retries (default 30), pgsql_ready_delay (default 1) +- name: Wait until PostgreSQL is ready in {{ container_name }} + shell: docker exec -u postgres {{ container_name }} pg_isready -q + register: pgsql_ready + retries: "{{ pgsql_ready_retries | default(30) }}" + delay: "{{ pgsql_ready_delay | default(1) }}" + until: pgsql_ready.rc == 0 + changed_when: false diff --git a/qa-integration/pmm_qa/tasks/wait_pmm_agent_ready.yml b/qa-integration/pmm_qa/tasks/wait_pmm_agent_ready.yml new file mode 100644 index 000000000..9a3f3693e --- /dev/null +++ b/qa-integration/pmm_qa/tasks/wait_pmm_agent_ready.yml @@ -0,0 +1,13 @@ +# Wait until pmm-agent is running and talking to PMM Server. +# Expects: container_name +- name: Wait until pmm-agent is ready in {{ container_name }} + shell: | + docker exec --user root {{ container_name }} sh -c ' + pmm-admin status 2>/dev/null | + grep -Eq "Connected[[:space:]]*: true" + ' + register: pmm_agent_ready + retries: "{{ pmm_agent_ready_retries | default(30) }}" + delay: "{{ pmm_agent_ready_delay | default(1) }}" + until: pmm_agent_ready.rc == 0 + changed_when: false diff --git a/qa-integration/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml b/qa-integration/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml index f47d8852b..95f918d71 100644 --- a/qa-integration/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml +++ b/qa-integration/pmm_qa/tls-ssl-setup/mlaunch_tls_setup.yml @@ -1,8 +1,6 @@ --- - hosts: all - become: true - become_method: sudo vars: mongodb_version: "{{ lookup('vars', 'extra_mongodb_version', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" mongodb_ssl_container: "{{ lookup('vars', 'extra_mongodb_ssl_container', default=lookup('env','MONGODB_SSL_CONTAINER') | default('mongodb_ssl', true) ) }}" diff --git a/qa-integration/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml b/qa-integration/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml index c2cb9f548..554d7d160 100644 --- a/qa-integration/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml +++ b/qa-integration/pmm_qa/tls-ssl-setup/mongodb_tls_setup.yml @@ -1,8 +1,6 @@ --- - hosts: all - become: true - become_method: sudo vars: mongodb_version: "{{ lookup('vars', 'extra_mongodb_version', default=lookup('env','MONGODB_VERSION') | default('4.4', true) ) }}" mongodb_ssl_container: "{{ lookup('vars', 'extra_mongodb_ssl_container', default=lookup('env','MONGODB_SSL_CONTAINER') | default('mongodb_ssl', true) ) }}" diff --git a/qa-integration/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml b/qa-integration/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml index e1e07b215..0fda7c8e7 100644 --- a/qa-integration/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml +++ b/qa-integration/pmm_qa/tls-ssl-setup/mysql_tls_setup.yml @@ -5,8 +5,6 @@ # Install all required tools for backups in compatible version - hosts: all - become: true - become_method: sudo vars: mysql_version: "{{ lookup('vars', 'extra_mysql_version', default=lookup('env','MYSQL_VERSION') | default('8.0', true) ) }}" mysql_ssl_container: "{{ lookup('vars', 'extra_mysql_ssl_container', default=lookup('env','MYSQL_SSL_CONTAINER') | default('mysql_ssl', true) ) }}" diff --git a/qa-integration/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml b/qa-integration/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml index ff6bbc3f6..483099417 100644 --- a/qa-integration/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml +++ b/qa-integration/pmm_qa/tls-ssl-setup/postgresql_tls_setup.yml @@ -5,8 +5,6 @@ # Install all required tools for backups in compatible version - hosts: all - become: true - become_method: sudo vars: pgsql_version: "{{ lookup('vars', 'extra_pgsql_version', default=lookup('env','PGSQL_VERSION') | default('13', true) ) }}" pgsql_ssl_container: "{{ lookup('vars', 'extra_pgsql_ssl_container', default=lookup('env','PGSQL_SSL_CONTAINER') | default('pgsql_ssl', true) ) }}" diff --git a/qa-integration/pmm_qa/vars/pinned_images.yml b/qa-integration/pmm_qa/vars/pinned_images.yml new file mode 100644 index 000000000..85a0a81ad --- /dev/null +++ b/qa-integration/pmm_qa/vars/pinned_images.yml @@ -0,0 +1,5 @@ +--- +# Third-party image versions shared by setup playbooks. Bump the version here +# rather than editing each playbook individually -- loaded into every +# playbook run by run_playbook() in pmm-framework/lib/ansible.sh. +busybox_image: busybox:1.37.0