PMM-15265 pmm framework migration

.github/workflows/PMM_PDPGSQL.yaml

@@ -66,12 +66,7 @@ jobs:
         working-directory: qa-integration/pmm_qa
         run: |
           mkdir -m 777 -p /tmp/backup_data
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install --force-reinstall -U setuptools
-          pip install -r requirements.txt
-          python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }}
+          ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PDPGSQL=${{ env.PDPGSQL_VERSION }}
 
       - name: Setup npm modules for CodeceptJS
         working-directory: codeceptjs-e2e

.github/workflows/PMM_PROXYSQL.yaml

@@ -99,15 +99,10 @@ jobs:
         working-directory: qa-integration/pmm_qa
         run: |
           mkdir -m 777 -p /tmp/backup_data
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install --force-reinstall -U setuptools
-          pip install -r requirements.txt
           if [[ "${PXC_VERSION}" != http* ]]; then
-            python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${PXC_VERSION}
+            ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC=${PXC_VERSION}
           else
-            python pmm-framework.py --verbose --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC,TARBALL=${PXC_TARBALL_PATH}
+            ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} --client-version=${{ env.PMM_CLIENT_VERSION }} --database PXC,TARBALL=${PXC_TARBALL_PATH}
           fi
 
       - name: Install npx dependencies

.github/workflows/gssapi-psmdb-tests-matrix.yml

@@ -185,7 +185,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose'
+      services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=8'
       test_name: 'PSMDB Replica 6.x'
 
   psmdb_7_replica-tests_ol8:
@@ -199,7 +199,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose'
+      services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=8'
       test_name: 'PSMDB Replica 7.x'
 
   psmdb_8_replica-tests_ol8:
@@ -213,7 +213,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol8 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol8-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=8 --verbose'
+      services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=8'
       test_name: 'PSMDB Replica 8.x'
 
   psmdb_6_replica-tests_ol9:
@@ -227,7 +227,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose'
+      services_list: '--database psmdb=6.0,SETUP_TYPE=pss,OL_VERSION=9'
       test_name: 'PSMDB Replica 6.x'
 
   psmdb_7_replica-tests_ol9:
@@ -241,7 +241,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose'
+      services_list: '--database psmdb=7.0,SETUP_TYPE=pss,OL_VERSION=9'
       test_name: 'PSMDB Replica 7.x'
 
   psmdb_8_replica-tests_ol9:
@@ -255,7 +255,7 @@ jobs:
       pmm_client_image: ${{ inputs.pmm_client_image || 'perconalab/pmm-client:3-dev-latest' }}
       pmm_client_version: ${{ inputs.pmm_client_version_ol9 || 'https://s3.us-east-2.amazonaws.com/pmm-build-cache/PR-BUILDS/pmm-client/pmm-client-dynamic-ol9-latest.tar.gz' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=9 --verbose'
+      services_list: '--database psmdb=8.0,SETUP_TYPE=pss,OL_VERSION=9'
       test_name: 'PSMDB Replica 8.x'
 
   mongo_package_tests_integration_ol8:

.github/workflows/integration-cli-tests.yml

@@ -379,7 +379,7 @@ jobs:
       pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }}
       launchable_confidence: ${{ inputs.launchable_confidence || '100%' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=6.0,SETUP_TYPE=pss --verbose'
+      services_list: '--database psmdb=6.0,SETUP_TYPE=pss'
       test_name: 'PSMDB Replica 6.x'
 
   psmdb-7-replica-tests:
@@ -395,7 +395,7 @@ jobs:
       pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }}
       launchable_confidence: ${{ inputs.launchable_confidence || '100%' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=7.0,SETUP_TYPE=pss --verbose'
+      services_list: '--database psmdb=7.0,SETUP_TYPE=pss'
       test_name: 'PSMDB Replica 7.x'
 
   psmdb-8-tests:
@@ -411,7 +411,7 @@ jobs:
       pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }}
       launchable_confidence: ${{ inputs.launchable_confidence || '100%' }}
       cli_tag: '@psmdb'
-      services_list: '--database psmdb=8.0,SETUP_TYPE=pss --verbose'
+      services_list: '--database psmdb=8.0,SETUP_TYPE=pss'
       test_name: 'PSMDB Replica 8.x'
 
   psmdb-7-shard-tests:
@@ -427,7 +427,7 @@ jobs:
       pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }}
       launchable_confidence: ${{ inputs.launchable_confidence || '100%' }}
       cli_tag: '@shard-psmdb'
-      services_list: '--database psmdb=7.0,SETUP_TYPE=sharding --verbose'
+      services_list: '--database psmdb=7.0,SETUP_TYPE=sharding'
       test_name: 'PSMDB Shard 7.x'
 
   psmdb-8-shard-tests:
@@ -443,7 +443,7 @@ jobs:
       pmm_client_version: ${{ inputs.pmm_client_version || 'latest-tarball' }}
       launchable_confidence: ${{ inputs.launchable_confidence || '100%' }}
       cli_tag: '@shard-psmdb'
-      services_list: '--database psmdb=8.0,SETUP_TYPE=sharding --verbose'
+      services_list: '--database psmdb=8.0,SETUP_TYPE=sharding'
       test_name: 'PSMDB Shard 8.x'
 
   haproxy-tests:

.github/workflows/runner-e2e-tests-codeceptjs-remote-nightly-setup.yml

@@ -127,16 +127,11 @@ jobs:
           command: |
             cd pmm-qa/qa-integration/pmm_qa
             mkdir -m 777 -p /tmp/backup_data || true
-            python3 -m venv virtenv
-            . virtenv/bin/activate
-            pip install --upgrade pip
-            pip install -r requirements.txt
-            pip install setuptools
-            python pmm-framework.py \
+            ./pmm-framework/pmm-framework \
+              --parallel \
               --pmm-server-ip "${SERVER_IP}" \
               --pmm-server-password "${{ env.ADMIN_PASSWORD }}" \
               --client-version "${{ env.PMM_CLIENT_VERSION }}" \
-              --verbose \
               ${{ env.WIZARD_ARGS }}
           on_retry_command: |
             docker rm -f $(docker ps -a -q) || true

.github/workflows/runner-e2e-tests-codeceptjs.yml

@@ -221,12 +221,7 @@ jobs:
         working-directory: pmm-qa/qa-integration/pmm_qa
         run: |
           mkdir -m 777 -p /tmp/backup_data
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install setuptools
-          python pmm-framework.py --pmm-server-password=${{ env.ADMIN_PASSWORD }} --verbose ${{ env.WIZARD_ARGS }}
+          ./pmm-framework/pmm-framework --parallel --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }}
 
       - name: Execute e2e tests with tags ${{ env.TAGS_FOR_TESTS }} with launchable
         if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }}

.github/workflows/runner-e2e-tests-playwright.yml

@@ -193,12 +193,7 @@ jobs:
         if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }}
         working-directory: ./pmm-qa/qa-integration/pmm_qa
         run: |
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install setuptools
-          python pmm-framework.py --verbosity-level=2 --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }}
+          ./pmm-framework/pmm-framework --parallel --verbosity-level=2 --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }}
 
       - name: 'Run UI tests: ${{ env.PMM_TEST_FLAG }} with launchable'
         if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }}

.github/workflows/runner-e2e-tests-podman.yml

@@ -222,12 +222,7 @@ jobs:
         working-directory: pmm-qa/qa-integration/pmm_qa
         run: |
           export HOST_IP=$(ip -4 addr show docker0 | awk '/inet / {print $2}' | cut -d/ -f1)
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install setuptools
-          python pmm-framework.py --pmm-server-ip=$HOST_IP --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }}
+          ./pmm-framework/pmm-framework --parallel --pmm-server-ip=$HOST_IP --pmm-server-password=${{ env.ADMIN_PASSWORD }} ${{ env.WIZARD_ARGS }}
 
       - name: Setup npm modules for e2e tests
         working-directory: ./pmm-qa/codeceptjs-e2e

.github/workflows/runner-integration-cli-tests.yml

@@ -21,7 +21,7 @@ on:
         required: true
         type: string
       services_list:
-        description: "pmm-framework.py flags to perform setup"
+        description: "pmm-framework flags to perform setup"
         required: false
         type: string
       cli_tag:
@@ -189,12 +189,7 @@ jobs:
         if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }}
         working-directory: pmm-qa/qa-integration/pmm_qa
         run: |
-          python3 -m venv virtenv
-          . virtenv/bin/activate
-          pip install --upgrade pip
-          pip install setuptools
-          pip install -r requirements.txt
-          python pmm-framework.py --client-debug ${{ env.WIZARD_ARGS }}
+          ./pmm-framework/pmm-framework --parallel --client-debug ${{ env.WIZARD_ARGS }}
 
       - name: 'Run CLI tests: ${{ inputs.cli_tag }}'
         if: ${{ steps.check_launchable_subset.outputs.has_subset == 'true' }}

qa-integration/pmm_qa/client_container_pxc_setup.sh

@@ -53,6 +53,12 @@ tar -xzf Percona-XtraDB-Cluster.tar.gz
 sleep 10
 rm -r Percona-XtraDB-Cluster.tar.gz
 mv Percona-XtraDB-Cluster* PXC
+
+# Docker Desktop's Rosetta emulation exposes /proc/<pid>/exe as an
+# unresolvable /run/rosetta/rosetta link. Do not let readlink's failure
+# terminate the SST helper before it can fall back to mysqld from PATH.
+sed -i 's#MYSQLD_PATH=$(readlink -f /proc/${WSREP_SST_OPT_PARENT}/exe)#MYSQLD_PATH=$(readlink -f /proc/${WSREP_SST_OPT_PARENT}/exe || true)#' PXC/bin/wsrep_sst_common
+
 cd PXC
 
 ## start PXC

qa-integration/pmm_qa/haproxy_setup.yml

@@ -44,7 +44,7 @@
   - name: Setup haproxy for monitoring
     shell: "{{ item }}"
     with_items:
-      - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh > setup_haproxy.log
+      - docker exec {{ haproxy_container }} bash -x ./haproxy_setup.sh
 
   - name: Install pmm3-client on the haproxy_container
     shell: "{{ item }}"

qa-integration/pmm_qa/mlaunch_modb_setup.yml

@@ -1,8 +1,6 @@
 ---
 
 - hosts: all
-  become: true
-  become_method: sudo
   vars:
     modb_version: "{{ lookup('vars', 'extra_modb_version', default=lookup('env','MODB_VERSION') | default('4.4', true) ) }}"
     modb_tarball: "{{ lookup('vars', 'extra_modb_tarball', default=lookup('env','MODB_TARBALL') | default('', true) ) }}"
@@ -82,4 +80,4 @@
     shell: "{{ item }}"
     with_items:
       - docker run --name mongodb_load_{{ modb_version }}_{{ modb_setup }} -d -e MONGODB_HOST={{ modb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_modb_{{ modb_version }}_{{ modb_setup }}.log
-    when: modb_setup == "sharded"
+    when: modb_setup == "sharded"

qa-integration/pmm_qa/mlaunch_psmdb_setup.yml

@@ -1,8 +1,6 @@
 ---
 
 - hosts: all
-  become: true
-  become_method: sudo
   vars:
     psmdb_version: "{{ lookup('vars', 'extra_psmdb_version', default=lookup('env','PSMDB_VERSION') | default('4.4', true) ) }}"
     psmdb_tarball: "{{ lookup('vars', 'extra_psmdb_tarball', default=lookup('env','PSMDB_TARBALL') | default('', true) ) }}"
@@ -82,4 +80,4 @@
     shell: "{{ item }}"
     with_items:
       - docker run --name mongodb_load_{{ psmdb_version }}_{{ psmdb_setup }} -d -e MONGODB_HOST={{ psmdb_container }} -e MONGODB_PORT=27017 -e TEST_TARGET_QPS=10 -e TEST_COLLECTION=10 -e TEST_DB=30 --network=pmm-qa -v $(pwd):/usr/src/myapp -w /usr/src/myapp php-db php mongodb_query.php >> setup_psmdb_{{ psmdb_version }}_{{ psmdb_setup }}.log
-    when: psmdb_setup == "sharded"
+    when: psmdb_setup == "sharded"

qa-integration/pmm_qa/ms_pmm_setup.yml

@@ -3,8 +3,6 @@
 #   enables Percona testing repository
 
 - hosts: all
-  become: true
-  become_method: sudo
   vars:
     ms_version: "{{ lookup('vars', 'extra_ms_version', default=lookup('env','MS_VERSION') | default('8.0', true) ) }}"
     ms_tarball: "{{ lookup('vars', 'extra_ms_tarball', default=lookup('env','MS_TARBALL') | default('', true) ) }}"

qa-integration/pmm_qa/mysql/mysql-setup.yml

@@ -43,8 +43,12 @@
       ignore_errors: true
 
     - name: Remove old data folders
-      shell: 'rm -fr {{ data_dir }}'
-      loop: "{{ range(1, nodes_count | int + 1) | list }}"
+      shell: |
+        if [ -d "{{ data_dir }}" ]; then
+          docker run --rm -v "{{ data_dir }}:/data" {{ busybox_image }} \
+            sh -c 'rm -rf /data/* /data/.[!.]* /data/..?*'
+          rm -rf "{{ data_dir }}"
+        fi
 
     - name: Create data directories
       file:
@@ -54,7 +58,10 @@
       loop: "{{ range(1, nodes_count | int + 1) | list }}"
 
     - name: Recursively change ownership of a directory
-      shell: "sudo chown -R 1001:1001 {{ data_dir }}/node{{ item }}/data"
+      shell: |
+        docker run --rm \
+          -v "{{ data_dir }}/node{{ item }}/data:/data" \
+          {{ busybox_image }} chown -R 1001:1001 /data
       loop: "{{ range(1, nodes_count | int + 1) | list }}"
 
     - name: Setup MySQL group replication

qa-integration/pmm_qa/mysql/tasks/prepare_install_mysql.yml

@@ -5,9 +5,11 @@
 
 - name: Prepare socket folders
   shell: |
-    sudo rm -fr /tmp/mysql-sockets/{{ item }}
-    sudo mkdir -p /tmp/mysql-sockets/{{ item }}
-    sudo chmod 0777 /tmp/mysql-sockets/{{ item }}
+    docker run --rm -v /tmp:/host-tmp {{ busybox_image }} sh -c '
+      rm -rf /host-tmp/mysql-sockets/{{ item }}
+      mkdir -p /host-tmp/mysql-sockets/{{ item }}
+      chmod 0777 /host-tmp/mysql-sockets/{{ item }}
+    '
 
 - name: Prepare Container for MySQL 8.0+
   shell: |

qa-integration/pmm_qa/pdpgsql_pgsm_setup.yml

@@ -3,8 +3,6 @@
 #   enables Percona testing repository
 
 - hosts: all
-  become: true
-  become_method: sudo
   vars:
     pdpgsql_version: "{{ lookup('vars', 'extra_pdpgsql_version', default=lookup('env','PDPGSQL_VERSION') | default('15', true) ) }}"
     pdpgsql_pgsm_container: "{{ lookup('vars', 'extra_pdpgsql_pgsm_container', default=lookup('env','PDPGSQL_PGSM_CONTAINER') | default('pdpgsql_pgsm', true) ) }}"
@@ -32,7 +30,7 @@
 
   - name: Prepare Container for PostgreSQL
     shell: >
-      sudo docker run -d --name={{ pdpgsql_pgsm_container }}
+      docker run -d --name={{ pdpgsql_pgsm_container }}
       -p {{ pdpgsql_pgsm_port }}:5432
       phusion/baseimage:jammy-1.0.1
 

qa-integration/pmm_qa/percona-distribution-postgresql/percona-distribution-postgres-setup.yml

@@ -34,7 +34,6 @@
     - name: Create Docker network
       shell: docker network create {{ network_name }}
       ignore_errors: yes
-      become: true
 
     - name: Install Percona Distribution for Postgres
       include_tasks: tasks/install-percona-distribution-postgresql-single.yml