[DRAFT] chore: improve system tests

.kokoro/continuous/system.cfg

@@ -1,5 +1,8 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
+# Increase timeout to 6 hours (default is 180 mins / 3 hours)
+timeout_mins: 360
+
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"

.kokoro/presubmit/system.cfg

@@ -1,5 +1,8 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
+# Increase timeout to 6 hours (default is 180 mins / 3 hours)
+timeout_mins: 360
+
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"

.kokoro/system.sh

@@ -128,6 +128,7 @@ run_package_test() {
 reap_parallel_results() {
   local retval=0
   local failed_count=0
+  local timed_out_count=0
   local succeeded_count=0
 
   if [ -z "$LOG_DIR" ]; then
@@ -142,6 +143,13 @@ reap_parallel_results() {
     fi
   done
 
+  # Count timed out packages by checking for .timed_out marker files
+  for timed_out in "$LOG_DIR"/*.timed_out; do
+    if [ -f "$timed_out" ]; then
+      timed_out_count=$((timed_out_count + 1))
+    fi
+  done
+
   local total_tested=${#PACKAGES_TO_TEST[@]}
   succeeded_count=$((total_tested - failed_count))
 
@@ -152,6 +160,9 @@ reap_parallel_results() {
   echo "Total Packages: $total_tested"
   echo "Succeeded:      $succeeded_count"
   echo "Failed:         $failed_count"
+  if [ "$timed_out_count" -gt 0 ]; then
+    echo "Timed Out:      $timed_out_count"
+  fi
   echo "=================================================="
 
   local succeeded_packages=()
@@ -168,14 +179,23 @@ reap_parallel_results() {
     # List failed packages
     for failed in "$LOG_DIR"/*.failed; do
       if [ -f "$failed" ]; then
-        basename "$failed" .failed
+        local pkg=$(basename "$failed" .failed)
+        if [ -f "$LOG_DIR/$pkg.timed_out" ]; then
+          echo "$pkg (TIMED OUT after ${PACKAGE_TEST_TIMEOUT})"
+        else
+          echo "$pkg"
+        fi
       fi
     done
     for failed in "$LOG_DIR"/*.failed; do
       if [ -f "$failed" ]; then
         local pkg=$(basename "$failed" .failed)
         echo "--------------------------------------------------"
-        echo "@PACKAGE (FAILED): $pkg"
+        if [ -f "$LOG_DIR/$pkg.timed_out" ]; then
+          echo "@PACKAGE (TIMED OUT after ${PACKAGE_TEST_TIMEOUT}): $pkg"
+        else
+          echo "@PACKAGE (FAILED): $pkg"
+        fi
         echo "--------------------------------------------------"
         if [ -n "$KOKORO_ARTIFACTS_DIR" ] && [ -f "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log" ]; then
           cat "$KOKORO_ARTIFACTS_DIR/$pkg/sponge_log.log"
@@ -287,6 +307,7 @@ done
 
 # Parallel Execution Logic
 MAX_JOBS=${MAX_JOBS:-4}
+PACKAGE_TEST_TIMEOUT="${PACKAGE_TEST_TIMEOUT:-35m}"
 
 # Temporary directory for clean log segregation
 LOG_DIR=$(mktemp -d -t test-logs-XXXXXX)
@@ -301,9 +322,10 @@ fi
 echo "=================================================="
 echo "Starting parallel test execution for ${#PACKAGES_TO_TEST[@]} packages"
 echo "Concurrency limit: ${MAX_JOBS}"
+echo "Per-package timeout: ${PACKAGE_TEST_TIMEOUT}"
 echo "=================================================="
 
-export LOG_DIR
+export LOG_DIR PACKAGE_TEST_TIMEOUT
 export -f run_package_test
 export system_test_script PROJECT_ROOT KOKORO_GFILE_DIR
 
@@ -323,8 +345,22 @@ printf '%s\n' "${PACKAGES_TO_TEST[@]}" \
         log_file="$LOG_DIR/$pkg.log"
       fi
 
-      # Run test; if it fails, create a .failed file to signal failure to the reaper
-      run_package_test "$pkg" > "$log_file" 2>&1 || touch "$LOG_DIR/$pkg.failed"
+      # Run test with timeout guard; if it fails or times out, mark as failed
+      set +e
+      timeout --kill-after=1m "${PACKAGE_TEST_TIMEOUT}" bash -c '\''run_package_test "$0"'\'' "$pkg" > "$log_file" 2>&1
+      status=$?
+      set -e
+
+      if [ $status -eq 124 ] || [ $status -eq 137 ]; then
+        echo "" >> "$log_file"
+        echo "==================================================" >> "$log_file"
+        echo "ERROR: Package test timed out after ${PACKAGE_TEST_TIMEOUT}!" >> "$log_file"
+        echo "==================================================" >> "$log_file"
+        touch "$LOG_DIR/$pkg.timed_out"
+        touch "$LOG_DIR/$pkg.failed"
+      elif [ $status -ne 0 ]; then
+        touch "$LOG_DIR/$pkg.failed"
+      fi
     '
 
 reap_parallel_results || RETVAL=1

packages/bigquery-magics/CHANGELOG.md

@@ -192,3 +192,5 @@
 
 * Clean up docs and let the nox pass ([#24](https://github.com/googleapis/python-bigquery-magics/issues/24)) ([275712f](https://github.com/googleapis/python-bigquery-magics/commit/275712f4e4b647cda2d253e1f6b7a2fa093ee7c1))
 * Reset the changelog for the new package ([#22](https://github.com/googleapis/python-bigquery-magics/issues/22)) ([f7d9c14](https://github.com/googleapis/python-bigquery-magics/commit/f7d9c1445feac32e468a3e06ca55c9474a1ae548))
+
+<!-- trigger system tests -->

packages/django-google-spanner/CHANGELOG.md

@@ -328,3 +328,5 @@
 
 * minor fixes to README.md ([#448](https://www.github.com/googleapis/python-spanner-django/issues/448)) ([f969000](https://www.github.com/googleapis/python-spanner-django/commit/f9690007603c94f4c99b244a92c639adfd360a8f))
 * move test suite information to CONTRIBUTING.md ([#442](https://www.github.com/googleapis/python-spanner-django/issues/442)) ([05280ae](https://www.github.com/googleapis/python-spanner-django/commit/05280aecdcbe933e113616b5705f4e76303d9637))
+
+<!-- trigger system tests -->

packages/gapic-generator/CHANGELOG.md

@@ -2120,3 +2120,5 @@
 ### Bug Fixes
 
 * update GOOGLE_API_USE_MTLS value ([#453](https://www.github.com/googleapis/gapic-generator-python/issues/453)) ([7449ad5](https://www.github.com/googleapis/gapic-generator-python/commit/7449ad5aad4a1fbbf9ca3796e097512fc80991e3))
+
+<!-- trigger system tests -->

packages/google-auth/CHANGELOG.md

@@ -1951,3 +1951,5 @@ Initial release with foundational functionality for cryptography and JWTs.
 
 - ``google.auth.crypt`` for creating and verifying cryptographic signatures.
 - ``google.auth.jwt`` for creating (encoding) and verifying (decoding) JSON Web tokens.
+
+<!-- trigger system tests -->

packages/google-cloud-access-approval/CHANGELOG.md

@@ -467,3 +467,5 @@
 ### Features
 
 * generate v1 ([88003fe](https://www.github.com/googleapis/python-access-approval/commit/88003fe05150ee653ba9a8ba072058b35d3f3c49))
+
+<!-- trigger system tests -->

packages/google-cloud-automl/CHANGELOG.md

@@ -713,3 +713,5 @@
 
 ### New Features
 - Initial Release of AutoML v1beta1
+
+<!-- trigger system tests -->

packages/google-cloud-bigquery-connection/CHANGELOG.md

@@ -506,3 +506,5 @@
 ### Features
 
 * generate v1 ([73b89dc](https://www.github.com/googleapis/python-bigquery-connection/commit/73b89dcb423026c4b4e537ff728d22be2cb5ff3f))
+
+<!-- trigger system tests -->

packages/google-cloud-bigquery-datatransfer/CHANGELOG.md

@@ -709,3 +709,5 @@ sources like Adwords, DoubleClick Campaign Manager, DoubleClick for Publishers
 and YouTube.
 
 PyPI: https://pypi.org/project/google-cloud-bigquery-datatransfer/0.1.0/
+
+<!-- trigger system tests -->

packages/google-cloud-bigquery-reservation/CHANGELOG.md

@@ -519,3 +519,5 @@
 ### Features
 
 * generate v1 ([6293404](https://www.github.com/googleapis/python-bigquery-reservation/commit/6293404e47ca2efdcb5f702e248f43250060eb8c))
+
+<!-- trigger system tests -->

packages/google-cloud-bigquery-storage/CHANGELOG.md

@@ -889,3 +889,5 @@
 11-29-2018 13:45 PST
 
 - Initial release of BigQuery Storage API client.
+
+<!-- trigger system tests -->

packages/google-cloud-bigquery/CHANGELOG.md

@@ -2779,3 +2779,5 @@ PyPI: https://pypi.org/project/google-cloud-bigquery/0.27.0/
   (#3598)
 
 PyPI: https://pypi.org/project/google-cloud-bigquery/0.26.0/
+
+<!-- trigger system tests -->

packages/google-cloud-bigtable/CHANGELOG.md

@@ -1226,3 +1226,5 @@ PyPI: https://pypi.org/project/google-cloud-bigtable/0.28.1/
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-bigtable/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-compute/CHANGELOG.md

@@ -774,3 +774,5 @@
 ### Features
 
 * generate v1 ([53f9a3d](https://www.github.com/googleapis/python-compute/commit/53f9a3d6f14ef45b5bc3e38a48e3fa17059591eb))
+
+<!-- trigger system tests -->

packages/google-cloud-container/CHANGELOG.md

@@ -1396,3 +1396,5 @@ resource efficiency, automated operations, and open source flexibility to
 accelerate your time to market.
 
 PyPI: https://pypi.org/project/google-cloud-container/0.1.0/
+
+<!-- trigger system tests -->

packages/google-cloud-dataproc/CHANGELOG.md

@@ -832,3 +832,5 @@
 
 - Re-enable lint for tests, remove usage of pylint (#4921)
 - Normalize all setup.py files (#4909)
+
+<!-- trigger system tests -->

packages/google-cloud-datastore/CHANGELOG.md

@@ -821,3 +821,5 @@
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-datastore/1.4.0/
+
+<!-- trigger system tests -->

packages/google-cloud-dlp/CHANGELOG.md

@@ -919,3 +919,5 @@
 ## 0.1.0
 
 Initial release of the DLP (Data Loss Prevention) client library. (#4879)
+
+<!-- trigger system tests -->

packages/google-cloud-dns/CHANGELOG.md

@@ -241,3 +241,5 @@
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-dns/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-error-reporting/CHANGELOG.md

@@ -539,3 +539,5 @@
 - Upgrading to `google-cloud-logging >= 1.4.0` (#4296)
 
 PyPI: https://pypi.org/project/google-cloud-error-reporting/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-firestore/CHANGELOG.md

@@ -1242,3 +1242,5 @@
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-firestore/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-kms/CHANGELOG.md

@@ -831,3 +831,5 @@
 
 ### New Features
 - KMS v1
+
+<!-- trigger system tests -->

packages/google-cloud-logging/CHANGELOG.md

@@ -917,3 +917,5 @@
   `googleapis-common-protos` dependencies (#4096, #4098)
 
 PyPI: https://pypi.org/project/google-cloud-logging/1.4.0/
+
+<!-- trigger system tests -->

packages/google-cloud-monitoring/CHANGELOG.md

@@ -856,3 +856,5 @@
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-monitoring/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-ndb/CHANGELOG.md

@@ -704,3 +704,5 @@
 ## 0.0.1dev1
 
 Initial development release of NDB client library.
+
+<!-- trigger system tests -->

packages/google-cloud-os-config/CHANGELOG.md

@@ -515,3 +515,5 @@
 ### Features
 
 * generate v1 ([5d1f582](https://www.github.com/googleapis/python-os-config/commit/5d1f582b5b02d128ef44120d285941805d234ec7))
+
+<!-- trigger system tests -->

packages/google-cloud-pubsub/CHANGELOG.md

@@ -1732,3 +1732,5 @@ PyPI: https://pypi.org/project/google-cloud-pubsub/0.29.1/
   `googleapis-common-protos` dependencies (#4096, #4098)
 
 PyPI: https://pypi.org/project/google-cloud-pubsub/0.29.0/
+
+<!-- trigger system tests -->

packages/google-cloud-scheduler/CHANGELOG.md

@@ -559,3 +559,5 @@
 
 ### New Features
 - Initial release of Cloud Scheduler library. ([#6482](https://github.com/googleapis/google-cloud-python/pull/6482))
+
+<!-- trigger system tests -->

packages/google-cloud-spanner-dbapi-driver/setup.py

@@ -81,3 +81,5 @@
     include_package_data=True,
     zip_safe=False,
 )
+
+# trigger system tests

packages/google-cloud-spanner/CHANGELOG.md

@@ -1705,3 +1705,5 @@ Return sessions from pool in LIFO order. ([#9454](https://github.com/googleapis/
   `googleapis-common-protos` dependencies (#4096, #4098)
 
 PyPI: https://pypi.org/project/google-cloud-spanner/0.29.0/
+
+<!-- trigger system tests -->

packages/google-cloud-speech/CHANGELOG.md

@@ -935,3 +935,5 @@ This is the (hopefully) final release candidate before 1.0.
   `googleapis-common-protos`dependencies (#4096, #4098)
 
 PyPI: https://pypi.org/project/google-cloud-speech/0.30.0/
+
+<!-- trigger system tests -->

packages/google-cloud-storage/CHANGELOG.md

@@ -1495,3 +1495,5 @@ Please consult the README for details on this major version release.
 - Requiring `google-resumable-media >= 0.3.1` (#4244)
 
 PyPI: https://pypi.org/project/google-cloud-storage/1.6.0/
+
+<!-- trigger system tests -->

packages/google-cloud-tasks/CHANGELOG.md

@@ -734,3 +734,5 @@
 
 ### New Features
 - Add v2beta2 endpoint for Tasks
+
+<!-- trigger system tests -->

packages/google-cloud-testutils/CHANGELOG.md

@@ -165,3 +165,5 @@
 ### Features
 
 * add lower bound checker ([#8](https://www.github.com/googleapis/python-test-utils/issues/8)) ([5ebac9f](https://www.github.com/googleapis/python-test-utils/commit/5ebac9fb0ad005f8ea947c14dfca6de3c0d2cac9))
+
+<!-- trigger system tests -->

packages/google-cloud-texttospeech/CHANGELOG.md

@@ -754,3 +754,5 @@
 ### Interface additions
 
 - Added text-to-speech v1beta1. (#5049)
+
+<!-- trigger system tests -->

packages/google-cloud-translate/CHANGELOG.md

@@ -760,3 +760,5 @@
   on `google-api-core` (#4221, #4280)
 
 PyPI: https://pypi.org/project/google-cloud-translate/1.3.0/
+
+<!-- trigger system tests -->

packages/google-cloud-videointelligence/CHANGELOG.md

@@ -717,3 +717,5 @@ change as the `v1` and `v1beta2` endpoints are identical. If you pinned to
   to `4 - Beta` (eb43849569556c6e47f11b8310864c5a280507f2)
 
 PyPI: https://pypi.org/project/google-cloud-videointelligence/0.28.0/
+
+<!-- trigger system tests -->

packages/google-cloud-vision/CHANGELOG.md

@@ -831,3 +831,5 @@ PyPI: https://pypi.org/project/google-cloud-vision/0.29.0/
   `googleapis-common-protos`dependencies (#4096, #4098)
 
 PyPI: https://pypi.org/project/google-cloud-vision/0.28.0/
+
+<!-- trigger system tests -->

packages/google-resumable-media/CHANGELOG.md

@@ -568,3 +568,5 @@ might break the hypothetical usecase of downloading a blob marked with
 2017-04-21
 
 - Initial public release.
+
+<!-- trigger system tests -->

packages/pandas-gbq/CHANGELOG.md

@@ -916,3 +916,5 @@ Includes patches since the 0.19.2 release on pandas with the following:
     [pandas-GH#14064](https://github.com/pandas-dev/pandas/pull/14064),
     and
     [pandas-GH#14305](https://github.com/pandas-dev/pandas/pull/14305)
+
+<!-- trigger system tests -->

packages/sqlalchemy-bigquery/CHANGELOG.md

@@ -459,3 +459,5 @@ Older versions of this project were distributed as [pybigquery][0].
 
 - Prefer explicitly provided dataset over default dataset in lookup. ([#53](https://github.com/mxmzdlv/pybigquery/pull/53))
 - Use the provided `project_id` when using a service account. ([#52](https://github.com/mxmzdlv/pybigquery/pull/52))
+
+<!-- trigger system tests -->

packages/sqlalchemy-spanner/CHANGELOG.md

@@ -349,3 +349,5 @@
 ### Miscellaneous Chores
 
 * setup release 1.0.0 ([#165](https://www.github.com/googleapis/python-spanner-sqlalchemy/issues/165)) ([37a415d](https://www.github.com/googleapis/python-spanner-sqlalchemy/commit/37a415d071d39e99f233a1c15c1c4b89bd436570))
+
+<!-- trigger system tests -->