diff --git a/utils/upload_sct_coredump.sh b/utils/upload_sct_coredump.sh index 6df403aac27..c727162eec3 100755 --- a/utils/upload_sct_coredump.sh +++ b/utils/upload_sct_coredump.sh @@ -15,12 +15,23 @@ if [[ -n "${RUNNER_IP}" ]] ; then EXTRA_HYDRA_ARGS="--execute-on-runner ${RUNNER_IP}" fi -# Check if the coredumps exists in directory -if ./docker/env/hydra.sh $EXTRA_HYDRA_ARGS "bash -c \"[[ -n \\\"\$( ls $COREDUMP_DIR )\\\" ]]\"" ; then +# Only coredumps from this build. On an ephemeral builder or runner the directory is empty at +# start, so mtime filtering changes nothing there - but on a long-lived agent it holds every dump +# the host ever produced (other jobs' included), and unbounded this used to tar and upload all of +# it. collectTestCoredumps passes the build start as SCT_COREDUMPS_SINCE_EPOCH; standalone runs +# fall back to the last 24h. +SINCE_EPOCH="${SCT_COREDUMPS_SINCE_EPOCH:-$(( $(date +%s) - 86400 ))}" - # Compress the coredumps into a tar.gz file - ./docker/env/hydra.sh $EXTRA_HYDRA_ARGS "bash -c \"sudo tar --zstd -cf $COREDUMP_TARBALL -C $COREDUMP_DIR .\"" +# Collect this build's coredumps, if any (find prints them; empty output means nothing new) +NEW_COREDUMPS=$(./docker/env/hydra.sh $EXTRA_HYDRA_ARGS "bash -c \"find $COREDUMP_DIR -maxdepth 1 -type f -newermt @$SINCE_EPOCH\"" | grep "^$COREDUMP_DIR/" || true) - # Upload the tar.gz file +if [[ -n "${NEW_COREDUMPS}" ]] ; then + + # Compress only the new coredumps into a tarball (relative paths, like the old -C invocation) + ./docker/env/hydra.sh $EXTRA_HYDRA_ARGS "bash -c \"cd $COREDUMP_DIR && find . -maxdepth 1 -type f -newermt @$SINCE_EPOCH -print0 | sudo tar --zstd -cf $COREDUMP_TARBALL --null -T -\"" + + # Upload the tarball ./docker/env/hydra.sh $EXTRA_HYDRA_ARGS upload --test-id $SCT_TEST_ID $COREDUMP_TARBALL +else + echo "no coredumps newer than @$SINCE_EPOCH in $COREDUMP_DIR - nothing to upload" fi diff --git a/vars/collectBuilderLogs.groovy b/vars/collectBuilderLogs.groovy index 0caa3816512..85cfc6611de 100644 --- a/vars/collectBuilderLogs.groovy +++ b/vars/collectBuilderLogs.groovy @@ -2,6 +2,10 @@ def call(Map params){ def test_config = groovy.json.JsonOutput.toJson(params.test_config) + // Bound the journal to this build. On an ephemeral builder the journal is minutes old so the + // bound changes nothing, but on a long-lived agent it is weeks of other jobs' history, and + // unbounded this used to tar and upload the entire host journal every build. + def sinceEpoch = (long) (currentBuild.startTimeInMillis / 1000) sh """#!/bin/bash set -xe @@ -9,9 +13,24 @@ def call(Map params){ echo "${params.test_config}" export SCT_CONFIG_FILES=${test_config} SHORT_SCT_TEST_ID=\$(echo \$SCT_TEST_ID | cut -c1-8) - sudo journalctl --no-tail --no-pager -o short-precise > builder-\$SHORT_SCT_TEST_ID.log - tar -zcvf builder-\$SHORT_SCT_TEST_ID.log.tar.gz builder-\$SHORT_SCT_TEST_ID.log + # sudo -n so an agent without passwordless sudo fails fast instead of hanging on a password + # prompt. Then fall back to an unprivileged read before giving up: on a static agent that is + # still the journal this build can see (its own units, and everything else when the agent user + # is in systemd-journal), which beats no builder log at all. The if/else keeps a journal we + # cannot read from aborting the stage under set -e before anything got uploaded. + if sudo -n journalctl --since "@${sinceEpoch}" --no-tail --no-pager -o short-precise > builder-\$SHORT_SCT_TEST_ID.log ; then + journal_source="sudo journalctl" + elif journalctl --since "@${sinceEpoch}" --no-tail --no-pager -o short-precise > builder-\$SHORT_SCT_TEST_ID.log ; then + journal_source="unprivileged journalctl" + else + journal_source="" + echo "WARNING: neither sudo -n journalctl nor an unprivileged journalctl could read the journal on \$(hostname) - skipping builder journal upload" + fi - ./docker/env/hydra.sh upload --test-id \$SCT_TEST_ID builder-\$SHORT_SCT_TEST_ID.log.tar.gz + if [[ -n "\${journal_source}" ]] ; then + echo "collected builder journal via \${journal_source}" + tar -zcvf builder-\$SHORT_SCT_TEST_ID.log.tar.gz builder-\$SHORT_SCT_TEST_ID.log + ./docker/env/hydra.sh upload --test-id \$SCT_TEST_ID builder-\$SHORT_SCT_TEST_ID.log.tar.gz + fi """ } diff --git a/vars/collectTestCoredumps.groovy b/vars/collectTestCoredumps.groovy index 4ccbe3fccd8..bf2cf32d039 100644 --- a/vars/collectTestCoredumps.groovy +++ b/vars/collectTestCoredumps.groovy @@ -1,8 +1,12 @@ #!groovy def call(){ + // Bound the coredump sweep to this build: on a long-lived agent the coredump directory holds + // every dump the host ever produced (other jobs' included), and unbounded the script tars and + // uploads all of it. Epoch seconds, consumed by upload_sct_coredump.sh via -newermt. + def sinceEpoch = (long) (currentBuild.startTimeInMillis / 1000) sh """#!/bin/bash - ./utils/upload_sct_coredump.sh + SCT_COREDUMPS_SINCE_EPOCH=${sinceEpoch} ./utils/upload_sct_coredump.sh """ }