Skip to content

Provide the ability to delete (remove) HDF5 files at the end of integration tests #157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bieryAtFnal merged 12 commits into from
Jun 18, 2026
Merged

Provide the ability to delete (remove) HDF5 files at the end of integration tests #157

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
1da8b5c
Modified a few attribute names in the CreateConfigResult data_class t…
May 20, 2026
3db6558
Added the --remove-hdf5-files option in integrationtest_commandline.p…
May 20, 2026
45412ae
Added utility_functions.py as a place to provide functions that can b…
May 20, 2026
9f1696b
Modified integrationtest_drunc.py to start using the latest CreateCon…
May 20, 2026
54a78fa
Changed the name of a function argument in utility_functions::remove_…
May 20, 2026
c9f0c0f
modified the --remove-hdf5-files pytest option to allow users to spec…
May 26, 2026
ca4c635
Added more help text to a couple of our custom pytest options
May 26, 2026
a7b9ea1
Merge remote-tracking branch 'origin/develop' into kbiery/hdf5_remova…
May 27, 2026
4f7846e
Added 'always' and 'never' choices to the --remove-hdf5-files pytest …
Jun 2, 2026
bcb238a
Merge remote-tracking branch 'origin/develop' into kbiery/hdf5_remova…
Jun 17, 2026
66b6aad
Re-worked the functionality that removes HDF5 files at the end of an …
Jun 17, 2026
377b56a
Created a function to delete a file and catch exceptions while doing …
Jun 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/integrationtest/data_classes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ class integtest_params_for_predefined_dunedaq_config(integtest_param_base_class)

@dataclass
class CreateConfigResult:
config: integtest_param_base_class
config_dir: str
config_file: str
integtest_params: integtest_param_base_class
dunedaq_config_dir: str
dunedaq_config_file: str
log_file: str
data_dirs: list[str]
tpstream_data_dirs: list[str]
Expand Down
9 changes: 8 additions & 1 deletion src/integrationtest/integrationtest_commandline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def pytest_addoption(parser):
"--integtest-verbosity",
action="store",
default=3,
help="The volume of messages that are printed out by the integration test infrastructure",
help="This controls the volume of messages that are printed out by the integration test infrastructure (1 is lowest, 6 is highest)",
required=False
)
parser.addoption(
Expand All @@ -56,6 +56,13 @@ def pytest_addoption(parser):
help="A phrase that, if found in run control messages, will trigger the printout of all RC messages",
required=False
)
parser.addoption(
"--remove-hdf5-files",
action="store",
default=None,
help="Whether to remove HDF5 (data) files when the testing has finished (this over-rides the choice in the integtest). 'always' forces files to be removed, 'never' forces files to be kept.",
required=False
)

def pytest_configure(config):
for opt in ("--dunerc-path",):
Expand Down
63 changes: 32 additions & 31 deletions src/integrationtest/integrationtest_drunc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,9 +410,9 @@ def apply_update(obj, substitution):
pass

result = CreateConfigResult(
config=integtest_params,
config_dir=config_dir,
config_file=config_db,
integtest_params=integtest_params,
dunedaq_config_dir=config_dir,
dunedaq_config_file=config_db,
log_file=logfile,
data_dirs=rawdata_dirs,
tpstream_data_dirs=tpstream_dirs,
Expand Down Expand Up @@ -442,8 +442,8 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity"))

if no_integtest_connsvc and \
isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config):
create_config_files.config.connsvc_control = ConnSvcControl.NONE
isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config):
create_config_files.integtest_params.connsvc_control = ConnSvcControl.NONE

run_dir = tmp_path_factory.mktemp("run")

Expand Down Expand Up @@ -497,36 +497,36 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact
# start the Connectivity Service, if requested (only supported for generated dune-daq configs, for now)
connsvc_obj = None
if (
isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config)
and create_config_files.config.connsvc_control == ConnSvcControl.INTEGRATIONTEST
isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config)
and create_config_files.integtest_params.connsvc_control == ConnSvcControl.INTEGRATIONTEST
):
# start connsvc
if integtest_verbosity_level >= IntegtestVerbosityLevels.full_output:
print(
f"Starting Connectivity Service on port {create_config_files.config.connsvc_port}"
f"Starting Connectivity Service on port {create_config_files.integtest_params.connsvc_port}"
)

connsvc_env = os.environ.copy()
if create_config_files.config.connsvc_debug_level is not None:
if create_config_files.integtest_params.connsvc_debug_level is not None:
connsvc_env["CONNECTION_FLASK_DEBUG"] = str(
create_config_files.config.connsvc_debug_level
create_config_files.integtest_params.connsvc_debug_level
)

connsvc_log = open(
run_dir
/ f"log_{getpass.getuser()}_{create_config_files.config.daq_session_name}_connectivity-service.txt",
/ f"log_{getpass.getuser()}_{create_config_files.integtest_params.daq_session_name}_connectivity-service.txt",
"w",
)
connsvc_obj = subprocess.Popen(
f"gunicorn -b 0.0.0.0:{create_config_files.config.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(),
f"gunicorn -b 0.0.0.0:{create_config_files.integtest_params.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(),
stdout=connsvc_log,
stderr=connsvc_log,
env=connsvc_env,
)

elif create_config_files.config.connsvc_debug_level is not None:
set_session_env_var(str(create_config_files.config_file), create_config_files.config.config_session_name,
"CONNECTION_FLASK_DEBUG", create_config_files.config.connsvc_debug_level, overwrite=True)
elif create_config_files.integtest_params.connsvc_debug_level is not None:
set_session_env_var(str(create_config_files.dunedaq_config_file), create_config_files.integtest_params.config_session_name,
"CONNECTION_FLASK_DEBUG", create_config_files.integtest_params.connsvc_debug_level, overwrite=True)

dunerc = request.config.getoption("--dunerc-path")
if dunerc is None:
Expand Down Expand Up @@ -569,13 +569,13 @@ class RunResult:
temp_suffix = ".temp_saved"
now = time.time()
for file_obj in rawdata_dir.glob(
f"{create_config_files.config.op_env}_raw*.hdf5"
f"{create_config_files.integtest_params.op_env}_raw*.hdf5"
):
print(f"Renaming raw data file from earlier test: {str(file_obj)}")
new_name = str(file_obj) + temp_suffix
file_obj.rename(new_name)
for file_obj in rawdata_dir.glob(
f"{create_config_files.config.op_env}_raw*.hdf5{temp_suffix}"
f"{create_config_files.integtest_params.op_env}_raw*.hdf5{temp_suffix}"
):
modified_time = file_obj.stat().st_mtime
if (now - modified_time) > 3600:
Expand All @@ -589,13 +589,13 @@ class RunResult:
temp_suffix = ".temp_saved"
now = time.time()
for file_obj in tpset_dir.glob(
f"{create_config_files.config.op_env}_tp*.hdf5"
f"{create_config_files.integtest_params.op_env}_tp*.hdf5"
):
print(f"Renaming TP data file from earlier test: {str(file_obj)}")
new_name = str(file_obj) + temp_suffix
file_obj.rename(new_name)
for file_obj in tpset_dir.glob(
f"{create_config_files.config.op_env}_tp*.hdf5{temp_suffix}"
f"{create_config_files.integtest_params.op_env}_tp*.hdf5{temp_suffix}"
):
modified_time = file_obj.stat().st_mtime
if (now - modified_time) > 3600:
Expand All @@ -609,13 +609,13 @@ class RunResult:
temp_suffix = ".temp_saved"
now = time.time()
for file_obj in trmon_dir.glob(
f"{create_config_files.config.op_env}_trmon*.hdf5"
f"{create_config_files.integtest_params.op_env}_trmon*.hdf5"
):
print(f"Renaming TRMon data file from earlier test: {str(file_obj)}")
new_name = str(file_obj) + temp_suffix
file_obj.rename(new_name)
for file_obj in trmon_dir.glob(
f"{create_config_files.config.op_env}_trmon*.hdf5{temp_suffix}"
f"{create_config_files.integtest_params.op_env}_trmon*.hdf5{temp_suffix}"
):
modified_time = file_obj.stat().st_mtime
if (now - modified_time) > 3600:
Expand All @@ -631,8 +631,8 @@ class RunResult:
# 25-Mar-2026, KAB: use subprocess.Popen to manage the run control session so that we can
# capture the console output and pass it back to the user for inspection and validation.
popen_command_list = [dunerc] + dunerc_option_strings + [process_manager_type] \
+ [str(create_config_files.config_file)] + [str(create_config_files.config.config_session_name)] \
+ [str(create_config_files.config.daq_session_name)] + run_control_commands
+ [str(create_config_files.dunedaq_config_file)] + [str(create_config_files.integtest_params.config_session_name)] \
+ [str(create_config_files.integtest_params.daq_session_name)] + run_control_commands
rc_process = subprocess.Popen(
popen_command_list,
stdout=subprocess.PIPE,
Expand Down Expand Up @@ -721,39 +721,40 @@ class RunResult:
pass
connsvc_obj.kill()

if create_config_files.config.attempt_cleanup:
if create_config_files.integtest_params.attempt_cleanup:
print(
"Checking for remaining gunicorn and drunc-controller processes", flush=True
)
subprocess.run(["killall", "gunicorn", "drunc-controller"])

result.confgen_config = create_config_files.config
result.config_session_name = create_config_files.config.config_session_name
result.daq_session_name = create_config_files.config.daq_session_name
result.confgen_config = create_config_files.integtest_params
result.config_session_name = create_config_files.integtest_params.config_session_name
result.daq_session_name = create_config_files.integtest_params.daq_session_name
result.dunerc_commands = run_control_commands
result.run_dir = run_dir
result.config_dir = create_config_files.config_dir
result.dunedaq_config_dir = create_config_files.dunedaq_config_dir
result.data_files = []
for rawdata_dir in rawdata_dirs:
result.data_files += list(
rawdata_dir.glob(f"{create_config_files.config.op_env}_raw_*.hdf5")
rawdata_dir.glob(f"{create_config_files.integtest_params.op_env}_raw_*.hdf5")
)
result.tpset_files = []
for tpset_dir in tpset_dirs:
result.tpset_files += list(
tpset_dir.glob(f"{create_config_files.config.op_env}_tp_*.hdf5")
tpset_dir.glob(f"{create_config_files.integtest_params.op_env}_tp_*.hdf5")
)
result.trmon_files = []
for trmon_dir in trmon_dirs:
result.trmon_files += list(
trmon_dir.glob(f"{create_config_files.config.op_env}_trmon_*.hdf5")
trmon_dir.glob(f"{create_config_files.integtest_params.op_env}_trmon_*.hdf5")
)
result.log_files = list(run_dir.glob("log_*.txt")) + list(run_dir.glob("log_*.log"))
result.opmon_files = list(run_dir.glob(f"info*{result.daq_session_name}*.json"))
# 10-Dec-2025, KAB: added the DAQ session overall time so that we can use this
# information in fine-tuning the allowed ranges in time-based checking of test results.
result.daq_session_overall_time = time_after - time_before
result.verbosity_helper = VerbosityHelper(integtest_verbosity_level)
result.user_requests_hdf5_file_removal = request.config.getoption("--remove-hdf5-files")
if number_of_lines_printed_to_the_console > 0:
print("---------- DRUNC Session END ----------", flush=True)
print("", flush=True)
Expand Down
84 changes: 84 additions & 0 deletions src/integrationtest/utility_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
import pytest
import os
import re
from integrationtest.verbosity_helper import IntegtestVerbosityLevels

import functools
print = functools.partial(print, flush=True) # always flush print() output

def basic_checks(run_dunerc, caplog, print_test_name: bool = True):

# print out the name of the current test, if requested
if print_test_name and run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.drunc_transitions):
# print the name of the current test
current_test = os.environ.get("PYTEST_CURRENT_TEST")
match_obj = re.search(r".*\[(.+)-run_.*rc.*\d].*", current_test)
if match_obj:
current_test = match_obj.group(1)
banner_line = re.sub(".", "=", current_test)
print(banner_line)
print(current_test)
print(banner_line)

# Check that dunerc completed correctly
if run_dunerc.completed_process.returncode != 0:
fail_msg = f"The run control session returned a non-zero status code ({run_dunerc.completed_process.returncode})."
pytest.fail(fail_msg, pytrace=False)

# Check that there weren't any warnings or errors during setup
setup_logs = caplog.get_records("setup")
if len(setup_logs) > 0:
fail_msg = f"One or more problems were encountered during the setup of the pytest: {setup_logs}"
pytest.fail(fail_msg, pytrace=False)


def remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal: bool = False):

# if the user requested that the files should be kept, we can simply exit early
if (run_dunerc.user_requests_hdf5_file_removal is not None and
("false" in run_dunerc.user_requests_hdf5_file_removal.lower() or
"never" in run_dunerc.user_requests_hdf5_file_removal.lower())):
return

# if either of the integtest writer or the user running the test requested that the HDF5 files
# be deleted at the end of the test, do that.
if ((run_dunerc.user_requests_hdf5_file_removal is not None and
("true" in run_dunerc.user_requests_hdf5_file_removal.lower() or
"always" in run_dunerc.user_requests_hdf5_file_removal.lower())) or
this_test_requests_hdf5_file_removal):
pathlist_string = ""
filelist_string = ""
for data_file in run_dunerc.data_files:
filelist_string += " " + str(data_file)
if str(data_file.parent) not in pathlist_string:
pathlist_string += " " + str(data_file.parent)
for data_file in run_dunerc.tpset_files:
filelist_string += " " + str(data_file)
if str(data_file.parent) not in pathlist_string:
pathlist_string += " " + str(data_file.parent)
for data_file in run_dunerc.trmon_files:
filelist_string += " " + str(data_file)
if str(data_file.parent) not in pathlist_string:
pathlist_string += " " + str(data_file.parent)

if pathlist_string and filelist_string:
if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
print("============================================")
print("Listing the hdf5 files before deleting them:")
print("============================================")

os.system(f"df -h {pathlist_string}")
print("--------------------")
os.system(f"ls -alF {filelist_string}")

for data_file in run_dunerc.data_files:
data_file.unlink()

@eflumerf eflumerf Jun 4, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we want a helper function that catches common errors and prints nicer messages?

 def remove_file(file):
    try:
      file.unlink()
    except FileNotFoundError:
      print(f"Data file {file} was not found")
    except IsADirectoryError:
      print(f"Data file {file} is actually a directory")

for data_file in run_dunerc.tpset_files:
data_file.unlink()
for data_file in run_dunerc.trmon_files:
data_file.unlink()

if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug):
print("--------------------")
os.system(f"df -h {pathlist_string}")
print("============================================")