diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bb3cea4..1fb8743b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,20 +22,21 @@ find_package(Boost COMPONENTS iostreams unit_test_framework REQUIRED) daq_protobuf_codegen( opmon/*.proto ) ############################################################################## -daq_add_library( TriggerInhibitAgent.cpp TriggerRecordBuilderData.cpp TPBundleHandler.cpp - LINK_LIBRARIES +daq_add_library( TriggerInhibitAgent.cpp TPBundleHandler.cpp + LINK_LIBRARIES opmonlib::opmonlib ers::ers HighFive appfwk::appfwk logging::logging stdc++fs dfmessages::dfmessages utilities::utilities trigger::trigger detdataformats::detdataformats trgdataformats::trgdataformats) - + daq_add_plugin( HDF5DataStore duneDataStore LINK_LIBRARIES dfmodules hdf5libs::hdf5libs stdc++fs) -daq_add_plugin( FragmentAggregatorModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) -daq_add_plugin( DataWriterModule duneDAQModule LINK_LIBRARIES dfmodules hdf5libs::hdf5libs iomanager::iomanager ) -daq_add_plugin( DFOModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) -daq_add_plugin( TRBModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) -daq_add_plugin( TRMonRequestorModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) -daq_add_plugin( FakeDataProdModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager) -daq_add_plugin( TPStreamWriterModule duneDAQModule LINK_LIBRARIES dfmodules hdf5libs::hdf5libs trigger::trigger Boost::iostreams ) +daq_add_plugin( FragmentAggregatorModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( DataWriterModule duneDAQModule LINK_LIBRARIES dfmodules hdf5libs::hdf5libs iomanager::iomanager ) +daq_add_plugin( DataflowStatusModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( DFOModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( TRBModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( TRMonRequestorModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( FakeDataProdModule duneDAQModule LINK_LIBRARIES dfmodules iomanager::iomanager ) +daq_add_plugin( TPStreamWriterModule duneDAQModule LINK_LIBRARIES dfmodules hdf5libs::hdf5libs trigger::trigger Boost::iostreams ) ############################################################################## daq_add_unit_test( HDF5FileUtils_test LINK_LIBRARIES dfmodules ) @@ -46,9 +47,17 @@ add_dependencies( HDF5Write_test dfmodules_HDF5DataStore_duneDataStore ) daq_add_unit_test( DFOModule_test LINK_LIBRARIES dfmodules ) add_dependencies( DFOModule_test dfmodules_DFOModule_duneDAQModule) -daq_add_unit_test( TriggerRecordBuilderData_test LINK_LIBRARIES dfmodules) +daq_add_unit_test( DataflowStatusModule_test LINK_LIBRARIES dfmodules ) +add_dependencies( DataflowStatusModule_test dfmodules_DataflowStatusModule_duneDAQModule) + +daq_add_unit_test( DFOProtocol_test LINK_LIBRARIES dfmodules ) +add_dependencies( DFOProtocol_test dfmodules_DFOModule_duneDAQModule dfmodules_DataflowStatusModule_duneDAQModule) + daq_add_unit_test( DataStoreFactory_test LINK_LIBRARIES dfmodules) +daq_add_unit_test( TRBModule_test LINK_LIBRARIES dfmodules ) +add_dependencies( TRBModule_test dfmodules_TRBModule_duneDAQModule) + ############################################################################## daq_install() diff --git a/integtest/dfo_protocol_test.py b/integtest/dfo_protocol_test.py new file mode 100755 index 00000000..1f102c7b --- /dev/null +++ b/integtest/dfo_protocol_test.py @@ -0,0 +1,247 @@ +""" +Integration Test for DFO Protocol + +This test module validates DAQ system behavior while using multiple DFO applications. +It also verifies that the protocol correctly handles DFO and DF application crashes. +""" + +import copy +import conffwk +import os +import pathlib +import pytest +import random +import string + +import integrationtest.data_classes as data_classes +import integrationtest.data_file_checks as data_file_checks +import integrationtest.log_file_checks as log_file_checks +import integrationtest.resource_validation as resource_validation +import integrationtest.utility_functions as utility_functions +from integrationtest.get_pytest_tmpdir import get_pytest_tmpdir +from integrationtest.verbosity_helper import IntegtestVerbosityLevels + +import functools + +print = functools.partial(print, flush=True) # always flush print() output + +pytest_plugins = "integrationtest.integrationtest_drunc" + +# Run setup +run_duration = 30 # seconds +check_for_logfile_errors = True + +# Default values for validation parameters +number_of_dataflow_apps = 3 +number_of_data_producers = 4 +number_of_readout_apps = 2 +trigger_rate = 4.0 +expected_number_of_data_files = number_of_dataflow_apps +check_for_logfile_errors = True +expected_event_count = run_duration * trigger_rate / number_of_dataflow_apps +expected_event_count_tolerance = expected_event_count / 10 +ta_prescale = 1000 + +wibeth_frag_params = { + "fragment_type_description": "WIBEth", + "fragment_type": "WIBEth", + "expected_fragment_count": number_of_readout_apps * number_of_data_producers, + "min_size_bytes": 7272, + "max_size_bytes": 28872, +} +# sizes: 128 is for one TC with zero TAs inside it (72+56) +# 208 is for one TC with one TA inside it (72+56+80) +# 264 is for two TCs with one TA in one of them (72+56+80+56) +triggercandidate_frag_params = { + "fragment_type_description": "Trigger Candidate", + "fragment_type": "Trigger_Candidate", + "expected_fragment_count": 1, + "min_size_bytes": 128, + "max_size_bytes": 264, + "debug_mask": 0x0, + "frag_sizes_by_TC_type": {"kPrescale": {"min_size_bytes": 208, "max_size_bytes": 264}, + "kRandom": {"min_size_bytes": 128, "max_size_bytes": 264}, + "default": {"min_size_bytes": 128, "max_size_bytes": 264} } +} +# sizes: 72 is for an empty TP fragment +# 168 is for a fragment with four TPs in it (72+24+24+24+24) +triggerprimitive_frag_params = { + "fragment_type_description": "Trigger Primitive", + "fragment_type": "Trigger_Primitive", + "expected_fragment_count": 3 * number_of_readout_apps, + "min_size_bytes": 72, + "max_size_bytes": 168, +} +# 03-Jul-2025, KAB: changing the default max size from 72 to 100 to handle cases in which there +# was a Random or Prescale trigger along with a coincidental HSI event within the readout window. +hsi_frag_params = { + "fragment_type_description": "HSI", + "fragment_type": "Hardware_Signal", + "expected_fragment_count": 1, + "min_size_bytes": 72, + "max_size_bytes": 100, + "frag_sizes_by_TC_type": {"kTiming": {"min_size_bytes": 100, "max_size_bytes": 100}, + "default": {"min_size_bytes": 72, "max_size_bytes": 100} } +} +ignored_logfile_problems = { + "-controller": [ + ], + "local-connection-server": [ + "errorlog: -", + ], + # 04-Mar-2026, KAB: added the absl::InitializeLog warning message to the ignored list for + # all DAQ processes, given that we currently don't have a way suppress it at its source. + r".*": [ + r"WARNING: All log messages before absl::InitializeLog\(\) is called are written to STDERR" + ] +} + +# Determine if this computer has enough resources for these tests +resource_validator = resource_validation.ResourceValidator() +resource_validator.cpu_count_needs( + 15, 30 +) # 3 for each data source (incl TPG) plus 3 more for everything else +resource_validator.free_memory_needs( + 9, 14 +) # 30% more than what we observe being used ('free -h') +actual_output_path = get_pytest_tmpdir() +resource_validator.free_disk_space_needs( + actual_output_path, 1 +) # more than what we observe + +### Config setup +common_config_obj = data_classes.integtest_params_for_predefined_dunedaq_config() +common_config_obj.op_env = "test" +common_config_obj.predefined_config_db ="config/daqsystemtest/example-configs.data.xml" + +common_config_obj.config_substitutions.append( + data_classes.attribute_substitution( + obj_class="TCDataProcessor", # 12-Nov-2025, KAB: turned off the merging of + obj_id="def-tc-processor", # overlapping TCs so that we get more consistent + updates={ # numbers of TriggerRecords in the output files. + "merge_overlapping_tcs": False + },) +) + +# Get default config +multidfo_local_conf = copy.deepcopy(common_config_obj) +multidfo_local_conf.config_session_name = "local-multidfo-config" + +# Prep configs +stopped_dfo_conf = copy.deepcopy(multidfo_local_conf) +killed_dfo_conf = copy.deepcopy(multidfo_local_conf) +stopped_df_conf = copy.deepcopy(multidfo_local_conf) +killed_df_conf = copy.deepcopy(multidfo_local_conf) + +stopped_dfo_conf.system_signal_configs = [ + data_classes.system_signal_config( + application_label="dfo-02", signal=data_classes.PosixSignal.SIGSTOP, delay_s=25 + ), + data_classes.system_signal_config( + application_label="dfo-02", signal=data_classes.PosixSignal.SIGCONT, delay_s=30 + ), +] + + +killed_dfo_conf.system_signal_configs = [ + data_classes.system_signal_config( + application_label="dfo-02", signal=data_classes.PosixSignal.SIGKILL, delay_s=25 + ), +] + +stopped_df_conf.system_signal_configs = [ + data_classes.system_signal_config( + application_label="df-02", signal=data_classes.PosixSignal.SIGSTOP, delay_s=25 + ), + data_classes.system_signal_config( + application_label="df-02", signal=data_classes.PosixSignal.SIGCONT, delay_s=30 + ), +] + +killed_df_conf.system_signal_configs = [ + data_classes.system_signal_config( + application_label="df-02", signal=data_classes.PosixSignal.SIGKILL, delay_s=25 + ), +] + +# Finally store configs in map +confgen_arguments = { + "default": multidfo_local_conf, + "stopped-dfo": stopped_dfo_conf, + "killed-dfo": killed_dfo_conf, + "stopped-df": stopped_df_conf, + "killed-df": killed_df_conf, +} + +# The commands to run in dunerc, as a list +dunerc_command_list = ( + "boot wait 2 conf start --run-number 101 wait 1 enable-triggers wait ".split() + + [str(run_duration)] + + "disable-triggers wait 2 drain-dataflow wait 2 stop-trigger-sources stop scrap terminate".split() +) + + +### Tests + +def test_dunerc_success(run_dunerc, caplog): + # checks for run control success, problems during pytest setup, etc. + utility_functions.basic_checks(run_dunerc, caplog, print_test_name=True) + + +def test_log_files(run_dunerc): + if check_for_logfile_errors: + # Check that there are no warnings or errors in the log files + assert log_file_checks.logs_are_error_free( + run_dunerc.log_files, True, True, ignored_logfile_problems, + verbosity_helper=run_dunerc.verbosity_helper + ) + + +def test_data_files(run_dunerc): + local_expected_event_count = expected_event_count + local_event_count_tolerance = expected_event_count_tolerance + low_number_of_files = expected_number_of_data_files + high_number_of_files = expected_number_of_data_files + fragment_check_list = [triggercandidate_frag_params, hsi_frag_params, wibeth_frag_params] + + local_expected_event_count += ( + (6250.0 / ta_prescale) + * number_of_data_producers + * number_of_readout_apps + * run_duration + / (100 * number_of_dataflow_apps) + ) + local_event_count_tolerance += ( + (250.0 / ta_prescale) + * number_of_data_producers + * number_of_readout_apps + * run_duration + / (100 * number_of_dataflow_apps) + ) + fragment_check_list.append(triggerprimitive_frag_params) + nontrig_fragment_check_list = [hsi_frag_params, wibeth_frag_params] + + # Run some tests on the output data file + assert ( + len(run_dunerc.data_files) == high_number_of_files + or len(run_dunerc.data_files) == low_number_of_files + ) + + all_ok = True + for idx in range(len(run_dunerc.data_files)): + data_file = data_file_checks.DataFile(run_dunerc.data_files[idx], run_dunerc.verbosity_helper) + all_ok &= data_file_checks.sanity_check(data_file) + all_ok &= data_file_checks.check_file_attributes(data_file) + all_ok &= data_file_checks.check_event_count( + data_file, local_expected_event_count, local_event_count_tolerance + ) + for jdx in range(len(fragment_check_list)): + all_ok &= data_file_checks.check_fragment_count( + data_file, fragment_check_list[jdx] + ) + all_ok &= data_file_checks.check_fragment_sizes( + data_file, fragment_check_list[jdx] + ) + for kdx in range(len(nontrig_fragment_check_list)): + all_ok &= data_file_checks.check_fragment_error_flags( data_file, nontrig_fragment_check_list[kdx]) + assert all_ok diff --git a/plugins/DFOModule.cpp b/plugins/DFOModule.cpp index 6c66e37e..ee0b98fa 100644 --- a/plugins/DFOModule.cpp +++ b/plugins/DFOModule.cpp @@ -13,6 +13,7 @@ #include "appmodel/DFOModule.hpp" #include "confmodel/Connection.hpp" +#include "dfmessages/DataflowStatusRequest.hpp" #include "iomanager/IOManager.hpp" #include "logging/Logging.hpp" @@ -68,8 +69,8 @@ DFOModule::init(std::shared_ptr mcfg) auto iom = iomanager::IOManager::get(); for (auto con : mdal->get_inputs()) { - if (con->get_data_type() == datatype_to_string()) { - m_token_connection = con->UID(); + if (con->get_data_type() == datatype_to_string()) { + m_status_connection = con->UID(); } if (con->get_data_type() == datatype_to_string()) { m_td_connection = con->UID(); @@ -84,21 +85,19 @@ DFOModule::init(std::shared_ptr mcfg) } } - if (m_token_connection == "") { - throw appfwk::MissingConnection( - ERS_HERE, get_name(), datatype_to_string(), "input"); + if (m_status_connection == "") { + throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "input"); } if (m_td_connection == "") { throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "input"); } if (m_busy_sender == nullptr) { throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "output"); - } m_dfo_conf = mdal->get_configuration(); // these are just tests to check if the connections are ok - iom->get_receiver(m_token_connection); + iom->get_receiver(m_status_connection); iom->get_receiver(m_td_connection); TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting init() method"; @@ -111,13 +110,17 @@ DFOModule::do_conf(const CommandData_t&) m_queue_timeout = std::chrono::milliseconds(m_dfo_conf->get_general_queue_timeout_ms()); m_stop_timeout = std::chrono::milliseconds(m_dfo_conf->get_stop_timeout_ms()); - m_busy_threshold = m_dfo_conf->get_busy_threshold(); - m_free_threshold = m_dfo_conf->get_free_threshold(); + m_request_reply_wait = std::chrono::milliseconds(m_dfo_conf->get_request_reply_wait_ms()); + m_status_watchdog_interval = std::chrono::milliseconds(m_dfo_conf->get_status_watchdog_interval_ms()); + m_dataflow_status_timeout = std::chrono::milliseconds(m_dfo_conf->get_dataflow_status_timeout_ms()); + + m_reallocate_building_triggers_on_timeout = m_dfo_conf->get_reallocate_building_triggers_on_timeout(); + m_reallocate_writing_triggers_on_timeout = m_dfo_conf->get_reallocate_writing_triggers_on_timeout(); m_td_send_retries = m_dfo_conf->get_td_send_retries(); - TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_conf() method, there are " - << m_dataflow_availability.size() << " TRB apps defined"; + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_conf() method, there are " << m_trb_conn_ids.size() + << " TRB apps defined"; } void @@ -125,14 +128,12 @@ DFOModule::do_start(const CommandData_t& payload) { TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_start() method"; - m_received_tokens = 0; m_run_number = payload.value("run", 0); m_running_status.store(true); m_last_notified_busy.store(false); - m_last_assignement_it = m_dataflow_availability.end(); - m_last_token_received = m_last_td_received = std::chrono::steady_clock::now(); + m_last_td_received = std::chrono::steady_clock::now(); // 19-Dec-2024, KAB: check that TriggerDecision senders are ready to send. This is done // so that the IOManager infrastructure fetches the necessary connection details from @@ -149,15 +150,20 @@ DFOModule::do_start(const CommandData_t& payload) auto sender = iom->get_sender(trb_conn); if (sender != nullptr) { bool is_ready = sender->is_ready_for_sending(std::chrono::milliseconds(100)); - TLOG_DEBUG(0) << "The TriggerDecision sender for " << trb_conn << " " << (is_ready ? "is" : "is not") << " ready."; + TLOG_DEBUG(0) << "The TriggerDecision sender for " << trb_conn << " " << (is_ready ? "is" : "is not") + << " ready."; } } - iom->add_callback( - m_token_connection, std::bind(&DFOModule::receive_trigger_complete_token, this, std::placeholders::_1)); + iom->add_callback( + m_status_connection, std::bind(&DFOModule::receive_dataflow_status, this, std::placeholders::_1)); iom->add_callback( m_td_connection, std::bind(&DFOModule::receive_trigger_decision, this, std::placeholders::_1)); + m_status_watchdog_thread = std::make_shared(std::bind_front(&DFOModule::status_watchdog_proc, this)); + + notify_trigger_if_needed(); + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_start() method"; } @@ -174,29 +180,32 @@ DFOModule::do_stop(const CommandData_t& /*args*/) const int wait_steps = 20; auto step_timeout = m_stop_timeout / wait_steps; int step_counter = 0; - while (!is_empty() && step_counter < wait_steps) { - TLOG() << get_name() << ": stop delayed while waiting for " << used_slots() << " TDs to completed"; + while (m_assigned_trigger_decisions.size() > 0 && step_counter < wait_steps) { + TLOG() << get_name() << ": stop delayed while waiting for " << m_assigned_trigger_decisions.size() + << " TDs to completed"; std::this_thread::sleep_for(step_timeout); ++step_counter; } - iom->remove_callback(m_token_connection); + m_status_watchdog_thread->request_stop(); + m_status_watchdog_thread->join(); + iom->remove_callback(m_status_connection); - std::list> remnants; - for (auto& app : m_dataflow_availability) { - auto temp = app.second->flush(); - for (auto& td : temp) { - remnants.push_back(td); + { + std::lock_guard status_lock(m_status_mutex); + std::list> remnants; + for (auto& td : m_assigned_trigger_decisions) { + remnants.push_back(td.second); } - } - for (auto& r : remnants) { - ers::error(IncompleteTriggerDecision(ERS_HERE, r->decision.trigger_number, m_run_number)); + for (auto& r : remnants) { + ers::error(IncompleteTriggerDecision(ERS_HERE, get_name(), r->decision.trigger_number, m_run_number)); + } } std::lock_guard guard(m_trigger_counters_mutex); m_trigger_counters.clear(); - + TLOG() << get_name() << " successfully stopped"; TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_stop() method"; } @@ -206,7 +215,7 @@ DFOModule::do_scrap(const CommandData_t& /*args*/) { TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_scrap() method"; - m_dataflow_availability.clear(); + m_dataflow_statuses.clear(); TLOG() << get_name() << " successfully scrapped"; TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_scrap() method"; @@ -220,32 +229,51 @@ DFOModule::receive_trigger_decision(const dfmessages::TriggerDecision& decision) << " (current run is " << m_run_number << ")"; if (decision.run_number != m_run_number) { ers::error(DFOModuleRunNumberMismatch( - ERS_HERE, decision.run_number, m_run_number, "MLT", decision.trigger_number)); + ERS_HERE, get_name(), decision.run_number, m_run_number, "MLT", decision.trigger_number)); return; } auto decision_received = std::chrono::steady_clock::now(); ++m_received_decisions; - auto trigger_types = unpack_types(decision.trigger_type); - for ( const auto t : trigger_types ) { + + auto trigger_types = DFOTriggerCounter::unpack_types(decision.trigger_type); + for (const auto t : trigger_types) { ++get_trigger_counter(t).received; } - + + { + std::lock_guard lk(m_status_mutex); + if (m_assigned_trigger_decisions.find(decision.trigger_number) != m_assigned_trigger_decisions.end()) { + if (m_assigned_trigger_decisions[decision.trigger_number]->decision.trigger_timestamp == + dfmessages::TypeDefaults::s_invalid_timestamp) { + TLOG() << "Received trigger decision for in progress trigger_number " << decision.trigger_number + << " on connection " << m_assigned_trigger_decisions[decision.trigger_number]->connection_name; + m_assigned_trigger_decisions[decision.trigger_number]->decision = decision; + } else { + ers::error(DuplicateTriggerDecision(ERS_HERE, get_name(), decision.trigger_number, m_run_number)); + } + return; + } + } + + m_processing_td.store(true); + std::chrono::steady_clock::time_point decision_assigned; + size_t iteration = 0; do { + send_status_requests(decision.trigger_number, iteration++); auto assignment = find_slot(decision); if (assignment == nullptr) { // this can happen if all application are in error state - ers::error(UnableToAssign(ERS_HERE, decision.trigger_number)); - usleep(500); + ers::error(UnableToAssign(ERS_HERE, get_name(), decision.trigger_number)); + usleep(5000); notify_trigger_if_needed(); continue; } TLOG_DEBUG(TLVL_TRIGDEC_RECEIVED) << get_name() << " Slot found for trigger_number " << decision.trigger_number - << " on connection " << assignment->connection_name - << ", number of used slots is " << used_slots(); + << " on connection " << assignment->connection_name; decision_assigned = std::chrono::steady_clock::now(); auto dispatch_successful = dispatch(assignment); @@ -256,12 +284,17 @@ DFOModule::receive_trigger_decision(const dfmessages::TriggerDecision& decision) break; } else { ers::error( - TRBModuleAppUpdate(ERS_HERE, assignment->connection_name, "Could not send Trigger Decision")); - m_dataflow_availability[assignment->connection_name]->set_in_error(true); + TRBModuleAppUpdate(ERS_HERE, get_name(), assignment->connection_name, "Could not send Trigger Decision")); + // Mark this DF app status as stale so it won't be selected again + auto it = m_dataflow_statuses.find(assignment->connection_name); + if (it != m_dataflow_statuses.end()) { + it->second->status_updated.store(false); + } } } while (m_running_status.load()); + m_processing_td.store(false); notify_trigger_if_needed(); m_waiting_for_decision += @@ -273,186 +306,51 @@ DFOModule::receive_trigger_decision(const dfmessages::TriggerDecision& decision) std::chrono::duration_cast(m_last_td_received - decision_assigned).count(); } -std::shared_ptr -DFOModule::find_slot(const dfmessages::TriggerDecision& decision) -{ - - // this find_slot assings the decision with a round-robin logic - // across all the available applications. - // Applications in error are skipped. - // we only probe the applications once. - // if they are all unavailable the assignment is set to - // the application with the lowest used slots - // returning a nullptr will be considered as an error - // from the upper level code - - std::shared_ptr output = nullptr; - auto minimum_occupied = m_dataflow_availability.end(); - size_t minimum = std::numeric_limits::max(); - unsigned int counter = 0; - - auto candidate_it = m_last_assignement_it; - if (candidate_it == m_dataflow_availability.end()) - candidate_it = m_dataflow_availability.begin(); - - while (output == nullptr && counter < m_dataflow_availability.size()) { - - ++counter; - ++candidate_it; - if (candidate_it == m_dataflow_availability.end()) - candidate_it = m_dataflow_availability.begin(); - - // get rid of the applications in error state - if (candidate_it->second->is_in_error()) { - continue; - } - - // monitor - auto slots = candidate_it->second->used_slots(); - if (slots < minimum) { - minimum = slots; - minimum_occupied = candidate_it; - } - - if (candidate_it->second->is_busy()) - continue; - - output = candidate_it->second->make_assignment(decision); - m_last_assignement_it = candidate_it; - } - - if (!output) { - // in this case all applications were busy - // so we assign the decision to that with the lowest - // number of assignments - if (minimum_occupied != m_dataflow_availability.end()) { - output = minimum_occupied->second->make_assignment(decision); - m_last_assignement_it = minimum_occupied; - ers::warning(AssignedToBusyApp(ERS_HERE, decision.trigger_number, minimum_occupied->first, minimum)); - } - } - - if (output != nullptr) { - TLOG_DEBUG(TLVL_WORK_STEPS) << "Assigned TriggerDecision with trigger number " << decision.trigger_number - << " to TRB at connection " << output->connection_name; - } - return output; -} - void -DFOModule::generate_opmon_data() +DFOModule::generate_opmon_data() { opmon::DFOInfo info; - info.set_tokens_received( m_received_tokens.exchange(0) ); + { + std::lock_guard lk(m_status_mutex); + info.set_pending_trigger_decisions(m_assigned_trigger_decisions.size()); + } + info.set_statuses_received(m_received_statuses.exchange(0)); info.set_decisions_sent(m_sent_decisions.exchange(0)); info.set_decisions_received(m_received_decisions.exchange(0)); + info.set_decisions_completed(m_completed_decisions.exchange(0)); info.set_waiting_for_decision(m_waiting_for_decision.exchange(0)); info.set_deciding_destination(m_deciding_destination.exchange(0)); info.set_forwarding_decision(m_forwarding_decision.exchange(0)); - info.set_waiting_for_token(m_waiting_for_token.exchange(0)); - info.set_processing_token(m_processing_token.exchange(0)); - publish( std::move(info) ); + publish(std::move(info)); - std::lock_guard guard(m_trigger_counters_mutex); - for ( auto & [type, counts] : m_trigger_counters ) { + std::lock_guard guard(m_trigger_counters_mutex); + for (auto& [type, counts] : m_trigger_counters) { opmon::TriggerInfo ti; ti.set_received(counts.received.exchange(0)); ti.set_completed(counts.completed.exchange(0)); auto name = dunedaq::trgdataformats::get_trigger_candidate_type_names()[type]; - publish( std::move(ti), {{"type", name}} ); - } -} - -void -DFOModule::receive_trigger_complete_token(const dfmessages::TriggerDecisionToken& token) -{ - if (token.run_number == 0 && token.trigger_number == 0) { - if (m_dataflow_availability.count(token.decision_destination) == 0) { - TLOG_DEBUG(TLVL_CONFIG) << "Creating dataflow availability struct for uid " << token.decision_destination; - auto entry = m_dataflow_availability[token.decision_destination] = - std::make_shared(token.decision_destination, m_busy_threshold, m_free_threshold); - register_node(token.decision_destination, entry); - } else { - TLOG() << TRBModuleAppUpdate(ERS_HERE, token.decision_destination, "Has reconnected"); - auto app_it = m_dataflow_availability.find(token.decision_destination); - app_it->second->set_in_error(false); - } - return; - } - - TLOG_DEBUG(TLVL_TDTOKEN_RECEIVED) << get_name() << " Received TriggerDecisionToken for trigger_number " - << token.trigger_number << " and run " << token.run_number - << " (current run is " << m_run_number << ")"; - // add a check to see if the application data found - if (token.run_number != m_run_number) { - std::ostringstream oss_source; - oss_source << "TRB at connection " << token.decision_destination; - ers::error(DFOModuleRunNumberMismatch( - ERS_HERE, token.run_number, m_run_number, oss_source.str(), token.trigger_number)); - return; - } - - auto app_it = m_dataflow_availability.find(token.decision_destination); - // check if application data exists; - if (app_it == m_dataflow_availability.end()) { - ers::error(UnknownTokenSource(ERS_HERE, token.decision_destination)); - return; - } - - ++m_received_tokens; - auto callback_start = std::chrono::steady_clock::now(); - - try { - auto dec_ptr = app_it->second->complete_assignment(token.trigger_number, m_metadata_function); - auto trigger_types = unpack_types(dec_ptr->decision.trigger_type); - for ( const auto t : trigger_types ) ++ get_trigger_counter(t).completed; - } catch (AssignedTriggerDecisionNotFound const& err) { - ers::error(err); - } - - if (app_it->second->is_in_error()) { - TLOG() << TRBModuleAppUpdate(ERS_HERE, token.decision_destination, "Has reconnected"); - app_it->second->set_in_error(false); + publish(std::move(ti), { { "type", name } }); } - - notify_trigger_if_needed(); - - m_waiting_for_token += - std::chrono::duration_cast(callback_start - m_last_token_received).count(); - m_last_token_received = std::chrono::steady_clock::now(); - m_processing_token += - std::chrono::duration_cast(m_last_token_received - callback_start).count(); } bool DFOModule::is_busy() const { - for (auto& dfapp : m_dataflow_availability) { - if (!dfapp.second->is_busy()) - return false; - } - return true; -} + if (m_processing_td.load()) + return true; // DFO is busy processing a TriggerDecision -bool -DFOModule::is_empty() const -{ - for (auto& dfapp : m_dataflow_availability) { - if (dfapp.second->used_slots() != 0) - return false; - } - return true; -} + for (auto& dfapp : m_dataflow_statuses) { + if (!dfapp.second->status_updated.load()) + continue; // Skip stale statuses -size_t -DFOModule::used_slots() const -{ - size_t total = 0; - for (auto& dfapp : m_dataflow_availability) { - total += dfapp.second->used_slots(); + // Check if this DF app is not busy (has available slots) + size_t occupied = dfapp.second->status.triggers_building.size() + dfapp.second->status.triggers_writing.size(); + if (!dfapp.second->status.is_busy && occupied < dfapp.second->status.busy_threshold) { + return false; // At least one DF app is available + } } - return total; + return true; // All DF apps are busy } void @@ -465,28 +363,30 @@ DFOModule::notify_trigger_if_needed() const // has changed. std::lock_guard guard(m_notify_trigger_mutex); + static bool first = true; // Always send one TriggerInhibit message at the start of a run bool busy = is_busy(); - if (busy == m_last_notified_busy.load()) + if (!first && busy == m_last_notified_busy.load()) return; bool wasSentSuccessfully = false; do { try { - dfmessages::TriggerInhibit message{ busy, m_run_number }; - m_busy_sender->send(std::move(message), m_queue_timeout); + dfmessages::TriggerInhibit message{ busy, m_run_number, m_td_connection }; + m_busy_sender->send(std::move(message), first ? ipm::Sender::s_block : m_queue_timeout); wasSentSuccessfully = true; TLOG_DEBUG(TLVL_NOTIFY_TRIGGER) << get_name() << " Sent BUSY status " << busy << " to trigger in run " << m_run_number; } catch (const ers::Issue& excpt) { std::ostringstream oss_warn; oss_warn << "Send with sender \"" << m_busy_sender->get_name() << "\" failed"; - ers::warning(iomanager::OperationFailed(ERS_HERE, oss_warn.str(), excpt)); + ers::warning(iomanager::OperationFailed(ERS_HERE, get_name(), oss_warn.str(), excpt)); } } while (!wasSentSuccessfully && m_running_status.load()); m_last_notified_busy.store(busy); + first = false; } bool @@ -513,7 +413,7 @@ DFOModule::dispatch(const std::shared_ptr& assignment) } catch (const ers::Issue& excpt) { std::ostringstream oss_warn; oss_warn << "Send to connection \"" << assignment->connection_name << "\" failed"; - ers::warning(iomanager::OperationFailed(ERS_HERE, oss_warn.str(), excpt)); + ers::warning(iomanager::OperationFailed(ERS_HERE, get_name(), oss_warn.str(), excpt)); } retries--; @@ -524,10 +424,261 @@ DFOModule::dispatch(const std::shared_ptr& assignment) return wasSentSuccessfully; } +DFOTriggerCounter& +DFOModule::get_trigger_counter(trgdataformats::TriggerCandidateData::Type type) +{ + auto it = m_trigger_counters.find(type); + if (it != m_trigger_counters.end()) + return it->second; + + std::lock_guard guard(m_trigger_counters_mutex); + return m_trigger_counters[type]; +} + +void +DFOModule::receive_dataflow_status(const dfmessages::DataflowStatus& status) +{ + TLOG_DEBUG(TLVL_WORK_STEPS) << get_name() << " Received DataflowStatus from " << status.decision_destination + << " for trigger_number " << status.trigger_number << " in run " << status.run_number; + + ++m_received_statuses; + { + std::lock_guard guard(m_status_mutex); + // Update or create entry for this dataflow app + auto it = m_dataflow_statuses.find(status.decision_destination); + if (it != m_dataflow_statuses.end()) { + it->second->update(status); + } else { + m_dataflow_statuses[status.decision_destination] = + std::make_shared(status, m_dataflow_status_timeout); + } + + if (status.trigger_number != 0) { + if (m_statuses_for_trigger.count(status.trigger_number) == 0 || + m_statuses_for_trigger[status.trigger_number].count(status.decision_destination) == 0 || + m_statuses_for_trigger[status.trigger_number][status.decision_destination].iteration_number < + status.iteration_number) { + m_statuses_for_trigger[status.trigger_number][status.decision_destination] = status; + } + } + + for (auto& trigger : status.triggers_building) { + if (!m_assigned_trigger_decisions.count(trigger)) { + ers::info(UnexpectedTriggerDecision(ERS_HERE, get_name(), trigger, status.decision_destination)); + m_assigned_trigger_decisions[trigger] = std::make_shared( + dfmessages::TriggerDecision(trigger, status.run_number), status.decision_destination); + } + } + for (auto& trigger : status.triggers_writing) { + if (!m_assigned_trigger_decisions.count(trigger)) { + ers::warning(UnexpectedTriggerDecision(ERS_HERE, get_name(), trigger, status.decision_destination)); + m_assigned_trigger_decisions[trigger] = std::make_shared( + dfmessages::TriggerDecision(trigger, status.run_number), status.decision_destination); + } + } + + for (auto& trigger : status.recently_completed_triggers) { + if (m_assigned_trigger_decisions.count(trigger)) { + ++m_completed_decisions; + } + m_statuses_for_trigger.erase(trigger); + m_assigned_trigger_decisions.erase(trigger); + } + } + notify_trigger_if_needed(); + m_status_cv.notify_all(); +} + +std::shared_ptr +DFOModule::find_slot(const dfmessages::TriggerDecision& decision) +{ + // Stable algorithm to select a Dataflow application for a Trigger Decision + // This algorithm ensures that multiple DFOs would pick the same DF app given identical state + + // Collect all candidates from known dataflow apps + std::vector candidates; + + { + std::lock_guard guard(m_status_mutex); + for (const auto& [name, received_status] : m_statuses_for_trigger[decision.trigger_number]) { + candidates.emplace_back(received_status); + } + } + + if (candidates.empty()) { + TLOG_DEBUG(TLVL_WORK_STEPS) << "No dataflow applications with valid status available"; + return nullptr; + } + + // Step 1: Filter by trigger type bit mask + std::vector filtered_candidates; + for (const auto& status : candidates) { + // Check if this DF app supports this trigger type + if ((status.trigger_type_mask & decision.trigger_type) != 0) { + filtered_candidates.push_back(status); + } + } + + if (filtered_candidates.empty()) { + TLOG_DEBUG(TLVL_WORK_STEPS) << "No dataflow applications support trigger type " << decision.trigger_type; + return nullptr; + } + + candidates = std::move(filtered_candidates); + + // Step 2: Find DF apps with most open slots (busy_threshold - building - writing) + if (candidates.size() > 1) { + std::map> open_slots_map; + for (const auto& status : candidates) { + size_t occupied = status.triggers_building.size() + status.triggers_writing.size(); + size_t open_slots = (status.busy_threshold > occupied) ? (status.busy_threshold - occupied) : 0; + open_slots_map[open_slots].push_back(status); + } + + candidates = std::move(open_slots_map.rbegin()->second); + } + + // Step 3: Find DF apps with fewest TriggerRecords processed + if (candidates.size() > 1) { + std::map> fewest_trs_map; + for (const auto& status : candidates) { + fewest_trs_map[status.trigger_records_processed].push_back(status); + } + + candidates = std::move(fewest_trs_map.begin()->second); + } + + // Step 4: Find DF apps with fewest bytes written + if (candidates.size() > 1) { + std::map> fewest_bytes_map; + for (const auto& status : candidates) { + fewest_bytes_map[status.data_size_written].push_back(status); + } + + candidates = std::move(fewest_bytes_map.begin()->second); + } + + // Step 5: Sort by name and pick first (stable tie-breaker) + if (candidates.size() > 1) { + std::sort(candidates.begin(), candidates.end(), [](const auto& a, const auto& b) { + return a.decision_destination < b.decision_destination; + }); + } + + // Create assignment for the selected DF app + const auto& selected_name = candidates[0].decision_destination; + auto assignment = std::make_shared(decision, selected_name); + + TLOG_DEBUG(TLVL_WORK_STEPS) << "Selected DF app " << selected_name << " for trigger_number " + << decision.trigger_number; + + return assignment; +} + void DFOModule::assign_trigger_decision(const std::shared_ptr& assignment) { - m_dataflow_availability[assignment->connection_name]->add_assignment(assignment); + TLOG_DEBUG(TLVL_WORK_STEPS) << "Assigning trigger_number " << assignment->decision.trigger_number << " to " + << assignment->connection_name; + + std::lock_guard lk(m_status_mutex); + m_assigned_trigger_decisions[assignment->decision.trigger_number] = assignment; +} + +void +DFOModule::status_watchdog_proc(std::stop_token stoken) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering status_watchdog_proc() method"; + while (!stoken.stop_requested()) { + std::this_thread::sleep_for(m_status_watchdog_interval); + std::vector triggers_to_reallocate; + { + std::lock_guard guard(m_status_mutex); + for (auto& [name, rstatus] : m_dataflow_statuses) { + if (!rstatus->status_updated) { + ers::error(StaleDataflowStatus(ERS_HERE, get_name(), name, m_dataflow_status_timeout.count())); + if (m_reallocate_building_triggers_on_timeout) { + for (auto& trigger : rstatus->status.triggers_building) { + if (m_assigned_trigger_decisions[trigger]->decision.trigger_timestamp == + dfmessages::TypeDefaults::s_invalid_timestamp) { + ers::error(LostTriggerDecision(ERS_HERE, get_name(), trigger, name)); + continue; + } + ers::error(ReallocatingTrigger(ERS_HERE, get_name(), trigger, name)); + triggers_to_reallocate.push_back(m_assigned_trigger_decisions[trigger]->decision); + m_statuses_for_trigger.erase(trigger); + m_assigned_trigger_decisions.erase(trigger); + } + } + if (m_reallocate_writing_triggers_on_timeout) { + for (auto& trigger : rstatus->status.triggers_writing) { + if (m_assigned_trigger_decisions[trigger]->decision.trigger_timestamp == + dfmessages::TypeDefaults::s_invalid_timestamp) { + ers::error(LostTriggerDecision(ERS_HERE, get_name(), trigger, name)); + continue; + } + ers::error(ReallocatingTrigger(ERS_HERE, get_name(), trigger, name)); + triggers_to_reallocate.push_back(m_assigned_trigger_decisions[trigger]->decision); + m_statuses_for_trigger.erase(trigger); + m_assigned_trigger_decisions.erase(trigger); + } + } + } + } + } + for (auto& trigger : triggers_to_reallocate) { + receive_trigger_decision(trigger); + } + } + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting status_watchdog_proc() method"; +} + +bool +DFOModule::send_status_requests(dfmessages::trigger_number_t trigger, size_t iteration) +{ + std::set destinations_to_request; + { + std::lock_guard guard(m_status_mutex); + for (auto& [name, rstatus] : m_dataflow_statuses) { + if (rstatus->status_updated.load()) { + destinations_to_request.insert(rstatus->status.request_destination); + } + } + } + + dfmessages::DataflowStatusRequest request; + request.run_number = m_run_number; + request.trigger_number = trigger; + request.iteration_number = iteration; + request.reply_destination = m_status_connection; + + for (auto& dest : destinations_to_request) { + + try { + auto iom = iomanager::IOManager::get(); + dfmessages::DataflowStatusRequest request_copy = request; + iom->get_sender(dest)->send(std::move(request_copy), m_queue_timeout); + TLOG_DEBUG(TLVL_WORK_STEPS) << "Sent DataflowStatusRequest for trigger_number " << trigger << " to " << dest; + } catch (const ers::Issue& excpt) { + std::ostringstream oss_warn; + oss_warn << "Send of DataflowStatusRequest to connection \"" << dest << "\" failed"; + ers::warning(iomanager::OperationFailed(ERS_HERE, get_name(), oss_warn.str(), excpt)); + } + } + + std::chrono::steady_clock::time_point start_time = std::chrono::steady_clock::now(); + size_t statuses_received = 0; + while (statuses_received < destinations_to_request.size() && + std::chrono::steady_clock::now() - start_time < m_request_reply_wait) { + std::unique_lock guard(m_status_mutex); + m_status_cv.wait_for(guard, m_request_reply_wait); + auto it = m_statuses_for_trigger.find(trigger); + if (it != m_statuses_for_trigger.end() && !it->second.empty()) { + statuses_received = it->second.size(); + } + } + + return statuses_received == destinations_to_request.size(); } } // namespace dunedaq::dfmodules diff --git a/plugins/DFOModule.hpp b/plugins/DFOModule.hpp index ad280b6c..e28fcff9 100644 --- a/plugins/DFOModule.hpp +++ b/plugins/DFOModule.hpp @@ -9,16 +9,15 @@ #ifndef DFMODULES_PLUGINS_DATAFLOWORCHESTRATOR_HPP_ #define DFMODULES_PLUGINS_DATAFLOWORCHESTRATOR_HPP_ -#include "dfmodules/TriggerRecordBuilderData.hpp" +#include "dfmodules/AssignedTriggerDecision.hpp" +#include "dfmodules/DFOTriggerCounter.hpp" +#include "dfmodules/ReceivedDataflowStatus.hpp" #include "appmodel/DFOConf.hpp" -#include "daqdataformats/TriggerRecord.hpp" -#include "dfmessages/DataRequest.hpp" +#include "dfmessages/DataflowStatus.hpp" #include "dfmessages/TriggerDecision.hpp" -#include "dfmessages/TriggerDecisionToken.hpp" #include "dfmessages/TriggerInhibit.hpp" -#include "trgdataformats/TriggerCandidateData.hpp" #include "iomanager/Sender.hpp" @@ -27,42 +26,86 @@ #include #include -#include -#include -#include #include +#include namespace dunedaq { // Disable coverage checking LCOV_EXCL_START -ERS_DECLARE_ISSUE(dfmodules, - TRBModuleAppUpdate, - "TRBModule app " << connection_name << ": " << message, - ((std::string)connection_name)((std::string)message)) -ERS_DECLARE_ISSUE(dfmodules, - UnknownTokenSource, - "Token from unknown source: " << connection_name, - ((std::string)connection_name)) -ERS_DECLARE_ISSUE(dfmodules, - DFOModuleRunNumberMismatch, - "DFOModule encountered run number mismatch: recvd (" - << received_run_number << ") != " << run_number << " from " << src_app << " for trigger_number " - << trig_num, - ((uint32_t)received_run_number)((uint32_t)run_number)((std::string)src_app)( - (uint32_t)trig_num)) // NOLINT(build/unsigned) -ERS_DECLARE_ISSUE(dfmodules, - IncompleteTriggerDecision, - "TriggerDecision " << trigger_number << " didn't complete within timeout in run " << run_number, - ((uint32_t)trigger_number)((uint32_t)run_number)) // NOLINT(build/unsigned) -ERS_DECLARE_ISSUE(dfmodules, - UnableToAssign, - "TriggerDecision " << trigger_number << " could not be assigned", - ((uint32_t)trigger_number)) // NOLINT(build/unsigned) -ERS_DECLARE_ISSUE(dfmodules, - AssignedToBusyApp, - "TriggerDecision " << trigger_number << " was assigned to DF app " << app << " that was busy with " - << used_slots << " TDs", - ((uint32_t)trigger_number)((std::string)app)((size_t)used_slots)) // NOLINT(build/unsigned) +ERS_DECLARE_ISSUE_BASE(dfmodules, + TRBModuleAppUpdate, + appfwk::GeneralDAQModuleIssue, + "TRBModule app " << connection_name << ": " << message, + ((std::string)name), + ((std::string)connection_name)((std::string)message)) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + UnknownTokenSource, + appfwk::GeneralDAQModuleIssue, + "Token from unknown source: " << connection_name, + ((std::string)name), + ((std::string)connection_name)) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + DFOModuleRunNumberMismatch, + appfwk::GeneralDAQModuleIssue, + "DFOModule encountered run number mismatch: recvd (" + << received_run_number << ") != " << run_number << " from " << src_app + << " for trigger_number " << trig_num, + ((std::string)name), + ((uint32_t)received_run_number)((uint32_t)run_number)((std::string)src_app)( + (uint32_t)trig_num)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + IncompleteTriggerDecision, + appfwk::GeneralDAQModuleIssue, + "TriggerDecision " << trigger_number << " didn't complete within timeout in run " << run_number, + ((std::string)name), + ((uint32_t)trigger_number)((uint32_t)run_number)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + UnexpectedTriggerDecision, + appfwk::GeneralDAQModuleIssue, + "TriggerDecision " << trigger_number << " has been reported by " << app + << " with no TriggerDecision message received", + ((std::string)name), + ((uint32_t)trigger_number)((std::string)app)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + LostTriggerDecision, + appfwk::GeneralDAQModuleIssue, + "TriggerDecision " << trigger_number << " was lost while being processed by " << app, + ((std::string)name), + ((uint32_t)trigger_number)((std::string)app)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + UnableToAssign, + appfwk::GeneralDAQModuleIssue, + "TriggerDecision " << trigger_number << " could not be assigned", + ((std::string)name), + ((uint32_t)trigger_number)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + AssignedToBusyApp, + appfwk::GeneralDAQModuleIssue, + "TriggerDecision " << trigger_number << " was assigned to DF app " << app + << " that was busy with " << used_slots << " TDs", + ((std::string)name), + ((uint32_t)trigger_number)((std::string)app)((size_t)used_slots)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + StaleDataflowStatus, + appfwk::GeneralDAQModuleIssue, + "No DataflowStatus received from " << app << " for " << timeout << " ms", + ((std::string)name), + ((std::string)app)((uint32_t)timeout)) // NOLINT(build/unsigned) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + ReallocatingTrigger, + appfwk::GeneralDAQModuleIssue, + "Reallocating trigger " << trigger << " from DF app " << app, + ((std::string)name), + ((uint32_t)trigger)((std::string)app)) // NOLINT(build/unsigned) // Re-enable coverage checking LCOV_EXCL_STOP namespace dfmodules { @@ -80,24 +123,13 @@ class DFOModule : public dunedaq::appfwk::DAQModule */ explicit DFOModule(const std::string& name); - DFOModule(const DFOModule&) = delete; ///< DFOModule is not copy-constructible - DFOModule& operator=(const DFOModule&) = - delete; ///< DFOModule is not copy-assignable - DFOModule(DFOModule&&) = delete; ///< DFOModule is not move-constructible - DFOModule& operator=(DFOModule&&) = delete; ///< DFOModule is not move-assignable + DFOModule(const DFOModule&) = delete; ///< DFOModule is not copy-constructible + DFOModule& operator=(const DFOModule&) = delete; ///< DFOModule is not copy-assignable + DFOModule(DFOModule&&) = delete; ///< DFOModule is not move-constructible + DFOModule& operator=(DFOModule&&) = delete; ///< DFOModule is not move-assignable void init(std::shared_ptr mcfg) override; -protected: - virtual std::shared_ptr find_slot(const dfmessages::TriggerDecision& decision); - // find_slot operates on a round-robin logic - - using trbd_ptr_t = std::shared_ptr; - using data_structure_t = std::map; - data_structure_t m_dataflow_availability; - data_structure_t::iterator m_last_assignement_it; - std::function m_metadata_function; - private: // Commands void do_conf(const CommandData_t&); @@ -107,73 +139,67 @@ class DFOModule : public dunedaq::appfwk::DAQModule void generate_opmon_data() override; - virtual void receive_trigger_complete_token(const dfmessages::TriggerDecisionToken&); - void receive_trigger_decision(const dfmessages::TriggerDecision&); - virtual bool is_busy() const; - bool is_empty() const; - size_t used_slots() const; - void notify_trigger_if_needed() const; - bool dispatch(const std::shared_ptr& assignment); - virtual void assign_trigger_decision(const std::shared_ptr& assignment); - // Configuration const appmodel::DFOConf* m_dfo_conf; std::chrono::milliseconds m_queue_timeout; std::chrono::microseconds m_stop_timeout; + std::chrono::milliseconds m_request_reply_wait; + std::chrono::milliseconds m_status_watchdog_interval; + std::chrono::milliseconds m_dataflow_status_timeout; dunedaq::daqdataformats::run_number_t m_run_number; + bool m_reallocate_building_triggers_on_timeout{ false }; + bool m_reallocate_writing_triggers_on_timeout{ false }; // Connections std::shared_ptr> m_busy_sender; - std::string m_token_connection; + std::string m_status_connection; std::string m_td_connection; - size_t m_td_send_retries; - size_t m_busy_threshold; - size_t m_free_threshold; std::vector m_trb_conn_ids; + size_t m_td_send_retries; + + void receive_dataflow_status(const dfmessages::DataflowStatus&); + void receive_trigger_decision(const dfmessages::TriggerDecision&); + void notify_trigger_if_needed() const; + + bool send_status_requests(dfmessages::trigger_number_t trigger, size_t iteration); + bool dispatch(const std::shared_ptr& assignment); + + // Dataflow application selection algorithm + std::shared_ptr find_slot(const dfmessages::TriggerDecision& decision); + void assign_trigger_decision(const std::shared_ptr& assignment); // Coordination + + std::mutex m_status_mutex; + std::condition_variable m_status_cv; + std::unordered_map> m_dataflow_statuses; + std::unordered_map> + m_statuses_for_trigger; + std::unordered_map> + m_assigned_trigger_decisions; + std::atomic m_running_status{ false }; mutable std::atomic m_last_notified_busy{ false }; - std::chrono::steady_clock::time_point m_last_token_received; + std::atomic m_processing_td{ false }; std::chrono::steady_clock::time_point m_last_td_received; mutable std::mutex m_notify_trigger_mutex; + std::shared_ptr m_status_watchdog_thread; + std::unordered_map> m_decision_assignment_threads; + + void status_watchdog_proc(std::stop_token stoken); + bool is_busy() const; - // Struct for statistic - struct TriggerData { - std::atomic received{0}; - std::atomic completed{0}; - }; - static std::set - unpack_types( decltype(dfmessages::TriggerDecision::trigger_type) t) { - std::set results; - if (t == dfmessages::TypeDefaults::s_invalid_trigger_type) - return results; - const std::bitset<64> bits(t); - for( size_t i = 0; i < bits.size(); ++i ) { - if ( bits[i] ) results.insert((trgdataformats::TriggerCandidateData::Type)i); - } - return results; - } - // Statistics - std::atomic m_received_tokens{ 0 }; // NOLINT (build/unsigned) + std::atomic m_received_statuses{ 0 }; // NOLINT (build/unsigned) std::atomic m_sent_decisions{ 0 }; // NOLINT (build/unsigned) std::atomic m_received_decisions{ 0 }; // NOLINT (build/unsigned) + std::atomic m_completed_decisions{ 0 }; // NOLINT(build/unsigned) std::atomic m_waiting_for_decision{ 0 }; // NOLINT (build/unsigned) std::atomic m_deciding_destination{ 0 }; // NOLINT (build/unsigned) std::atomic m_forwarding_decision{ 0 }; // NOLINT (build/unsigned) - std::atomic m_waiting_for_token{ 0 }; // NOLINT (build/unsigned) - std::atomic m_processing_token{ 0 }; // NOLINT (build/unsigned) - std::map m_trigger_counters; - std::mutex m_trigger_counters_mutex; // used to safely handle the map above - TriggerData & get_trigger_counter(trgdataformats::TriggerCandidateData::Type type) { - auto it = m_trigger_counters.find(type); - if (it != m_trigger_counters.end()) return it->second; - - std::lock_guard guard(m_trigger_counters_mutex); - return m_trigger_counters[type]; - } - + std::map m_trigger_counters; + std::mutex m_trigger_counters_mutex; // used to safely handle the map above + DFOTriggerCounter& get_trigger_counter(trgdataformats::TriggerCandidateData::Type type); }; } // namespace dfmodules } // namespace dunedaq diff --git a/plugins/DataWriterModule.cpp b/plugins/DataWriterModule.cpp index 09c6f3a1..840de3c0 100644 --- a/plugins/DataWriterModule.cpp +++ b/plugins/DataWriterModule.cpp @@ -93,32 +93,6 @@ DataWriterModule::init(std::shared_ptr mcfg) m_trigger_record_connection = inputs[0]->UID(); - auto modules = mcfg->get_modules(); - std::string trb_uid = ""; - bool is_trmon = false; - for (auto& mod : modules) { - if (mod->class_name() == "TRBModule") { - trb_uid = mod->UID(); - break; - } - if (mod->class_name() == "TRMonRequestorModule") { - is_trmon = true; - break; - } - } - - if (!is_trmon) { - auto trbdal = mcfg->get_dal(trb_uid); - if (!trbdal) { - throw appfwk::CommandFailed(ERS_HERE, "init", get_name(), "Unable to retrieve TRB configuration object"); - } - for (auto con : trbdal->get_inputs()) { - if (con->get_data_type() == datatype_to_string()) { - m_trigger_decision_connection = con->UID(); - } - } - } - // try to create the receiver to see test the connection anyway m_tr_receiver = iom->get_receiver>(m_trigger_record_connection); @@ -188,26 +162,6 @@ DataWriterModule::do_start(const CommandData_t& payload) m_data_storage_is_enabled = (!start_params.disable_data_storage); m_run_number = start_params.run; - TLOG_DEBUG(TLVL_WORK_STEPS) << get_name() << ": Sending initial TriggerDecisionToken to DFO to announce my presence"; - dfmessages::TriggerDecisionToken token; - token.run_number = 0; - token.trigger_number = 0; - token.decision_destination = m_trigger_decision_connection; - - int wasSentSuccessfully = 5; - do { - try { - m_token_output->send(std::move(token), m_queue_timeout); - wasSentSuccessfully = 0; - } catch (const ers::Issue& excpt) { - std::ostringstream oss_warn; - oss_warn << "Send with sender \"" << m_token_output->get_name() << "\" failed"; - ers::warning(iomanager::OperationFailed(ERS_HERE, oss_warn.str(), excpt)); - wasSentSuccessfully--; - std::this_thread::sleep_for(std::chrono::microseconds(5000)); - } - } while (wasSentSuccessfully); - // 04-Feb-2021, KAB: added this call to allow DataStore to prepare for the run. // I've put this call fairly early in this method because it could throw an // exception and abort the run start. And, it seems sensible to avoid starting @@ -381,9 +335,10 @@ DataWriterModule::receive_trigger_record(std::unique_ptrget_header_ref().get_trigger_number() << " onto the relevant output queue"; dfmessages::TriggerDecisionToken token; - token.run_number = m_run_number; - token.trigger_number = trigger_record_ptr->get_header_ref().get_trigger_number(); - token.decision_destination = m_trigger_decision_connection; + token.trigger_id = dfmessages::TriggerId{ m_run_number, + trigger_record_ptr->get_header_ref().get_trigger_number(), + trigger_record_ptr->get_header_ref().get_sequence_number() }; + token.writer_identifier = m_writer_identifier; bool wasSentSuccessfully = false; do { diff --git a/plugins/DataWriterModule.hpp b/plugins/DataWriterModule.hpp index 54127105..53b789f0 100644 --- a/plugins/DataWriterModule.hpp +++ b/plugins/DataWriterModule.hpp @@ -81,7 +81,6 @@ class DataWriterModule : public dunedaq::appfwk::DAQModule using token_sender_t = iomanager::SenderConcept; std::shared_ptr m_token_output; - std::string m_trigger_decision_connection; // Worker(s) dunedaq::utilities::WorkerThread m_thread; diff --git a/plugins/DataflowStatusModule.cpp b/plugins/DataflowStatusModule.cpp new file mode 100755 index 00000000..6c9b80a2 --- /dev/null +++ b/plugins/DataflowStatusModule.cpp @@ -0,0 +1,526 @@ +/** + * @file DataflowStatusModule.cpp DataflowStatusModule class implementation + * + * This is part of the DUNE DAQ Software Suite, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#include "DataflowStatusModule.hpp" +#include "appmodel/DataflowStatusModule.hpp" +#include "dfmodules/CommonIssues.hpp" + +#include "dfmodules/opmon/DataflowStatusModule.pb.h" + +/** + * @brief Name used by TRACE TLOG calls from this source file + */ +#define TRACE_NAME "DataflowStatusModule" // NOLINT +enum +{ + TLVL_ENTER_EXIT_METHODS = 5, + TLVL_CONFIG = 7, + TLVL_WORK_STEPS = 10, + TLVL_TRIGDEC_RECEIVED = 21, + TLVL_HEARTBEAT = 22, + TLVL_SEND_STATE = 23, + TLVL_TRIGCOMPLETE_RECEIVED = 24, +}; + +namespace dunedaq::dfmodules { + +DataflowStatusModule::DataflowStatusModule(const std::string& name) + : dunedaq::appfwk::DAQModule(name) + , m_heartbeat_thread(std::bind(&DataflowStatusModule::status_heartbeat_thread, this, std::placeholders::_1)) +{ + register_command("conf", &DataflowStatusModule::do_conf); + register_command("start", &DataflowStatusModule::do_start); + register_command("stop", &DataflowStatusModule::do_stop); + register_command("scrap", &DataflowStatusModule::do_scrap); +} + +DataflowStatusModule::~DataflowStatusModule() +{ + if (m_heartbeat_thread.thread_running()) { + m_heartbeat_thread.stop_working_thread(); + } + m_status_update_cv.notify_all(); +} + +void +DataflowStatusModule::init(std::shared_ptr mcfg) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering init() method"; + + auto mdal = mcfg->get_dal(get_name()); + if (!mdal) { + throw appfwk::CommandFailed(ERS_HERE, "init", get_name(), "Unable to retrieve configuration object"); + } + auto iom = iomanager::IOManager::get(); + + for (auto con : mdal->get_inputs()) { + if (con->get_data_type() == datatype_to_string()) { + m_token_connection = con->UID(); + } + if (con->get_data_type() == datatype_to_string()) { + m_td_connection = con->UID(); + } + if (con->get_data_type() == datatype_to_string()) { + m_status_request_connection = con->UID(); + } + if (con->get_data_type() == datatype_to_string()) { + m_trb_completion_connection = con->UID(); + } + } + for (auto con : mdal->get_outputs()) { + if (con->get_data_type() == datatype_to_string()) { + m_known_dfos.insert(con->UID()); + } + if (con->get_data_type() == datatype_to_string()) { + m_trigger_decision_sender = iom->get_sender(con->UID()); + } + } + + if (m_token_connection == "") { + throw appfwk::MissingConnection( + ERS_HERE, get_name(), datatype_to_string(), "input"); + } + if (m_td_connection == "") { + throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "input"); + } + if (m_trb_completion_connection == "") { + throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "input"); + } + if (m_status_request_connection == "") { + throw appfwk::MissingConnection( + ERS_HERE, get_name(), datatype_to_string(), "input"); + } + if (m_trigger_decision_sender == nullptr) { + throw appfwk::MissingConnection(ERS_HERE, get_name(), datatype_to_string(), "output"); + } + + m_current_status.decision_destination = m_td_connection; + m_current_status.request_destination = m_status_request_connection; + m_conf = mdal->get_configuration(); + // these are just tests to check if the connections are ok + iom->get_receiver(m_token_connection); + iom->get_receiver(m_td_connection); + iom->get_receiver(m_status_request_connection); + iom->get_receiver(m_trb_completion_connection); + + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting init() method"; +} + +void +DataflowStatusModule::generate_opmon_data() +{ + opmon::DataflowStatusInfo info; + info.set_decisions_received(m_num_trigger_decisions_received.exchange(0)); + info.set_trb_completions_received(m_num_trb_completions_received.exchange(0)); + info.set_tokens_received(m_num_trigger_decision_tokens_received.exchange(0)); + info.set_requests_received(m_num_status_requests_received.exchange(0)); + info.set_status_messages_sent(m_num_status_messages_sent.exchange(0)); + info.set_decisions_sent(m_num_trigger_decisions_sent.exchange(0)); + + info.set_duplicate_decisions_received(m_num_duplicate_decisions_received.exchange(0)); + info.set_unexpected_trb_completions_received(m_num_unexpected_trb_completions_received.exchange(0)); + info.set_unexpected_tokens_received(m_num_unexpected_trigger_decision_tokens_received.exchange(0)); + info.set_early_tokens_received(m_num_early_trigger_decision_tokens_received.exchange(0)); + + publish(std::move(info)); +} + +void +DataflowStatusModule::do_conf(const CommandData_t&) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_conf() method"; + + m_heartbeat_interval = std::chrono::milliseconds(m_conf->get_heartbeat_interval_ms()); + m_stop_timeout = std::chrono::milliseconds(m_conf->get_stop_timeout_ms()); + m_td_queue_timeout = std::chrono::milliseconds(m_conf->get_td_queue_timeout_ms()); + m_snapshot_history_size = m_conf->get_snapshot_history_size(); + m_completed_trigger_history_size = m_conf->get_completed_trigger_history_size(); + + m_current_status.busy_threshold = m_conf->get_busy_threshold(); + m_current_status.free_threshold = m_conf->get_free_threshold(); + m_current_status.trigger_type_mask = static_cast(m_conf->get_trigger_type_mask()); + + auto iom = iomanager::IOManager::get(); + iom->add_callback( + m_status_request_connection, std::bind(&DataflowStatusModule::receive_status_request, this, std::placeholders::_1)); + + m_heartbeat_thread.start_working_thread(); + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_conf() method"; +} + +void +DataflowStatusModule::do_start(const CommandData_t& payload) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_start() method"; + + m_current_status.run_number = payload.value("run", 0); + + { + std::lock_guard lock(m_status_mutex); + m_status_snapshots.clear(); + m_building_sequences.clear(); + m_writing_sequences.clear(); + m_current_status.triggers_building.clear(); + m_current_status.triggers_writing.clear(); + m_current_status.recently_completed_triggers.clear(); + m_current_status.trigger_records_processed = 0; + m_current_status.data_size_written = 0; + } + + // 19-Dec-2024, KAB: check that TriggerDecision senders are ready to send. This is done + // so that the IOManager infrastructure fetches the necessary connection details from + // the ConnectivityService at 'start' time, instead of the first time that the sender + // is used to send a message. This avoids delays in the sending of the first TD in + // the first data-taking run in a DAQ session. Such delays can lead to undesirable + // system behavior like trigger inhibits. + auto iom = iomanager::IOManager::get(); + if (m_trigger_decision_sender != nullptr) { + bool is_ready = m_trigger_decision_sender->is_ready_for_sending(std::chrono::milliseconds(100)); + TLOG_DEBUG(0) << "The sender for TriggerDecision messages " << (is_ready ? "is" : "is not") << " ready."; + } + + for (auto& known_dfo : m_known_dfos) { + + auto sender = iom->get_sender(known_dfo); + if (sender != nullptr) { + bool is_ready = sender->is_ready_for_sending(std::chrono::milliseconds(100)); + TLOG_DEBUG(0) << "The DataflowStatus sender for " << known_dfo << " " << (is_ready ? "is" : "is not") + << " ready."; + } + } + + iom->add_callback( + m_token_connection, std::bind(&DataflowStatusModule::receive_trigger_decision_token, this, std::placeholders::_1)); + + iom->add_callback( + m_td_connection, std::bind(&DataflowStatusModule::receive_trigger_decision, this, std::placeholders::_1)); + + iom->add_callback( + m_trb_completion_connection, std::bind(&DataflowStatusModule::receive_trb_completion, this, std::placeholders::_1)); + + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_start() method"; +} + +void +DataflowStatusModule::do_stop(const CommandData_t& /*args*/) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_stop() method"; + + auto iom = iomanager::IOManager::get(); + // Stop receiving new TDs + iom->remove_callback(m_td_connection); + + // Wait for TRB to finish building TriggerRecords + const int wait_steps = 20; + auto step_timeout = m_stop_timeout / wait_steps; + int step_counter = 0; + while (m_current_status.triggers_building.size() > 0 && step_counter < wait_steps) { + TLOG() << get_name() << ": stop delayed while waiting for " << m_current_status.triggers_building.size() + << " TDs to completed (building)"; + std::this_thread::sleep_for(step_timeout); + ++step_counter; + } + iom->remove_callback(m_trb_completion_connection); + + // Wait for DataWriter(s) to finish writing data and sending TriggerDecisionTokens + while (m_current_status.triggers_writing.size() > 0 && step_counter < wait_steps) { + TLOG() << get_name() << ": stop delayed while waiting for " << m_current_status.triggers_writing.size() + << " TDs to completed (writing)"; + std::this_thread::sleep_for(step_timeout); + ++step_counter; + } + iom->remove_callback(m_token_connection); + + m_current_status.run_number = 0; + + TLOG() << get_name() << " successfully stopped"; + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_stop() method"; +} + +void +DataflowStatusModule::do_scrap(const CommandData_t& /*args*/) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering do_scrap() method"; + + if (m_heartbeat_thread.thread_running()) { + m_heartbeat_thread.stop_working_thread(); + } + + TLOG_DEBUG(TLVL_WORK_STEPS) << get_name() << ": Removing request callback"; + auto iom = iomanager::IOManager::get(); + iom->remove_callback(m_status_request_connection); + + TLOG() << get_name() << " successfully scrapped"; + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting do_scrap() method"; +} + +void +DataflowStatusModule::receive_status_request(const dfmessages::DataflowStatusRequest& request) +{ + TLOG_DEBUG(TLVL_TRIGDEC_RECEIVED) << get_name() << " Received DataflowStatusRequest for run " << request.run_number + << " (current run is " << m_current_status.run_number << ")"; + if (request.run_number != m_current_status.run_number) { + return; + } + ++m_num_status_requests_received; + + { + std::unique_lock lock(m_status_mutex); + if (m_status_snapshots.count(request.trigger_number) == 0 || request.iteration_number > m_status_snapshots[request.trigger_number].iteration_number) { + m_status_snapshots[request.trigger_number] = m_current_status; + m_status_snapshots[request.trigger_number].trigger_number = request.trigger_number; + m_status_snapshots[request.trigger_number].iteration_number = request.iteration_number; + + while (m_status_snapshots.size() > m_snapshot_history_size) { + m_status_snapshots.erase(m_status_snapshots.begin()); + } + } + m_known_dfos.insert(request.reply_destination); + } + + send_dataflow_status_update(request.reply_destination, request.trigger_number); +} + +void +DataflowStatusModule::receive_trigger_decision(dfmessages::TriggerDecision& decision) +{ + TLOG_DEBUG(TLVL_TRIGDEC_RECEIVED) << get_name() << " Received TriggerDecision for trigger_number " + << decision.trigger_number << " and run " << decision.run_number + << " (current run is " << m_current_status.run_number << ")"; + if (decision.run_number != m_current_status.run_number) { + return; + } + + { + std::unique_lock lock(m_status_mutex); + + if (m_current_status.triggers_building.count(decision.trigger_number) > 0 || + m_current_status.triggers_writing.count(decision.trigger_number) > 0) { + TLOG() << DuplicateTriggerDecision(ERS_HERE, get_name(), decision.trigger_number, m_current_status.run_number); + ++m_num_duplicate_decisions_received; + return; + } + + ++m_num_trigger_decisions_received; + + m_current_status.triggers_building.insert(decision.trigger_number); + update_busy_status(lock); + m_status_updated.store(true); + m_status_update_cv.notify_all(); + } + + m_trigger_decision_sender->send(std::move(decision), m_td_queue_timeout); + ++m_num_trigger_decisions_sent; +} + +void +DataflowStatusModule::receive_trb_completion(const dfmessages::TRBCompletion& completion) +{ + TLOG_DEBUG(TLVL_TRIGCOMPLETE_RECEIVED) << get_name() << " Received TRBCompletion for trigger/sequence number " + << completion.trigger_id.trigger_number << "/" + << completion.trigger_id.sequence_number << " and run " + << completion.trigger_id.run_number << " (current run is " + << m_current_status.run_number << ")"; + if (completion.trigger_id.run_number != m_current_status.run_number) { + return; + } + + { + std::lock_guard lock(m_status_mutex); + if (!m_current_status.triggers_building.count(completion.trigger_id.trigger_number)) { + ers::error(UnexpectedTRBCompletion( + ERS_HERE, get_name(), completion.trigger_id.trigger_number, m_current_status.run_number)); + ++m_num_unexpected_trb_completions_received; + return; + } + + ++m_num_trb_completions_received; + + if (completion.trigger_record_max_sequence_number > 0) { + if (m_building_sequences.count(completion.trigger_id.trigger_number) > 0) { + m_building_sequences[completion.trigger_id.trigger_number].first++; + } else { + m_building_sequences[completion.trigger_id.trigger_number] = + std::make_pair(1, completion.trigger_record_max_sequence_number); + m_writing_sequences[completion.trigger_id.trigger_number] = + std::make_pair(0, completion.trigger_record_max_sequence_number); + } + + if (m_building_sequences[completion.trigger_id.trigger_number].first == + m_building_sequences[completion.trigger_id.trigger_number].second + 1) { + TLOG_DEBUG(TLVL_TRIGCOMPLETE_RECEIVED) << get_name() << " All sequences for trigger number " + << completion.trigger_id.trigger_number << " have been built."; + } else { + TLOG_DEBUG(TLVL_TRIGCOMPLETE_RECEIVED) + << get_name() << " Received TRBComplete for sequence " << completion.trigger_id.sequence_number + << " of trigger number " << completion.trigger_id.trigger_number + << ". Total completed sequences: " << m_building_sequences[completion.trigger_id.trigger_number].first + << " of " << m_building_sequences[completion.trigger_id.trigger_number].second + 1; + return; + } + } + + m_building_sequences.erase(completion.trigger_id.trigger_number); + m_current_status.triggers_building.erase(completion.trigger_id.trigger_number); + m_current_status.triggers_writing.insert(completion.trigger_id.trigger_number); + m_status_updated.store(true); + m_status_update_cv.notify_all(); + } +} + +void +DataflowStatusModule::receive_trigger_decision_token(const dfmessages::TriggerDecisionToken& token) +{ + TLOG_DEBUG(TLVL_TRIGDEC_RECEIVED) << get_name() << " Received TriggerDecisionToken for trigger_number " + << token.trigger_id.trigger_number << " and run " << token.trigger_id.run_number + << " (current run is " + << m_current_status.run_number << ")"; + if (token.trigger_id.run_number != m_current_status.run_number) { + return; + } + { + std::unique_lock lock(m_status_mutex); + if (m_current_status.triggers_writing.count(token.trigger_id.trigger_number) == 0 && + m_current_status.triggers_building.count(token.trigger_id.trigger_number) == 0) { + ers::error(UnexpectedTriggerDecisionToken( + ERS_HERE, get_name(), token.trigger_id.trigger_number, m_current_status.run_number)); + ++m_num_unexpected_trigger_decision_tokens_received; + return; + } + if (m_current_status.triggers_writing.count(token.trigger_id.trigger_number) == 0 && + m_current_status.triggers_building.count(token.trigger_id.trigger_number) == 1) { + ers::warning(UnexpectedTriggerDecisionToken( + ERS_HERE, get_name(), token.trigger_id.trigger_number, m_current_status.run_number)); + ++m_num_early_trigger_decision_tokens_received; + } + ++m_num_trigger_decision_tokens_received; + + if (m_writing_sequences.count(token.trigger_id.trigger_number) > 0) { + m_writing_sequences[token.trigger_id.trigger_number].first++; + + if (m_writing_sequences[token.trigger_id.trigger_number].first == + m_writing_sequences[token.trigger_id.trigger_number].second + 1) { + TLOG_DEBUG(TLVL_TRIGCOMPLETE_RECEIVED) << get_name() << " All sequences for trigger number " + << token.trigger_id.trigger_number << " have been written."; + } else { + TLOG_DEBUG(TLVL_TRIGCOMPLETE_RECEIVED) + << get_name() << " Received Topen for sequence " << token.trigger_id.sequence_number + << " of trigger number " << token.trigger_id.trigger_number + << ". Total completed sequences: " << m_writing_sequences[token.trigger_id.trigger_number].first + << " of " << m_writing_sequences[token.trigger_id.trigger_number].second + 1; + return; + } + } + + m_writing_sequences.erase(token.trigger_id.trigger_number); + m_building_sequences.erase(token.trigger_id.trigger_number); + m_current_status.triggers_building.erase(token.trigger_id.trigger_number); + m_current_status.triggers_writing.erase(token.trigger_id.trigger_number); + m_current_status.recently_completed_triggers.insert(token.trigger_id.trigger_number); + while (m_current_status.recently_completed_triggers.size() > m_completed_trigger_history_size) { + m_current_status.recently_completed_triggers.erase(m_current_status.recently_completed_triggers.begin()); + } + + m_current_status.trigger_records_processed++; + m_current_status.data_size_written += token.data_size; + update_busy_status(lock); + + m_status_updated.store(true); + m_status_update_cv.notify_all(); + } +} + +void +DataflowStatusModule::send_dataflow_status_update(std::string const& destination, + dfmessages::trigger_number_t trigger_number) +{ + TLOG_DEBUG(TLVL_SEND_STATE) << get_name() << ": Sending DataflowStatus update to " << destination + << " for trigger number " << trigger_number; + dfmessages::DataflowStatus status_to_send; + { + std::lock_guard lock(m_status_mutex); + if (trigger_number == 0) { + status_to_send = m_current_status; + } else { + auto it = m_status_snapshots.find(trigger_number); + if (it != m_status_snapshots.end()) { + status_to_send = it->second; + } else { + ers::warning(SnapshotNotFound(ERS_HERE, get_name(),trigger_number)); + status_to_send = m_current_status; + } + } + } + + status_to_send.trigger_number = trigger_number; + TLOG_DEBUG(TLVL_SEND_STATE) << get_name() + << ": Sending DataflowStatus, trigger_number: " << status_to_send.trigger_number + << ", is_busy: " << status_to_send.is_busy; + auto iom = iomanager::IOManager::get(); + auto sender = iom->get_sender(destination); + if (sender) { + try { + sender->send(std::move(status_to_send), m_heartbeat_interval); + ++m_num_status_messages_sent; + } catch (iomanager::TimeoutExpired& e) { + TLOG() << get_name() << ": Timeout expired while sending DataflowStatus update to " << destination + << ". Error: " << e.what(); + } + } else { + TLOG() << get_name() << ": No sender found for destination " << destination + << ". Unable to send DataflowStatus update."; + } +} + +void +DataflowStatusModule::status_heartbeat_thread(std::atomic& running_flag) +{ + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Entering status_heartbeat_thread() method"; + while (running_flag.load()) { + TLOG_DEBUG(TLVL_HEARTBEAT) << get_name() << ": Heartbeat thread waiting for " << m_heartbeat_interval.count() + << " ms or status update notification"; + { + std::unique_lock lk(m_status_mutex); + m_status_update_cv.wait_for( + lk, m_heartbeat_interval, [&]() { return !running_flag.load() || m_status_updated.load(); }); + } + if (!running_flag.load()) { + break; + } + TLOG_DEBUG(TLVL_HEARTBEAT) << get_name() << ": Heartbeat thread woke up. Sending status updates to " + << m_known_dfos.size() << " known DFOs."; + for (const auto& dfo : m_known_dfos) { + send_dataflow_status_update(dfo); + } + m_status_updated.store(false); + TLOG_DEBUG(TLVL_HEARTBEAT) << get_name() << ": Heartbeat thread finished sending status updates."; + } + TLOG_DEBUG(TLVL_ENTER_EXIT_METHODS) << get_name() << ": Exiting status_heartbeat_thread() method"; +} + +void +DataflowStatusModule::update_busy_status(std::unique_lock& lk) +{ + if (!lk.owns_lock()) { + throw std::runtime_error("update_busy_status must be called with a unique_lock that owns the lock"); + } + auto current_workload = m_current_status.triggers_building.size() + m_current_status.triggers_writing.size(); + if (m_current_status.is_busy && current_workload < m_current_status.free_threshold) { + m_current_status.is_busy = false; + TLOG() << get_name() << ": Transitioning to FREE status. Current workload: " << current_workload; + } + + if (!m_current_status.is_busy && current_workload > m_current_status.busy_threshold) { + m_current_status.is_busy = true; + TLOG() << get_name() << ": Transitioning to BUSY status. Current workload: " << current_workload; + } +} + +} // namespace dunedaq::dfmodules + +DEFINE_DUNE_DAQ_MODULE(dunedaq::dfmodules::DataflowStatusModule) \ No newline at end of file diff --git a/plugins/DataflowStatusModule.hpp b/plugins/DataflowStatusModule.hpp new file mode 100755 index 00000000..e65c9c08 --- /dev/null +++ b/plugins/DataflowStatusModule.hpp @@ -0,0 +1,131 @@ +/** + * @file DataflowStatusModule.hpp + * + * This is part of the DUNE DAQ Software Suite, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#ifndef DFMODULES_PLUGINS_DATAFLOWSTATUSMODULE_HPP_ +#define DFMODULES_PLUGINS_DATAFLOWSTATUSMODULE_HPP_ + +#include "appfwk/DAQModule.hpp" +#include "appmodel/DataflowStatusModuleConf.hpp" +#include "dfmessages/DataflowStatus.hpp" +#include "dfmessages/DataflowStatusRequest.hpp" +#include "dfmessages/TRBCompletion.hpp" +#include "dfmessages/TriggerDecision.hpp" +#include "dfmessages/TriggerDecisionToken.hpp" +#include "utilities/WorkerThread.hpp" + +#include +#include +#include +#include + +namespace dunedaq { + +// Disable coverage checking LCOV_EXCL_START + +ERS_DECLARE_ISSUE_BASE(dfmodules, + UnexpectedTRBCompletion, + appfwk::GeneralDAQModuleIssue, + "Received TRBCompletion message for trigger number " + << trigger_number << " in run " << run_number << " which is not in the building list", + ((std::string)name), + ((uint32_t)trigger_number)((uint32_t)run_number)) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + UnexpectedTriggerDecisionToken, + appfwk::GeneralDAQModuleIssue, + "Received TriggerDecisionToken message for trigger number " + << trigger_number << " in run " << run_number << " which is not in the writing list", + ((std::string)name), + ((uint32_t)trigger_number)((uint32_t)run_number)) + +ERS_DECLARE_ISSUE_BASE(dfmodules, + SnapshotNotFound, + appfwk::GeneralDAQModuleIssue, + "No snapshot found for trigger number " << trigger_number << ". Sending current status instead.", + ((std::string)name), + ((uint32_t)trigger_number)) +// Re-enable coverage checking LCOV_EXCL_STOP +namespace dfmodules { + +class DataflowStatusModule : public dunedaq::appfwk::DAQModule +{ +public: + explicit DataflowStatusModule(const std::string& name); + virtual ~DataflowStatusModule(); + + DataflowStatusModule(const DataflowStatusModule&) = delete; ///< DataflowStatusModule is not copy-constructible + DataflowStatusModule& operator=(const DataflowStatusModule&) = + delete; ///< DataflowStatusModule is not copy-assignable + DataflowStatusModule(DataflowStatusModule&&) = delete; ///< DataflowStatusModule is not move-constructible + DataflowStatusModule& operator=(DataflowStatusModule&&) = delete; ///< DataflowStatusModule is not move-assignable + + void init(std::shared_ptr mcfg) override; + void generate_opmon_data() override; + +private: + // Commands + void do_conf(const CommandData_t&); + void do_start(const CommandData_t&); + void do_stop(const CommandData_t&); + void do_scrap(const CommandData_t&); + + // Callback + void receive_status_request(const dfmessages::DataflowStatusRequest&); + void receive_trigger_decision(dfmessages::TriggerDecision&); + void receive_trb_completion(const dfmessages::TRBCompletion&); + void receive_trigger_decision_token(const dfmessages::TriggerDecisionToken&); + + // Configuration + const appmodel::DataflowStatusModuleConf* m_conf; + std::chrono::milliseconds m_heartbeat_interval{ 100 }; + std::chrono::milliseconds m_stop_timeout{ 1000 }; + std::chrono::milliseconds m_td_queue_timeout{ 1000 }; + size_t m_snapshot_history_size{ 100 }; + size_t m_completed_trigger_history_size{ 100 }; + + // Connections + std::shared_ptr> m_trigger_decision_sender; + std::string m_status_request_connection; + std::string m_td_connection; + std::string m_trb_completion_connection; + std::string m_token_connection; + std::set m_known_dfos; + + // Status + dfmessages::DataflowStatus m_current_status; + std::map m_status_snapshots; + std::map> + m_building_sequences; + std::map> + m_writing_sequences; + std::mutex m_status_mutex; + std::atomic m_status_updated; + std::condition_variable m_status_update_cv; + dunedaq::utilities::WorkerThread m_heartbeat_thread; + void send_dataflow_status_update(std::string const& destination, dfmessages::trigger_number_t trigger_number = 0); + void status_heartbeat_thread(std::atomic& running_flag); + void update_busy_status(std::unique_lock& lk); + + // Counters + std::atomic m_num_trigger_decisions_received; + std::atomic m_num_trb_completions_received; + std::atomic m_num_trigger_decision_tokens_received; + std::atomic m_num_status_requests_received; + std::atomic m_num_status_messages_sent; + std::atomic m_num_trigger_decisions_sent; + + std::atomic m_num_duplicate_decisions_received; + std::atomic m_num_unexpected_trb_completions_received; + std::atomic m_num_unexpected_trigger_decision_tokens_received; + std::atomic m_num_early_trigger_decision_tokens_received; +}; + +} // namespace dfmodules +} // namespace dunedaq + +#endif // DFMODULES_PLUGINS_DATAFLOWSTATUSMODULE_HPP_ \ No newline at end of file diff --git a/plugins/TRBModule.cpp b/plugins/TRBModule.cpp index bd622f7b..5d3008b8 100644 --- a/plugins/TRBModule.cpp +++ b/plugins/TRBModule.cpp @@ -102,14 +102,27 @@ TRBModule::init(std::shared_ptr mcfg) } } + for (auto con : mdal->get_outputs()) { + if (con->get_data_type() == datatype_to_string>()) { + m_trigger_record_output = iom->get_sender>(con->UID()); + } + if (con->get_data_type() == datatype_to_string()) { + m_trb_complete_output = iom->get_sender(con->UID()); + } + } + if (m_trigger_decision_input == nullptr) { throw InvalidQueueFatalError(ERS_HERE, get_name(), "TriggerDecision Input queue"); } if (m_fragment_input == nullptr) { throw InvalidQueueFatalError(ERS_HERE, get_name(), "Fragment Input queue"); } - - m_trigger_record_output = iom->get_sender>(mdal->get_trigger_record_output()->UID()); + if (m_trigger_record_output == nullptr) { + throw InvalidQueueFatalError(ERS_HERE, get_name(), "Trigger Record Output queue"); + } + if (m_trb_complete_output == nullptr) { + throw InvalidQueueFatalError(ERS_HERE, get_name(), "TRBCompletion Output queue"); + } for (auto con : mdal->get_request_connections()) { for (auto source_id : con->get_source_ids()) { @@ -180,6 +193,7 @@ TRBModule::do_conf(const CommandData_t&) m_trigger_timeout = std::chrono::milliseconds(m_trb_conf->get_trigger_record_timeout_ms()); m_tr_queue_timeout = std::chrono::milliseconds(m_trb_conf->get_tr_queue_timeout()); + m_trb_complete_timeout = std::chrono::milliseconds(m_trb_conf->get_trb_complete_timeout()); m_dreq_queue_timeout = std::chrono::milliseconds(m_trb_conf->get_request_queue_timeout()); TLOG() << get_name() << ": timeouts (ms): TR = " << m_tr_queue_timeout.count() @@ -241,7 +255,7 @@ TRBModule::do_start(const CommandData_t& args) } } - m_run_number.reset(new const daqdataformats::run_number_t(args.at("run").get())); + m_run_number = args.at("run").get(); m_stop_requested = false; // Register the callback to receive monitoring requests @@ -284,7 +298,7 @@ TRBModule::tr_requested(const dfmessages::TRMonRequest& req) ++m_trmon_request_counter; // Ignore requests that don't belong to the ongoing run - if (req.run_number != *m_run_number) + if (req.run_number != m_run_number.load()) return; // Add requests to pending requests @@ -304,7 +318,7 @@ TRBModule::flush_trigger_records() // //-------------------------------------------------- // create all possible trigger record - std::vector triggers; + std::vector triggers; for (const auto& entry : m_trigger_records) { triggers.push_back(entry.first); } @@ -338,8 +352,8 @@ TRBModule::fragments_callback(std::unique_ptr& temp_fr << temp_fragment->get_sequence_number() << " from " << temp_fragment->get_element_id(); - TriggerId temp_id(*temp_fragment); - std::vector complete; + dfmessages::TriggerId temp_id(*temp_fragment); + std::vector complete; bool requested = false; { // Begin mutex block @@ -433,8 +447,8 @@ TRBModule::trigger_decision_callback(dfmessages::TriggerDecision& td) auto start_time = std::chrono::steady_clock::now(); - if (td.run_number != *m_run_number) { - ers::error(UnexpectedTriggerDecision(ERS_HERE, td.trigger_number, td.run_number, *m_run_number)); + if (td.run_number != m_run_number.load()) { + ers::error(UnexpectedTriggerDecision(ERS_HERE, td.trigger_number, td.run_number, m_run_number.load())); ++m_unexpected_trigger_decisions; return; } @@ -449,7 +463,7 @@ TRBModule::trigger_decision_callback(dfmessages::TriggerDecision& td) } TRBModule::trigger_record_ptr_t -TRBModule::extract_trigger_record(const TriggerId& id) +TRBModule::extract_trigger_record(const dfmessages::TriggerId& id) { std::unique_lock lk(m_trigger_records_mutex); auto it = m_trigger_records.extract(id); @@ -545,7 +559,7 @@ TRBModule::create_trigger_records_and_dispatch(const dfmessages::TriggerDecision // The code keeps them. // create the book entry - TriggerId slice_id(td, sequence); + dfmessages::TriggerId slice_id(td, sequence); { std::unique_lock lk(m_trigger_records_mutex); m_open_trigger_record_cv.wait(lk, [&] { return m_trigger_records.size() < m_max_open_trigger_records; }); @@ -654,7 +668,7 @@ TRBModule::dispatch_data_requests(dfmessages::DataRequest dr, const daqdataforma } bool -TRBModule::send_trigger_record(const TriggerId& id) +TRBModule::send_trigger_record(const dfmessages::TriggerId& id) { trigger_record_ptr_t temp_record(extract_trigger_record(id)); @@ -701,6 +715,7 @@ TRBModule::send_trigger_record(const TriggerId& id) } // if m_mon_receiver bool wasSentSuccessfully = false; + auto max_seq_num = static_cast(temp_record->get_header_ref().get_max_sequence_number()); do { try { m_trigger_record_output->send(std::move(temp_record), m_tr_queue_timeout); @@ -717,6 +732,9 @@ TRBModule::send_trigger_record(const TriggerId& id) ers::error(dunedaq::dfmodules::AbandonedTriggerDecision(ERS_HERE, id)); } + dfmessages::TRBCompletion completion_msg{ id, m_this_trb_source_id, max_seq_num }; + m_trb_complete_output->try_send(std::move(completion_msg), m_trb_complete_timeout); + return wasSentSuccessfully; } @@ -732,7 +750,8 @@ TRBModule::check_stale_requests() if (m_trigger_timeout.count() > 0) { - std::vector stale_triggers; + TLOG_DEBUG(TLVL_WORK_STEPS) << get_name() << ": Checking for stale trigger records"; + std::vector stale_triggers; { std::lock_guard lk(m_trigger_records_mutex); for (auto it = m_trigger_records.begin(); it != m_trigger_records.end(); ++it) { diff --git a/plugins/TRBModule.hpp b/plugins/TRBModule.hpp index 55977ea3..e2054025 100644 --- a/plugins/TRBModule.hpp +++ b/plugins/TRBModule.hpp @@ -15,8 +15,10 @@ #include "daqdataformats/TriggerRecord.hpp" #include "daqdataformats/Types.hpp" #include "appmodel/ReadoutApplication.hpp" +#include "dfmessages/TriggerId.hpp" #include "appmodel/SmartDaqApplication.hpp" #include "dfmessages/DataRequest.hpp" +#include "dfmessages/TRBCompletion.hpp" #include "dfmessages/TRMonRequest.hpp" #include "dfmessages/TriggerDecision.hpp" #include "dfmessages/Types.hpp" @@ -41,65 +43,6 @@ namespace dunedaq { -namespace dfmodules { - -/** - * @brief TriggerId is a little class that defines a unique identifier for a - * trigger decision/record It also provides an operator < to be used by map to - * optimise bookkeeping - */ -struct TriggerId -{ - - TriggerId() = default; - - explicit TriggerId(const dfmessages::TriggerDecision& td, - daqdataformats::sequence_number_t s = daqdataformats::TypeDefaults::s_invalid_sequence_number) - : trigger_number(td.trigger_number) - , sequence_number(s) - , run_number(td.run_number) - { - ; - } - explicit TriggerId(daqdataformats::Fragment& f) - : trigger_number(f.get_trigger_number()) - , sequence_number(f.get_sequence_number()) - , run_number(f.get_run_number()) - { - ; - } - - daqdataformats::trigger_number_t trigger_number; - daqdataformats::sequence_number_t sequence_number; - daqdataformats::run_number_t run_number; - - bool operator<(const TriggerId& other) const noexcept - { - return std::tuple(trigger_number, sequence_number, run_number) < - std::tuple(other.trigger_number, other.sequence_number, other.run_number); - } - - friend std::ostream& operator<<(std::ostream& out, const TriggerId& id) noexcept - { - out << id.trigger_number << '-' << id.sequence_number << '/' << id.run_number; - return out; - } - - friend TraceStreamer& operator<<(TraceStreamer& out, const TriggerId& id) noexcept - { - return out << id.trigger_number << '.' << id.sequence_number << "/" << id.run_number; - } - - friend std::istream& operator>>(std::istream& in, TriggerId& id) - { - char t1, t2; - in >> id.trigger_number >> t1 >> id.sequence_number >> t2 >> id.run_number; - return in; - } -}; - -} // namespace dfmodules - /** * @brief Unexpected trigger decision */ @@ -117,7 +60,7 @@ ERS_DECLARE_ISSUE(dfmodules, ///< Namespace ERS_DECLARE_ISSUE(dfmodules, ///< Namespace TimedOutTriggerDecision, ///< Issue class name "trigger id: " << trigger_id << " generate at: " << trigger_timestamp << " timed out", ///< Message - ((dfmodules::TriggerId)trigger_id) ///< Message parameters + ((dfmessages::TriggerId)trigger_id) ///< Message parameters ((daqdataformats::timestamp_t)trigger_timestamp) ///< Message parameters ) @@ -127,7 +70,7 @@ ERS_DECLARE_ISSUE(dfmodules, ///< Namespace ERS_DECLARE_ISSUE(dfmodules, ///< Namespace UnexpectedFragment, ///< Issue class name "Unexpected Fragment for triggerID " << trigger_id << ", type " << fragment_type << ", " << source_id, - ((dfmodules::TriggerId)trigger_id) ///< Message parameters + ((dfmessages::TriggerId)trigger_id) ///< Message parameters ((daqdataformats::fragment_type_t)fragment_type) ///< Message parameters ((daqdataformats::SourceID)source_id) ///< Message parameters ) @@ -138,7 +81,7 @@ ERS_DECLARE_ISSUE(dfmodules, ///< Namespace ERS_DECLARE_ISSUE(dfmodules, ///< Namespace DuplicatedTriggerDecision, ///< Issue class name "Duplicated trigger ID " << trigger_id, - ((dfmodules::TriggerId)trigger_id) ///< Message parameters + ((dfmessages::TriggerId)trigger_id) ///< Message parameters ) /** @@ -147,7 +90,7 @@ ERS_DECLARE_ISSUE(dfmodules, ///< Namespace ERS_DECLARE_ISSUE(dfmodules, ///< Namespace AbandonedTriggerDecision, ///< Issue class name "trigger ID " << trigger_id << " could not be sent to writing and it's lost", - ((dfmodules::TriggerId)trigger_id) ///< Message parameters + ((dfmessages::TriggerId)trigger_id) ///< Message parameters ) /** @@ -156,7 +99,8 @@ ERS_DECLARE_ISSUE(dfmodules, ///< Namespace ERS_DECLARE_ISSUE(dfmodules, ///< Namespace IncompleteTriggerRecord , ///< Issue class name "sending incomplete TriggerRecord downstream " << optional_stop_time_phrase << " (trigger/run_number=" << id << ", " << num_frags_present << " of " << num_components_requested << " fragments included)", - ((std::string)optional_stop_time_phrase)((dfmodules::TriggerId)id)((int)num_frags_present)((int)num_components_requested) ///< Message parameters + ((std::string)optional_stop_time_phrase)((dfmessages::TriggerId)id)((int)num_frags_present)( + (int)num_components_requested) ///< Message parameters ) /** @@ -203,11 +147,12 @@ class TRBModule : public dunedaq::appfwk::DAQModule using trigger_record_ptr_t = std::unique_ptr; using trigger_record_sender_t = iomanager::SenderConcept; + using trb_complete_sender_t = iomanager::SenderConcept; void trigger_decision_callback(dfmessages::TriggerDecision& td); void fragments_callback(std::unique_ptr& frag); - trigger_record_ptr_t extract_trigger_record(const TriggerId&); + trigger_record_ptr_t extract_trigger_record(const dfmessages::TriggerId&); // build_trigger_record will allocate memory and then orphan it to the caller // via the returned pointer Plese note that the method will destroy the memory // saved in the bookkeeping map @@ -217,7 +162,7 @@ class TRBModule : public dunedaq::appfwk::DAQModule bool dispatch_data_requests(dfmessages::DataRequest, const daqdataformats::SourceID&); - bool send_trigger_record(const TriggerId&); + bool send_trigger_record(const dfmessages::TriggerId&); // this creates a trigger record and send it bool check_stale_requests(); @@ -241,6 +186,7 @@ class TRBModule : public dunedaq::appfwk::DAQModule // Configuration const appmodel::TRBConf* m_trb_conf; std::chrono::milliseconds m_tr_queue_timeout; + std::chrono::milliseconds m_trb_complete_timeout; std::chrono::milliseconds m_dreq_queue_timeout; std::string m_reply_connection; size_t m_max_open_trigger_records; @@ -252,6 +198,7 @@ class TRBModule : public dunedaq::appfwk::DAQModule // Output connections std::shared_ptr m_trigger_record_output; + std::shared_ptr m_trb_complete_output; mutable std::mutex m_map_sourceid_connections_mutex; std::map> m_map_sourceid_connections; ///< Mappinng between SourceID and connections @@ -259,14 +206,14 @@ class TRBModule : public dunedaq::appfwk::DAQModule using clock_type = std::chrono::steady_clock; std::mutex m_trigger_records_mutex; clock_type::time_point m_last_bookkeeping{}; - std::map> m_trigger_records; + std::map> m_trigger_records; std::condition_variable m_open_trigger_record_cv; // Data request properties daqdataformats::timestamp_diff_t m_max_sequence_length; // Run information - std::unique_ptr m_run_number = nullptr; + std::atomic m_run_number{ 0 }; // Monitoring related variables std::mutex m_mon_mutex; @@ -297,7 +244,7 @@ class TRBModule : public dunedaq::appfwk::DAQModule mutable std::atomic m_td_processing_us = { 0 }; // in between calls mutable std::atomic m_fragment_processing_us = { 0 }; // in between calls - + mutable std::atomic m_trmon_request_counter = { 0 }; mutable std::atomic m_trmon_sent_counter = { 0 }; diff --git a/schema/dfmodules/opmon/DFOModule.proto b/schema/dfmodules/opmon/DFOModule.proto index 42c83ac5..5ecaa532 100644 --- a/schema/dfmodules/opmon/DFOModule.proto +++ b/schema/dfmodules/opmon/DFOModule.proto @@ -5,20 +5,19 @@ package dunedaq.dfmodules.opmon; // regular metric published byt the DFO message DFOInfo { + // status metrics + uint64 pending_trigger_decisions = 1; // Present number of TriggerDecisions being worked on by the dataflow - // counters - uint64 tokens_received = 1; - uint64 decisions_received = 2; - uint64 decisions_sent = 3; + // counters + uint64 statuses_received = 20; + uint64 decisions_received = 21; + uint64 decisions_sent = 22; + uint64 decisions_completed = 23; // time management of the decision thread - uint64 waiting_for_decision = 10 ; // Time spent waiting on Trigger Decisions, in microseconds - uint64 deciding_destination = 11 ; // Time spent making a decision on the receving DF app, in microseconds - uint64 forwarding_decision = 12 ; // Time spent sending the Trigger Decision to TRBs, in microseconds - - // time management of the token thread - uint64 waiting_for_token = 15 ; // Time spent waiting in token thread for tokens, in microseconds - uint64 processing_token = 16 ; // Time spent in token thread updating data structure, in microseconds + uint64 waiting_for_decision = 30 ; // Time spent waiting on Trigger Decisions, in microseconds + uint64 deciding_destination = 31 ; // Time spent making a decision on the receving DF app, in microseconds + uint64 forwarding_decision = 32 ; // Time spent sending the Trigger Decision to TRBs, in microseconds } diff --git a/schema/dfmodules/opmon/DataflowStatusModule.proto b/schema/dfmodules/opmon/DataflowStatusModule.proto new file mode 100755 index 00000000..189c0c9d --- /dev/null +++ b/schema/dfmodules/opmon/DataflowStatusModule.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; + +package dunedaq.dfmodules.opmon; + + +// regular metric published byt the DataflowStateModule +message DataflowStatusInfo { + + // counters + uint64 decisions_received = 1; + uint64 trb_completions_received = 2; + uint64 tokens_received = 3; + uint64 requests_received = 4; + uint64 status_messages_sent = 5; + uint64 decisions_sent = 6; + + // errors + uint64 duplicate_decisions_received = 7; + uint64 unexpected_trb_completions_received = 8; + uint64 unexpected_tokens_received = 9; + uint64 early_tokens_received = 10; + +} \ No newline at end of file diff --git a/src/TriggerRecordBuilderData.cpp b/src/TriggerRecordBuilderData.cpp deleted file mode 100644 index 7f44ac7d..00000000 --- a/src/TriggerRecordBuilderData.cpp +++ /dev/null @@ -1,226 +0,0 @@ -/** - * @file TriggerRecordBuilderData.cpp TriggerRecordBuilderData Class Implementation - * - * The TriggerRecordBuilderData class represents the current state of a dataflow application's Trigger Record buffers - * for use by the DFO. - * - * This is part of the DUNE DAQ Application Framework, copyright 2020. - * Licensing/copyright details are in the COPYING file that you should have - * received with this code. - */ - -#include "dfmodules/TriggerRecordBuilderData.hpp" -#include "dfmodules/opmon/TRBuilderData.pb.h" - -#include "logging/Logging.hpp" - -#include -#include -#include -#include - -/** - * @brief Name used by TRACE TLOG calls from this source file - */ -#define TRACE_NAME "TRBData" // NOLINT - -namespace dunedaq { -namespace dfmodules { - -TriggerRecordBuilderData::TriggerRecordBuilderData(std::string connection_name, size_t busy_threshold) - : m_busy_threshold(busy_threshold) - , m_free_threshold(busy_threshold) - , m_is_busy(false) - , m_in_error(false) - , m_connection_name(connection_name) -{} - -TriggerRecordBuilderData::TriggerRecordBuilderData(std::string connection_name, - size_t busy_threshold, - size_t free_threshold) - : m_busy_threshold(busy_threshold) - , m_free_threshold(busy_threshold) - , m_is_busy(false) - , m_in_error(false) - , m_connection_name(connection_name) -{ - if (busy_threshold < free_threshold) - throw dfmodules::DFOThresholdsNotConsistent(ERS_HERE, busy_threshold, free_threshold); -} - -std::shared_ptr -TriggerRecordBuilderData::extract_assignment(daqdataformats::trigger_number_t trigger_number) -{ - std::shared_ptr dec_ptr; - auto lk = std::lock_guard(m_assigned_trigger_decisions_mutex); - for (auto it = m_assigned_trigger_decisions.begin(); it != m_assigned_trigger_decisions.end(); ++it) { - if ((*it)->decision.trigger_number == trigger_number) { - dec_ptr = *it; - m_assigned_trigger_decisions.erase(it); - break; - } - } - - if (m_assigned_trigger_decisions.size() < m_free_threshold.load()) - m_is_busy.store(false); - - return dec_ptr; -} - -std::shared_ptr -TriggerRecordBuilderData::get_assignment(daqdataformats::trigger_number_t trigger_number) const -{ - auto lk = std::lock_guard(m_assigned_trigger_decisions_mutex); - for (auto ptr : m_assigned_trigger_decisions) { - if (ptr->decision.trigger_number == trigger_number) { - return ptr; - } - } - - return nullptr; -} - -std::shared_ptr -TriggerRecordBuilderData::complete_assignment(daqdataformats::trigger_number_t trigger_number, - std::function metadata_fun) -{ - - auto dec_ptr = extract_assignment(trigger_number); - - if (dec_ptr == nullptr) - throw AssignedTriggerDecisionNotFound(ERS_HERE, trigger_number, m_connection_name); - - auto now = std::chrono::steady_clock::now(); - auto time = std::chrono::duration_cast(now - dec_ptr->assigned_time); - { - auto lk = std::lock_guard(m_latency_info_mutex); - m_latency_info.emplace_back(now, time); - - if (m_latency_info.size() > 1000) - m_latency_info.pop_front(); - } - - if (metadata_fun) - metadata_fun(m_metadata); - - ++m_complete_counter; - auto completion_time = - std::chrono::duration_cast(now - dec_ptr->assigned_time); - if (completion_time.count() < m_min_complete_time.load()) - m_min_complete_time.store(completion_time.count()); - if (completion_time.count() > m_max_complete_time.load()) - m_max_complete_time.store(completion_time.count()); - - opmon::TRCompleteInfo i; - i.set_completion_time(completion_time.count()); - i.set_tr_number( dec_ptr->decision.trigger_number ); - i.set_run_number( dec_ptr->decision.run_number ); - i.set_trigger_type( dec_ptr->decision.trigger_type ); - publish( std::move(i), {}, opmonlib::to_level(opmonlib::EntryOpMonLevel::kEventDriven) ); - - return dec_ptr; -} - -std::list> -TriggerRecordBuilderData::flush() -{ - - auto lk = std::lock_guard(m_assigned_trigger_decisions_mutex); - std::list> ret; - - for (const auto& td : m_assigned_trigger_decisions) { - ret.push_back(td); - } - m_assigned_trigger_decisions.clear(); - - auto stat_lock = std::lock_guard(m_latency_info_mutex); - m_latency_info.clear(); - m_is_busy = false; - - m_in_error = false; - m_metadata = nlohmann::json(); - - return ret; -} - -std::shared_ptr -TriggerRecordBuilderData::make_assignment(dfmessages::TriggerDecision decision) -{ - return std::make_shared(decision, m_connection_name); -} - -void -TriggerRecordBuilderData::add_assignment(std::shared_ptr assignment) -{ - auto lk = std::lock_guard(m_assigned_trigger_decisions_mutex); - - if (is_in_error()) - throw NoSlotsAvailable(ERS_HERE, assignment->decision.trigger_number, m_connection_name); - - m_assigned_trigger_decisions.push_back(assignment); - TLOG_DEBUG(13) << "Size of assigned_trigger_decision list is " << m_assigned_trigger_decisions.size(); - - if (m_assigned_trigger_decisions.size() >= m_busy_threshold.load()) { - m_is_busy.store(true); - } -} - -void -TriggerRecordBuilderData::generate_opmon_data() -{ - metric_t info; - info.set_min_time_since_assignment( std::numeric_limits::max() ); - info.set_max_time_since_assignment(0); - - time_counter_t time = 0; - - auto lk = std::unique_lock(m_assigned_trigger_decisions_mutex); - info.set_outstanding_decisions(m_assigned_trigger_decisions.size()); - auto current_time = std::chrono::steady_clock::now(); - for (const auto& dec_ptr : m_assigned_trigger_decisions) { - auto us_since_assignment = - std::chrono::duration_cast(current_time - dec_ptr->assigned_time); - time += us_since_assignment.count(); - if (us_since_assignment.count() < info.min_time_since_assignment()) - info.set_min_time_since_assignment(us_since_assignment.count()); - if (us_since_assignment.count() > info.max_time_since_assignment()) - info.set_max_time_since_assignment(us_since_assignment.count()); - } - lk.unlock(); - - info.set_total_time_since_assignment(time); - - // estimate of the capcity - auto completed_trigger_records = m_complete_counter.exchange(0); - if ( completed_trigger_records > 0 ) { - m_last_average_time = 1e-6*0.5*(m_min_complete_time.exchange(0) + m_max_complete_time.exchange(0)); // in seconds - } - - if ( m_last_average_time > 0. ) { - // prediction rate metrics - info.set_capacity_rate( 0.5*(m_busy_threshold.load()+m_free_threshold.load())/m_last_average_time ); - } - - publish(std::move(info)); - -} - -std::chrono::microseconds -TriggerRecordBuilderData::average_latency(std::chrono::steady_clock::time_point since) const -{ - auto lk = std::lock_guard(m_latency_info_mutex); - std::chrono::microseconds sum = std::chrono::microseconds(0); - size_t count = 0; - for (auto it = m_latency_info.rbegin(); it != m_latency_info.rend(); ++it) { - if (it->first < since) - break; - - count++; - sum += it->second; - } - - return sum / count; -} - -} // namespace dfmodules -} // namespace dunedaq diff --git a/src/dfmodules/AssignedTriggerDecision.hpp b/src/dfmodules/AssignedTriggerDecision.hpp new file mode 100755 index 00000000..c9dc2aff --- /dev/null +++ b/src/dfmodules/AssignedTriggerDecision.hpp @@ -0,0 +1,34 @@ +/** + * @file AssignedTriggerDecision.hpp + * + * This is part of the DUNE DAQ Software Suite, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#ifndef DFMODULES_SRC_DFMODULES_ASSIGNEDTRIGGERDECISION_HPP_ +#define DFMODULES_SRC_DFMODULES_ASSIGNEDTRIGGERDECISION_HPP_ + +#include "dfmessages/TriggerDecision.hpp" + +#include +#include + +namespace dunedaq::dfmodules { + + struct AssignedTriggerDecision +{ + dfmessages::TriggerDecision decision; + std::chrono::steady_clock::time_point assigned_time; + std::string connection_name; + AssignedTriggerDecision(dfmessages::TriggerDecision dec, std::string conn_name) + : decision(dec) + , assigned_time(std::chrono::steady_clock::now()) + , connection_name(conn_name) + { + } +}; + +} // namespace dunedaq::dfmodules + +#endif // DFMODULES_SRC_DFMODULES_ASSIGNEDTRIGGERDECISION_HPP_ diff --git a/src/dfmodules/CommonIssues.hpp b/src/dfmodules/CommonIssues.hpp index ffe8a8fd..fb5a957a 100644 --- a/src/dfmodules/CommonIssues.hpp +++ b/src/dfmodules/CommonIssues.hpp @@ -65,6 +65,13 @@ ERS_DECLARE_ISSUE_BASE(dfmodules, ((std::string)name), ((size_t)run_number)) +ERS_DECLARE_ISSUE_BASE(dfmodules, + DuplicateTriggerDecision, + appfwk::GeneralDAQModuleIssue, + "Received duplicate TriggerDecision message for trigger number " << trigger_number << " in run " + << run_number, + ((std::string)name), + ((uint32_t)trigger_number)((uint32_t)run_number)) /** * @brief Data Request sender lookup failed */ diff --git a/src/dfmodules/DFOTriggerCounter.hpp b/src/dfmodules/DFOTriggerCounter.hpp new file mode 100755 index 00000000..8ba1ae81 --- /dev/null +++ b/src/dfmodules/DFOTriggerCounter.hpp @@ -0,0 +1,43 @@ +/** + * @file DFOTriggerCounter.hpp + * + * This is part of the DUNE DAQ Software Suite, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#ifndef DFMODULES_SRC_DFMODULES_DFOTRIGGERCOUNTER_HPP_ +#define DFMODULES_SRC_DFMODULES_DFOTRIGGERCOUNTER_HPP_ + +#include "trgdataformats/TriggerCandidateData.hpp" + +#include +#include +#include +#include + +namespace dunedaq::dfmodules { + +struct DFOTriggerCounter +{ + std::atomic received{ 0 }; + std::atomic completed{ 0 }; + + static std::set unpack_types( + decltype(dfmessages::TriggerDecision::trigger_type) t) + { + std::set results; + if (t == dfmessages::TypeDefaults::s_invalid_trigger_type) + return results; + const std::bitset<64> bits(t); + for (size_t i = 0; i < bits.size(); ++i) { + if (bits[i]) + results.insert((trgdataformats::TriggerCandidateData::Type)i); + } + return results; + } +}; + +} // namespace dunedaq::dfmodules + +#endif // DFMODULES_SRC_DFMODULES_DFOTRIGGERCOUNTER_HPP_ diff --git a/src/dfmodules/ReceivedDataflowStatus.hpp b/src/dfmodules/ReceivedDataflowStatus.hpp new file mode 100755 index 00000000..f2013d38 --- /dev/null +++ b/src/dfmodules/ReceivedDataflowStatus.hpp @@ -0,0 +1,74 @@ +/** + * @file ReceivedDataflowStatus.hpp + * + * This is part of the DUNE DAQ Software Suite, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#ifndef DFMODULES_SRC_DFMODULES_RECEIVEDDATAFLOWSTATUS_HPP_ +#define DFMODULES_SRC_DFMODULES_RECEIVEDDATAFLOWSTATUS_HPP_ + +#include "dfmessages/DataflowStatus.hpp" + +#include +#include +#include +#include +#include + +namespace dunedaq::dfmodules { + +struct ReceivedDataflowStatus +{ + dfmessages::DataflowStatus status; + std::chrono::steady_clock::time_point received_time; + std::mutex status_update_mutex; + std::condition_variable status_update_cv; + std::shared_ptr timeout_thread; + std::atomic status_updated{ true }; + std::chrono::milliseconds status_timeout_ms{ 5000 }; + + ReceivedDataflowStatus(dfmessages::DataflowStatus s, std::chrono::milliseconds status_timeout) + : status(s) + , received_time(std::chrono::steady_clock::now()) + , timeout_thread( + std::make_shared(std::bind_front(&ReceivedDataflowStatus::dataflow_status_timeout_proc, this))) + , status_timeout_ms(status_timeout) + { + } + + ~ReceivedDataflowStatus() + { + if (timeout_thread && timeout_thread->joinable()) { + timeout_thread->request_stop(); + status_update_cv.notify_all(); + timeout_thread->join(); + } + } + + void update(dfmessages::DataflowStatus s) + { + std::lock_guard lock(status_update_mutex); + status = s; + status_updated.store(true); + received_time = std::chrono::steady_clock::now(); + status_update_cv.notify_all(); + } + + void dataflow_status_timeout_proc(std::stop_token stoken) + { + while (!stoken.stop_requested()) { + + std::unique_lock lock(status_update_mutex); + status_update_cv.wait_for(lock, status_timeout_ms); + if (std::chrono::steady_clock::now() - received_time >= status_timeout_ms) { + status_updated.store(false); + } + } + } +}; + +} // namespace dunedaq::dfmodules + +#endif // DFMODULES_SRC_DFMODULES_RECEIVEDDATAFLOWSTATUS_HPP_ diff --git a/src/dfmodules/TriggerRecordBuilderData.hpp b/src/dfmodules/TriggerRecordBuilderData.hpp deleted file mode 100644 index 17cd4e2e..00000000 --- a/src/dfmodules/TriggerRecordBuilderData.hpp +++ /dev/null @@ -1,131 +0,0 @@ -/** - * @file TriggerRecordBuilderData.hpp TriggerRecordBuilderData Class - * - * The TriggerRecordBuilderData class represents the current state of a TRBModule's Trigger Record buffers - * for use by the DFO. - * - * This is part of the DUNE DAQ Application Framework, copyright 2020. - * Licensing/copyright details are in the COPYING file that you should have - * received with this code. - */ - -#ifndef DFMODULES_SRC_DFMODULES_TRIGGERRECORDBUILDERDATA_HPP_ -#define DFMODULES_SRC_DFMODULES_TRIGGERRECORDBUILDERDATA_HPP_ - -#include "daqdataformats/Types.hpp" -#include "dfmessages/TriggerDecision.hpp" -#include "dfmodules/opmon/TRBuilderData.pb.h" - -#include "ers/Issue.hpp" -#include "nlohmann/json.hpp" -#include "opmonlib/MonitorableObject.hpp" -#include "logging/Logging.hpp" // NOTE: if ISSUES ARE DECLARED BEFORE include logging/Logging.hpp, TLOG_DEBUG< -#include -#include -#include -#include -#include -#include -#include -#include - -namespace dunedaq { -// Disable coverage checking LCOV_EXCL_START -ERS_DECLARE_ISSUE(dfmodules, - DFOThresholdsNotConsistent, - "Busy threshold (" << busy << ") is smaller than free threshold (" << free << ')', - ((size_t)busy)((size_t)free)) -ERS_DECLARE_ISSUE(dfmodules, - AssignedTriggerDecisionNotFound, - "The Trigger Decision with trigger number " - << trigger_number << " was not found for dataflow application at " << connection_name, - ((daqdataformats::trigger_number_t)trigger_number)((std::string)connection_name)) -ERS_DECLARE_ISSUE(dfmodules, - NoSlotsAvailable, - "The Trigger Decision with trigger number " - << trigger_number << " could not be assigned to the dataflow application at " << connection_name - << " because no slots were available.", - ((daqdataformats::trigger_number_t)trigger_number)((std::string)connection_name)) -// Re-enable coverage checking LCOV_EXCL_STOP - -namespace dfmodules { -struct AssignedTriggerDecision -{ - dfmessages::TriggerDecision decision; - std::chrono::steady_clock::time_point assigned_time; - std::string connection_name; - - AssignedTriggerDecision(dfmessages::TriggerDecision dec, std::string conn_name) - : decision(dec) - , assigned_time(std::chrono::steady_clock::now()) - , connection_name(conn_name) - {} -}; - -class TriggerRecordBuilderData : public opmonlib::MonitorableObject -{ -public: - TriggerRecordBuilderData() = default; - TriggerRecordBuilderData(std::string connection_name, size_t busy_threshold); - TriggerRecordBuilderData(std::string connection_name, size_t busy_threshold, size_t free_threshold); - - TriggerRecordBuilderData(TriggerRecordBuilderData const&) = delete; - TriggerRecordBuilderData(TriggerRecordBuilderData&&) = delete; - TriggerRecordBuilderData& operator=(TriggerRecordBuilderData const&) = delete; - TriggerRecordBuilderData& operator=(TriggerRecordBuilderData&&) = delete; - - ~TriggerRecordBuilderData() = default; - - bool is_busy() const { return m_in_error || m_is_busy; } - size_t used_slots() const { return m_assigned_trigger_decisions.size(); } - - size_t busy_threshold() const { return m_busy_threshold.load(); } - size_t free_threshold() const { return m_free_threshold.load(); } - - std::shared_ptr get_assignment(daqdataformats::trigger_number_t trigger_number) const; - std::shared_ptr extract_assignment(daqdataformats::trigger_number_t trigger_number); - std::shared_ptr make_assignment(dfmessages::TriggerDecision decision); - void add_assignment(std::shared_ptr assignment); - std::shared_ptr complete_assignment( - daqdataformats::trigger_number_t trigger_number, - std::function metadata_fun = nullptr); - std::list> flush(); - - void generate_opmon_data() override; - - std::chrono::microseconds average_latency(std::chrono::steady_clock::time_point since) const; - - bool is_in_error() const { return m_in_error.load(); } - void set_in_error(bool err) { m_in_error = err; } - -private: - std::atomic m_busy_threshold{ 0 }; - std::atomic m_free_threshold{ std::numeric_limits::max() }; - std::atomic m_is_busy{ false }; - std::list> m_assigned_trigger_decisions; - mutable std::mutex m_assigned_trigger_decisions_mutex; - - // TODO: Eric Flumerfelt Dec-03-2021: Replace with circular buffer - std::list> m_latency_info; - mutable std::mutex m_latency_info_mutex; - - std::atomic m_in_error{ true }; - - nlohmann::json m_metadata; - std::string m_connection_name{ "" }; - - // monitoring - using metric_t = dunedaq::dfmodules::opmon::DFApplicationInfo; - using const_time_counter_t = std::invoke_result::type; - using time_counter_t = std::remove_const::type; - std::atomic m_complete_counter{ 0 }; - std::atomic m_min_complete_time{ std::numeric_limits::max() }, m_max_complete_time{ 0 }; // in us - double m_last_average_time{0.}; -}; -} // namespace dfmodules -} // namespace dunedaq - -#endif // DFMODULES_SRC_DFMODULES_TRIGGERRECORDBUILDERDATA_HPP_ diff --git a/test/config/datafloworchestrator_test.data.xml b/test/config/datafloworchestrator_test.data.xml index be3b19b1..f8c5099b 100755 --- a/test/config/datafloworchestrator_test.data.xml +++ b/test/config/datafloworchestrator_test.data.xml @@ -63,7 +63,7 @@ - + @@ -86,24 +86,38 @@ - + + - - + + + + - + + + + + + + + + + + + @@ -113,6 +127,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + @@ -272,12 +310,88 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -304,6 +418,14 @@ + + + + + + + + @@ -331,9 +453,9 @@ + - @@ -361,27 +483,39 @@ - + - + - + - + + + + + + + - + + + + + + + @@ -398,6 +532,21 @@ + + + + + + + + + + + + + + + diff --git a/test/config/dataflowstatus_test.data.xml b/test/config/dataflowstatus_test.data.xml new file mode 100755 index 00000000..b626b8e6 --- /dev/null +++ b/test/config/dataflowstatus_test.data.xml @@ -0,0 +1,523 @@ + + + + + + + + + + + + + + + + + + + + + + + + +]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/config/dfoprotocol_test.data.xml b/test/config/dfoprotocol_test.data.xml new file mode 100755 index 00000000..39b7788f --- /dev/null +++ b/test/config/dfoprotocol_test.data.xml @@ -0,0 +1,829 @@ + + + + + + + + + + + + + + + + + + + + + + + + +]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/config/triggerrecordbuilder_test.data.xml b/test/config/triggerrecordbuilder_test.data.xml new file mode 100755 index 00000000..3507a086 --- /dev/null +++ b/test/config/triggerrecordbuilder_test.data.xml @@ -0,0 +1,506 @@ + + + + + + + + + + + + + + + + + + + + + + + + +]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/unittest/DFOModule_test.cxx b/unittest/DFOModule_test.cxx index 690ca08a..c09f0744 100644 --- a/unittest/DFOModule_test.cxx +++ b/unittest/DFOModule_test.cxx @@ -9,8 +9,9 @@ #include "DFOModule.hpp" -#include "dfmessages/TriggerDecisionToken.hpp" #include "dfmessages/TriggerInhibit.hpp" +#include "dfmessages/DataflowStatus.hpp" +#include "dfmessages/DataflowStatusRequest.hpp" #include "dfmodules/CommonIssues.hpp" #include "dfmodules/opmon/DFOModule.pb.h" #include "iomanager/IOManager.hpp" @@ -65,45 +66,36 @@ struct CfgFixture BOOST_FIXTURE_TEST_SUITE(DFOModule_test, CfgFixture) +std::vector received_decisions; void -send_init_token(std::string connection_name = "trigdec_0") +recv_trigdec(const dfmessages::TriggerDecision& decision) { - dfmessages::TriggerDecisionToken token; - token.run_number = 0; - token.trigger_number = 0; - token.decision_destination = connection_name; - - TLOG() << "Sending Init TriggerDecisionToken to DFO"; - get_iom_sender("token")->send(std::move(token), iomanager::Sender::s_block); + TLOG() << "Received TriggerDecision with trigger number " << decision.trigger_number << " from DFO"; + received_decisions.push_back(decision); } + +std::atomic busy_signal_recvd = false; void -send_token(dfmessages::trigger_number_t trigger_number, - std::string connection_name = "trigdec_0", - bool different_run = false) +recv_triginh(const dfmessages::TriggerInhibit& inhibit) { - dfmessages::TriggerDecisionToken token; - token.run_number = different_run ? 2 : 1; - token.trigger_number = trigger_number; - token.decision_destination = connection_name; - - TLOG() << "Sending TriggerDecisionToken with trigger number " << trigger_number << " to DFO"; - get_iom_sender("token")->send(std::move(token), iomanager::Sender::s_block); + TLOG() << "Received TriggerInhibit with busy=" << std::boolalpha << inhibit.busy << " from DFO"; + busy_signal_recvd = inhibit.busy; } +std::unordered_map received_status_requests; void -recv_trigdec(const dfmessages::TriggerDecision& decision) +recv_status_request(const dfmessages::DataflowStatusRequest& request) { - TLOG() << "Received TriggerDecision with trigger number " << decision.trigger_number << " from DFO"; - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - send_token(decision.trigger_number); + TLOG() << "Received DataflowStatusRequest with trigger number " << request.trigger_number << " from DFO"; + received_status_requests[request.trigger_number] = request; } -std::atomic busy_signal_recvd = false; void -recv_triginh(const dfmessages::TriggerInhibit& inhibit) +send_status(dfmessages::DataflowStatus status) { - TLOG() << "Received TriggerInhibit with busy=" << std::boolalpha << inhibit.busy << " from DFO"; - busy_signal_recvd = inhibit.busy; + auto iom = iomanager::IOManager::get(); + TLOG() << "Sending DataflowStatus with trigger number " << status.trigger_number; + iom->get_sender("df_status")->send(std::move(status), iomanager::Sender::s_block); } void @@ -145,6 +137,11 @@ BOOST_AUTO_TEST_CASE(Commands) opmgr.register_node("dfo", dfo); dfo->init(cfgMgr); + // start sends a TriggerInhibit + auto iom = iomanager::IOManager::get(); + auto inh_recv = iom->get_receiver("triginh"); + inh_recv->add_callback(recv_triginh); + appfwk::DAQModule::CommandData_t null_data; appfwk::DAQModule::CommandData_t start_data; start_data.emplace("run", 1); @@ -155,14 +152,14 @@ BOOST_AUTO_TEST_CASE(Commands) dfo->execute_command("scrap", null_data); auto metric = get_dfo_info(); - BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 0); BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); BOOST_REQUIRE_EQUAL(metric.forwarding_decision(), 0); BOOST_REQUIRE_EQUAL(metric.waiting_for_decision(), 0); BOOST_REQUIRE_EQUAL(metric.deciding_destination(), 0); - BOOST_REQUIRE_EQUAL(metric.waiting_for_token(), 0); - BOOST_REQUIRE_EQUAL(metric.processing_token(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); } BOOST_AUTO_TEST_CASE(DataFlow) @@ -182,57 +179,237 @@ BOOST_AUTO_TEST_CASE(DataFlow) dec_recv->add_callback(recv_trigdec); auto inh_recv = iom->get_receiver("triginh"); inh_recv->add_callback(recv_triginh); + auto req_recv = iom->get_receiver("df_status_request"); + req_recv->add_callback(recv_status_request); send_trigdec(1, true); std::this_thread::sleep_for(std::chrono::milliseconds(50)); - send_token(999, "trigdec_0", true); - send_token(9999, "trigdec_0", true); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - // Note: Counters are reset by calling get_dfo_info! auto metric = get_dfo_info(); - BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); dfo->execute_command("start", start_data); - send_init_token(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + send_trigdec(1); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + dfmessages::DataflowStatus status; + status.decision_destination = "trigdec_0"; + status.request_destination = "df_status_request"; + status.trigger_number = 0; + status.run_number = 1; + + status.trigger_type_mask = 0xFFFFFFFF; + status.is_busy = false; + status.busy_threshold = 1; + status.free_threshold = 0; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + BOOST_REQUIRE_EQUAL(received_status_requests.size(), 1); + BOOST_REQUIRE(received_status_requests.find(1) != received_status_requests.end()); + BOOST_REQUIRE_EQUAL(received_status_requests[1].reply_destination, "df_status"); + + status.trigger_number = 1; + send_status(status); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), false); metric = get_dfo_info(); - BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + + BOOST_REQUIRE_EQUAL(received_decisions.size(), 1); + BOOST_REQUIRE_EQUAL(received_decisions[0].trigger_number, 1); + status.trigger_number = 0; + status.triggers_building.insert(1); + status.is_busy = true; + received_decisions.clear(); + + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); - send_trigdec(2); - send_trigdec(3); + status.triggers_building.clear(); + status.recently_completed_triggers.insert(1); + status.is_busy = false; + send_status(status); std::this_thread::sleep_for(std::chrono::milliseconds(50)); - send_trigdec(4); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), false); metric = get_dfo_info(); - BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); - BOOST_REQUIRE_EQUAL(metric.decisions_received(), 2); - BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 2); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + auto start_time = std::chrono::steady_clock::now(); + dfo->execute_command("drain_dataflow", null_data); + dfo->execute_command("scrap", null_data); + BOOST_REQUIRE(std::chrono::steady_clock::now() - start_time < std::chrono::milliseconds(200)); + + dec_recv->remove_callback(); + inh_recv->remove_callback(); + req_recv->remove_callback(); +} + +BOOST_AUTO_TEST_CASE(DelayedStatusResponse) +{ + TLOG() << "Test case DelayedStatusResponse BEGIN"; + auto dfo = appfwk::make_module("DFOModule", "test"); + opmgr.register_node("dfo", dfo); + dfo->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + dfo->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + received_decisions.clear(); + received_status_requests.clear(); + busy_signal_recvd.store(false); + + auto dec_recv = iom->get_receiver("trigdec_0"); + dec_recv->add_callback(recv_trigdec); + auto inh_recv = iom->get_receiver("triginh"); + inh_recv->add_callback(recv_triginh); + auto req_recv = iom->get_receiver("df_status_request"); + req_recv->add_callback(recv_status_request); + + dfo->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); - BOOST_REQUIRE(busy_signal_recvd.load()); - std::this_thread::sleep_for(std::chrono::milliseconds(400)); + auto metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + // Send trigger decision + send_trigdec(10); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); metric = get_dfo_info(); - BOOST_REQUIRE_EQUAL(metric.tokens_received(), 3); BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + // Send initial heartbeat status + dfmessages::DataflowStatus status; + status.decision_destination = "trigdec_0"; + status.request_destination = "df_status_request"; + status.trigger_number = 0; + status.run_number = 1; + status.trigger_type_mask = 0xFFFFFFFF; + status.is_busy = false; + status.busy_threshold = 1; + status.free_threshold = 0; + + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); + + BOOST_REQUIRE_EQUAL(received_status_requests.size(), 1); + BOOST_REQUIRE(received_status_requests.find(10) != received_status_requests.end()); + + // Simulate delayed response by NOT sending status for trigger 10 immediately + // Wait to ensure timeout period could expire + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // The decision should still be pending (not sent yet) + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + // Now send the delayed status response for trigger 10 + status.trigger_number = 10; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Now the decision should be dispatched + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.statuses_received(), 1); BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); - BOOST_REQUIRE(!busy_signal_recvd.load()); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + + BOOST_REQUIRE_EQUAL(received_decisions.size(), 1); + BOOST_REQUIRE_EQUAL(received_decisions[0].trigger_number, 10); + + // Complete the trigger + status.trigger_number = 0; + status.triggers_building.insert(10); + status.is_busy = true; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + status.triggers_building.clear(); + status.recently_completed_triggers.insert(10); + status.is_busy = false; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); dfo->execute_command("drain_dataflow", null_data); dfo->execute_command("scrap", null_data); dec_recv->remove_callback(); inh_recv->remove_callback(); + req_recv->remove_callback(); + TLOG() << "Test case DelayedStatusResponse END"; } -BOOST_AUTO_TEST_CASE(SendTrigDecFailed) +BOOST_AUTO_TEST_CASE(UnresponsiveDFAppRecovery) { + TLOG() << "Test case UnresponsiveDFAppRecovery BEGIN"; auto dfo = appfwk::make_module("DFOModule", "test"); opmgr.register_node("dfo", dfo); dfo->init(cfgMgr); @@ -243,30 +420,141 @@ BOOST_AUTO_TEST_CASE(SendTrigDecFailed) dfo->execute_command("conf", null_data); + auto iom = iomanager::IOManager::get(); + received_decisions.clear(); + received_status_requests.clear(); + busy_signal_recvd.store(false); + + auto dec_recv = iom->get_receiver("trigdec_0"); + dec_recv->add_callback(recv_trigdec); + auto inh_recv = iom->get_receiver("triginh"); + inh_recv->add_callback(recv_triginh); + auto req_recv = iom->get_receiver("df_status_request"); + req_recv->add_callback(recv_status_request); + dfo->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); - send_init_token("invalid_connection"); + // Send initial heartbeat status from the DF app + dfmessages::DataflowStatus status; + status.decision_destination = "trigdec_0"; + status.request_destination = "df_status_request"; + status.trigger_number = 0; + status.run_number = 1; + status.trigger_type_mask = 0xFFFFFFFF; + status.is_busy = false; + status.busy_threshold = 2; + status.free_threshold = 0; + + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + // Send first trigger decision - should be assigned to trigdec_0 + send_trigdec(30); std::this_thread::sleep_for(std::chrono::milliseconds(50)); - send_trigdec(1); + auto metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + + // Respond with status for trigger 30 + status.trigger_number = 30; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(received_decisions.size() >= 1); + BOOST_REQUIRE_EQUAL(received_decisions.back().trigger_number, 30); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + + // Mark trigger 30 as building + status.trigger_number = 0; + status.triggers_building.insert(30); + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Now simulate the DF app becoming unresponsive by: + // 1. Waiting for the heartbeat timeout (5 seconds as per ReceivedDataflowStatus default) + // 2. NOT sending any more status messages + TLOG() << "Waiting for DF app to become stale (5+ seconds)..."; + std::this_thread::sleep_for(std::chrono::milliseconds(5500)); + + // At this point, the DF app status should be marked as stale (status_updated = false) + // Send a second trigger decision - it should not be assigned to trigdec_0 because it's stale + send_trigdec(31); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + + // The DFO should not request status, since the DF app is unresponsive + BOOST_REQUIRE(received_status_requests.find(31) == received_status_requests.end()); + + // Wait for request timeout - decision should not be dispatched + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); // No decision sent because no DF app available + + // The DFO should now be in a busy state because no DF apps are responsive + BOOST_REQUIRE_EQUAL(busy_signal_recvd.load(), true); + + // Now simulate recovery: the DF app becomes responsive again with fresh heartbeat + status.trigger_number = 0; + status.triggers_building.clear(); // Clear the stale state + status.triggers_building.insert(30); // Still building trigger 30 + status.is_busy = false; + send_status(status); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // The DF app should now be marked as available again (status_updated = true) + // The pending trigger decision 31 should now be assignable + BOOST_REQUIRE(received_status_requests.find(31) != received_status_requests.end()); + + // Respond to status request for trigger 31 + status.trigger_number = 31; + send_status(status); std::this_thread::sleep_for(std::chrono::milliseconds(150)); - auto info = get_dfo_info(); - BOOST_REQUIRE_EQUAL(info.tokens_received(), 0); - BOOST_REQUIRE_EQUAL(info.decisions_received(), 1); - BOOST_REQUIRE_EQUAL(info.decisions_sent(), 0); + // Now trigger 31 should be dispatched + BOOST_REQUIRE(received_decisions.size() >= 2); + bool found_31 = false; + for (const auto& dec : received_decisions) { + if (dec.trigger_number == 31) { + found_31 = true; + break; + } + } + BOOST_REQUIRE(found_31); - // FWIW, tell the DFO to retry the invalid connection - send_token(1000, "invalid_connection"); + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 2); + + // Complete both triggers + status.trigger_number = 0; + status.triggers_building.insert(31); + send_status(status); std::this_thread::sleep_for(std::chrono::milliseconds(50)); - // Token for unknown dataflow app - send_token(1000); + status.triggers_building.clear(); + status.recently_completed_triggers.insert(30); + status.recently_completed_triggers.insert(31); + send_status(status); std::this_thread::sleep_for(std::chrono::milliseconds(50)); + metric = get_dfo_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_completed(), 2); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + dfo->execute_command("drain_dataflow", null_data); dfo->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + inh_recv->remove_callback(); + req_recv->remove_callback(); + TLOG() << "Test case UnresponsiveDFAppRecovery END"; } BOOST_AUTO_TEST_SUITE_END() diff --git a/unittest/DFOProtocol_test.cxx b/unittest/DFOProtocol_test.cxx new file mode 100755 index 00000000..7b072b40 --- /dev/null +++ b/unittest/DFOProtocol_test.cxx @@ -0,0 +1,1385 @@ +/** + * @file DFOProtocol_test.cxx Integration test for the DFO protocol + * with multiple DFO and DataflowStatus modules. + * + * This is part of the DUNE DAQ Application Framework, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#include "DFOModule.hpp" +#include "DataflowStatusModule.hpp" + +#include "dfmessages/DataflowStatus.hpp" +#include "dfmessages/DataflowStatusRequest.hpp" +#include "dfmessages/TRBCompletion.hpp" +#include "dfmessages/TriggerDecision.hpp" +#include "dfmessages/TriggerDecisionToken.hpp" +#include "dfmessages/TriggerInhibit.hpp" +#include "dfmodules/CommonIssues.hpp" +#include "dfmodules/opmon/DFOModule.pb.h" +#include "dfmodules/opmon/DataflowStatusModule.pb.h" +#include "iomanager/IOManager.hpp" +#include "iomanager/Sender.hpp" +#include "opmonlib/TestOpMonManager.hpp" + +#define BOOST_TEST_MODULE DFOProtocol_test // NOLINT + +#include "boost/test/unit_test.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dunedaq::dfmodules; + +namespace dunedaq { + +struct EnvFixture +{ + EnvFixture() { setenv("DUNEDAQ_PARTITION", "partition_name", 0); } +}; +BOOST_TEST_GLOBAL_FIXTURE(EnvFixture); + +struct CfgFixture +{ + CfgFixture() + { + std::string oksConfig = "oksconflibs:test/config/dfoprotocol_test.data.xml"; + std::string appName = "TestApp"; + std::string sessionName = "partition_name"; + cfgMgr = std::make_shared(oksConfig, appName, sessionName); + get_iomanager()->configure(sessionName, cfgMgr->get_queues(), cfgMgr->get_networkconnections(), nullptr, opmgr); + } + ~CfgFixture() { get_iomanager()->reset(); } + + void collect_opmon_entries() + { + opmgr.collect(); + auto opmon_facility = opmgr.get_backend_facility(); + entries = opmon_facility->get_entries(std::regex(".*(DFOInfo|DataflowStatusInfo)")); + BOOST_REQUIRE_EQUAL(entries.size(), 6); + } + + std::unordered_map get_dfo_info() + { + auto entry = entries.begin(); + std::unordered_map output; + for (; entry != entries.end(); ++entry) { + if (entry->measurement() != "dunedaq.dfmodules.opmon.DFOInfo") { + continue; + } + auto name = entry->origin().substructure()[0]; + output[name] = opmonlib::from_entry(*entry); + } + + BOOST_REQUIRE_EQUAL(output.size(), 3); + return output; + } + + std::unordered_map get_dfs_info() + { + auto entry = entries.begin(); + std::unordered_map output; + for (; entry != entries.end(); ++entry) { + if (entry->measurement() != "dunedaq.dfmodules.opmon.DataflowStatusInfo") { + continue; + } + auto name = entry->origin().substructure()[0]; + output[name] = opmonlib::from_entry(*entry); + } + + BOOST_REQUIRE_EQUAL(output.size(), 3); + return output; + } + + std::list entries; + dunedaq::opmonlib::TestOpMonManager opmgr; + std::shared_ptr cfgMgr; +}; + +BOOST_FIXTURE_TEST_SUITE(DFOProtocol_test, CfgFixture) + +// Thread-safe collectors for received messages +struct MessageCollector +{ + std::mutex mutex; + std::vector trigger_decisions; + std::vector trigger_inhibits; + std::map> latest_inhibit_state; + + void reset() + { + std::lock_guard lock(mutex); + trigger_decisions.clear(); + trigger_inhibits.clear(); + latest_inhibit_state.clear(); + } + + void add_trigger_decision(const dfmessages::TriggerDecision& decision) + { + std::lock_guard lock(mutex); + trigger_decisions.push_back(decision); + } + + void add_trigger_inhibit(const std::string& source, const dfmessages::TriggerInhibit& inhibit) + { + std::lock_guard lock(mutex); + trigger_inhibits.push_back(inhibit); + latest_inhibit_state[source].store(inhibit.busy); + } + + std::vector get_decisions() + { + std::lock_guard lock(mutex); + return trigger_decisions; + } + + std::vector get_inhibits() + { + std::lock_guard lock(mutex); + return trigger_inhibits; + } + + bool get_latest_inhibit(const std::string& source) { return latest_inhibit_state[source].load(); } + + size_t count_decisions() + { + std::lock_guard lock(mutex); + return trigger_decisions.size(); + } +}; + +static MessageCollector df1_collector; +static MessageCollector df2_collector; +static MessageCollector df3_collector; +static MessageCollector dfo_collector; + +static void +df1_recv_trigdec(const dfmessages::TriggerDecision& decision) +{ + TLOG() << "DF1 received TriggerDecision " << decision.trigger_number; + df1_collector.add_trigger_decision(decision); +} + +static void +df2_recv_trigdec(const dfmessages::TriggerDecision& decision) +{ + TLOG() << "DF2 received TriggerDecision " << decision.trigger_number; + df2_collector.add_trigger_decision(decision); +} + +static void +df3_recv_trigdec(const dfmessages::TriggerDecision& decision) +{ + TLOG() << "DF3 received TriggerDecision " << decision.trigger_number; + df3_collector.add_trigger_decision(decision); +} + +static void +dfo1_recv_inhibit(const dfmessages::TriggerInhibit& inhibit) +{ + TLOG() << "DFO1 sent TriggerInhibit: busy=" << std::boolalpha << inhibit.busy; + dfo_collector.add_trigger_inhibit("dfo1", inhibit); +} + +static void +dfo2_recv_inhibit(const dfmessages::TriggerInhibit& inhibit) +{ + TLOG() << "DFO2 sent TriggerInhibit: busy=" << std::boolalpha << inhibit.busy; + dfo_collector.add_trigger_inhibit("dfo2", inhibit); +} + +static void +dfo3_recv_inhibit(const dfmessages::TriggerInhibit& inhibit) +{ + TLOG() << "DFO3 sent TriggerInhibit: busy=" << std::boolalpha << inhibit.busy; + dfo_collector.add_trigger_inhibit("dfo3", inhibit); +} + +static void +send_trigdec_to_all_dfos(dfmessages::trigger_number_t trigger_number, uint32_t run_number = 1) +{ + auto iom = iomanager::IOManager::get(); + + for (const auto& conn : { "dfo1_trigdec_in", "dfo2_trigdec_in", "dfo3_trigdec_in" }) { + dfmessages::TriggerDecision td; + td.trigger_number = trigger_number; + td.run_number = run_number; + td.trigger_timestamp = trigger_number * 1000; + td.trigger_type = 1; + td.readout_type = dfmessages::ReadoutType::kLocalized; + + auto sender = iom->get_sender(conn); + sender->send(std::move(td), iomanager::Sender::s_block); + } + TLOG() << "Sent TriggerDecision " << trigger_number << " to all DFOs"; +} + +static void +send_trb_completion(const std::string& df_module, + dfmessages::trigger_number_t trigger_number, + dfmessages::sequence_number_t sequence_number = 0, + size_t max_sequence_number = 0) +{ + auto iom = iomanager::IOManager::get(); + + dfmessages::TRBCompletion completion; + completion.trigger_id = dfmessages::TriggerId{ 1, trigger_number, sequence_number }; + completion.source_id = daqdataformats::SourceID(daqdataformats::SourceID::Subsystem::kTRBuilder, 1); + completion.trigger_record_max_sequence_number = max_sequence_number; + + auto sender = iom->get_sender(df_module + "_trb_completion_in"); + sender->send(std::move(completion), iomanager::Sender::s_block); + TLOG() << "Sent TRBCompletion for trigger " << trigger_number << " to " << df_module; +} + +static void +send_token(const std::string& df_module, + dfmessages::trigger_number_t trigger_number, + dfmessages::sequence_number_t sequence_number = 0) +{ + auto iom = iomanager::IOManager::get(); + + dfmessages::TriggerDecisionToken token; + token.trigger_id = dfmessages::TriggerId{ 1, trigger_number, sequence_number }; + token.writer_identifier = "test_writer"; + token.data_size = 1234; + + auto sender = iom->get_sender(df_module + "_token_in"); + sender->send(std::move(token), iomanager::Sender::s_block); + TLOG() << "Sent Token for trigger " << trigger_number << " to " << df_module; +} + +BOOST_AUTO_TEST_CASE(StableAlgorithmOptimalConditions) +{ + TLOG() << "Test case StableAlgorithmOptimalConditions BEGIN"; + + // Create modules + auto dfo1 = appfwk::make_module("DFOModule", "dfo1"); + auto dfo2 = appfwk::make_module("DFOModule", "dfo2"); + auto dfo3 = appfwk::make_module("DFOModule", "dfo3"); + auto df1 = appfwk::make_module("DataflowStatusModule", "df1"); + auto df2 = appfwk::make_module("DataflowStatusModule", "df2"); + auto df3 = appfwk::make_module("DataflowStatusModule", "df3"); + + // Register with OpMon + opmgr.register_node("dfo1", dfo1); + opmgr.register_node("dfo2", dfo2); + opmgr.register_node("dfo3", dfo3); + opmgr.register_node("df1", df1); + opmgr.register_node("df2", df2); + opmgr.register_node("df3", df3); + + // Initialize all modules + dfo1->init(cfgMgr); + dfo2->init(cfgMgr); + dfo3->init(cfgMgr); + df1->init(cfgMgr); + df2->init(cfgMgr); + df3->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + // Configure all modules + dfo1->execute_command("conf", null_data); + dfo2->execute_command("conf", null_data); + dfo3->execute_command("conf", null_data); + df1->execute_command("conf", null_data); + df2->execute_command("conf", null_data); + df3->execute_command("conf", null_data); + + // Set up receivers + auto iom = iomanager::IOManager::get(); + df1_collector.reset(); + df2_collector.reset(); + df3_collector.reset(); + dfo_collector.reset(); + + auto df1_recv = iom->get_receiver("df1_trigdec_out"); + df1_recv->add_callback(df1_recv_trigdec); + auto df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + auto df3_recv = iom->get_receiver("df3_trigdec_out"); + df3_recv->add_callback(df3_recv_trigdec); + + auto dfo1_inh_recv = iom->get_receiver("dfo1_triginh_out"); + dfo1_inh_recv->add_callback(dfo1_recv_inhibit); + auto dfo2_inh_recv = iom->get_receiver("dfo2_triginh_out"); + dfo2_inh_recv->add_callback(dfo2_recv_inhibit); + auto dfo3_inh_recv = iom->get_receiver("dfo3_triginh_out"); + dfo3_inh_recv->add_callback(dfo3_recv_inhibit); + + // Start all modules + dfo1->execute_command("start", start_data); + dfo2->execute_command("start", start_data); + dfo3->execute_command("start", start_data); + df1->execute_command("start", start_data); + df2->execute_command("start", start_data); + df3->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send a series of trigger decisions + for (dfmessages::trigger_number_t trig = 1; trig <= 9; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // Check that decisions were distributed across DF modules + size_t df1_count = df1_collector.count_decisions(); + size_t df2_count = df2_collector.count_decisions(); + size_t df3_count = df3_collector.count_decisions(); + + TLOG() << "DF1 received " << df1_count << " decisions"; + TLOG() << "DF2 received " << df2_count << " decisions"; + TLOG() << "DF3 received " << df3_count << " decisions"; + + // Each DF module should have received some decisions + BOOST_REQUIRE(df1_count > 0); + BOOST_REQUIRE(df2_count > 0); + BOOST_REQUIRE(df3_count > 0); + + // Total should be 9 * 3 (each DFO sends to one DF, but the same DF for each trigger) + // Actually, each trigger is sent to 3 DFOs, but stable algorithm means all DFOs pick the same DF + // So total decisions received should be 9 (one per trigger) + BOOST_REQUIRE_EQUAL(df1_count + df2_count + df3_count, 9); + + // Verify stable algorithm: for each trigger, all DFOs chose the same DF module + auto df1_decisions = df1_collector.get_decisions(); + auto df2_decisions = df2_collector.get_decisions(); + auto df3_decisions = df3_collector.get_decisions(); + + std::set df1_triggers; + std::set df2_triggers; + std::set df3_triggers; + + for (const auto& dec : df1_decisions) + df1_triggers.insert(dec.trigger_number); + for (const auto& dec : df2_decisions) + df2_triggers.insert(dec.trigger_number); + for (const auto& dec : df3_decisions) + df3_triggers.insert(dec.trigger_number); + + // No trigger should appear in multiple DF modules + for (const auto& trig : df1_triggers) { + BOOST_REQUIRE(df2_triggers.find(trig) == df2_triggers.end()); + BOOST_REQUIRE(df3_triggers.find(trig) == df3_triggers.end()); + } + for (const auto& trig : df2_triggers) { + BOOST_REQUIRE(df3_triggers.find(trig) == df3_triggers.end()); + } + + collect_opmon_entries(); + auto dfo_infos = get_dfo_info(); + auto dfs_infos = get_dfs_info(); + auto dfo1_metrics = dfo_infos["dfo1"]; + auto dfo2_metrics = dfo_infos["dfo2"]; + auto dfo3_metrics = dfo_infos["dfo3"]; + auto df1_metrics = dfs_infos["df1"]; + auto df2_metrics = dfs_infos["df2"]; + auto df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 9); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 9); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 9); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 9); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 9); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 9); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 0); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 9 * 3); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 9 * 3); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 9 * 3); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), df1_triggers.size() * 2); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), df2_triggers.size() * 2); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), df3_triggers.size() * 2); + + // Complete all triggers + for (const auto& trig : df1_triggers) { + send_trb_completion("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df2_triggers) { + send_trb_completion("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df3_triggers) { + send_trb_completion("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + collect_opmon_entries(); + dfo_infos = get_dfo_info(); + dfs_infos = get_dfs_info(); + dfo1_metrics = dfo_infos["dfo1"]; + dfo2_metrics = dfo_infos["dfo2"]; + dfo3_metrics = dfo_infos["dfo3"]; + df1_metrics = dfs_infos["df1"]; + df2_metrics = dfs_infos["df2"]; + df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 9); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 9); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 9); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 0); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), 0); + + // Stop all modules + dfo1->execute_command("drain_dataflow", null_data); + dfo2->execute_command("drain_dataflow", null_data); + dfo3->execute_command("drain_dataflow", null_data); + df1->execute_command("stop", null_data); + df2->execute_command("stop", null_data); + df3->execute_command("stop", null_data); + + dfo1->execute_command("scrap", null_data); + dfo2->execute_command("scrap", null_data); + dfo3->execute_command("scrap", null_data); + df1->execute_command("scrap", null_data); + df2->execute_command("scrap", null_data); + df3->execute_command("scrap", null_data); + + df1_recv->remove_callback(); + df2_recv->remove_callback(); + df3_recv->remove_callback(); + dfo1_inh_recv->remove_callback(); + dfo2_inh_recv->remove_callback(); + dfo3_inh_recv->remove_callback(); + + TLOG() << "Test case StableAlgorithmOptimalConditions END"; +} + +BOOST_AUTO_TEST_CASE(AllDFOsAgreeOnAssignments) +{ + TLOG() << "Test case AllDFOsAgreeOnAssignments BEGIN"; + + // Create modules + auto dfo1 = appfwk::make_module("DFOModule", "dfo1"); + auto dfo2 = appfwk::make_module("DFOModule", "dfo2"); + auto dfo3 = appfwk::make_module("DFOModule", "dfo3"); + auto df1 = appfwk::make_module("DataflowStatusModule", "df1"); + auto df2 = appfwk::make_module("DataflowStatusModule", "df2"); + auto df3 = appfwk::make_module("DataflowStatusModule", "df3"); + + opmgr.register_node("dfo1", dfo1); + opmgr.register_node("dfo2", dfo2); + opmgr.register_node("dfo3", dfo3); + opmgr.register_node("df1", df1); + opmgr.register_node("df2", df2); + opmgr.register_node("df3", df3); + + dfo1->init(cfgMgr); + dfo2->init(cfgMgr); + dfo3->init(cfgMgr); + df1->init(cfgMgr); + df2->init(cfgMgr); + df3->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + dfo1->execute_command("conf", null_data); + dfo2->execute_command("conf", null_data); + dfo3->execute_command("conf", null_data); + df1->execute_command("conf", null_data); + df2->execute_command("conf", null_data); + df3->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + df1_collector.reset(); + df2_collector.reset(); + df3_collector.reset(); + dfo_collector.reset(); + + auto df1_recv = iom->get_receiver("df1_trigdec_out"); + df1_recv->add_callback(df1_recv_trigdec); + auto df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + auto df3_recv = iom->get_receiver("df3_trigdec_out"); + df3_recv->add_callback(df3_recv_trigdec); + + auto dfo1_inh_recv = iom->get_receiver("dfo1_triginh_out"); + dfo1_inh_recv->add_callback(dfo1_recv_inhibit); + auto dfo2_inh_recv = iom->get_receiver("dfo2_triginh_out"); + dfo2_inh_recv->add_callback(dfo2_recv_inhibit); + auto dfo3_inh_recv = iom->get_receiver("dfo3_triginh_out"); + dfo3_inh_recv->add_callback(dfo3_recv_inhibit); + + dfo1->execute_command("start", start_data); + dfo2->execute_command("start", start_data); + dfo3->execute_command("start", start_data); + df1->execute_command("start", start_data); + df2->execute_command("start", start_data); + df3->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send trigger decisions + for (dfmessages::trigger_number_t trig = 10; trig <= 15; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // Verify each trigger was received by exactly one DF module + auto df1_decisions = df1_collector.get_decisions(); + auto df2_decisions = df2_collector.get_decisions(); + auto df3_decisions = df3_collector.get_decisions(); + + std::map trigger_counts; + for (const auto& dec : df1_decisions) + trigger_counts[dec.trigger_number]++; + for (const auto& dec : df2_decisions) + trigger_counts[dec.trigger_number]++; + for (const auto& dec : df3_decisions) + trigger_counts[dec.trigger_number]++; + + // Each trigger should have been received exactly once + for (dfmessages::trigger_number_t trig = 10; trig <= 15; ++trig) { + BOOST_REQUIRE_EQUAL(trigger_counts[trig], 1); + } + + // Check that all DFOs have matching inhibit states + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + bool dfo1_busy = dfo_collector.get_latest_inhibit("dfo1"); + bool dfo2_busy = dfo_collector.get_latest_inhibit("dfo2"); + bool dfo3_busy = dfo_collector.get_latest_inhibit("dfo3"); + + TLOG() << "DFO1 inhibit state: " << std::boolalpha << dfo1_busy; + TLOG() << "DFO2 inhibit state: " << std::boolalpha << dfo2_busy; + TLOG() << "DFO3 inhibit state: " << std::boolalpha << dfo3_busy; + + // All DFOs should agree on busy state + BOOST_REQUIRE_EQUAL(dfo1_busy, dfo2_busy); + BOOST_REQUIRE_EQUAL(dfo2_busy, dfo3_busy); + + // Complete triggers + std::set df1_triggers; + std::set df2_triggers; + std::set df3_triggers; + + for (const auto& dec : df1_decisions) + df1_triggers.insert(dec.trigger_number); + for (const auto& dec : df2_decisions) + df2_triggers.insert(dec.trigger_number); + for (const auto& dec : df3_decisions) + df3_triggers.insert(dec.trigger_number); + + collect_opmon_entries(); + auto dfo_infos = get_dfo_info(); + auto dfs_infos = get_dfs_info(); + auto dfo1_metrics = dfo_infos["dfo1"]; + auto dfo2_metrics = dfo_infos["dfo2"]; + auto dfo3_metrics = dfo_infos["dfo3"]; + auto df1_metrics = dfs_infos["df1"]; + auto df2_metrics = dfs_infos["df2"]; + auto df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 6); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 0); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 6 * 3); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 6 * 3); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 6 * 3); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), df1_triggers.size() * 2); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), df2_triggers.size() * 2); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), df3_triggers.size() * 2); + + + for (const auto& trig : df1_triggers) { + send_trb_completion("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df2_triggers) { + send_trb_completion("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df3_triggers) { + send_trb_completion("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + collect_opmon_entries(); + dfo_infos = get_dfo_info(); + dfs_infos = get_dfs_info(); + dfo1_metrics = dfo_infos["dfo1"]; + dfo2_metrics = dfo_infos["dfo2"]; + dfo3_metrics = dfo_infos["dfo3"]; + df1_metrics = dfs_infos["df1"]; + df2_metrics = dfs_infos["df2"]; + df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 6); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 0); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), 0); + + dfo1->execute_command("drain_dataflow", null_data); + dfo2->execute_command("drain_dataflow", null_data); + dfo3->execute_command("drain_dataflow", null_data); + df1->execute_command("stop", null_data); + df2->execute_command("stop", null_data); + df3->execute_command("stop", null_data); + + dfo1->execute_command("scrap", null_data); + dfo2->execute_command("scrap", null_data); + dfo3->execute_command("scrap", null_data); + df1->execute_command("scrap", null_data); + df2->execute_command("scrap", null_data); + df3->execute_command("scrap", null_data); + + df1_recv->remove_callback(); + df2_recv->remove_callback(); + df3_recv->remove_callback(); + dfo1_inh_recv->remove_callback(); + dfo2_inh_recv->remove_callback(); + dfo3_inh_recv->remove_callback(); + + TLOG() << "Test case AllDFOsAgreeOnAssignments END"; +} + +BOOST_AUTO_TEST_CASE(DelayedDFOStillMatchesAssignments) +{ + TLOG() << "Test case DelayedDFOStillMatchesAssignments BEGIN"; + + // Create modules + auto dfo1 = appfwk::make_module("DFOModule", "dfo1"); + auto dfo2 = appfwk::make_module("DFOModule", "dfo2"); + auto dfo3 = appfwk::make_module("DFOModule", "dfo3"); + auto df1 = appfwk::make_module("DataflowStatusModule", "df1"); + auto df2 = appfwk::make_module("DataflowStatusModule", "df2"); + auto df3 = appfwk::make_module("DataflowStatusModule", "df3"); + + opmgr.register_node("dfo1", dfo1); + opmgr.register_node("dfo2", dfo2); + opmgr.register_node("dfo3", dfo3); + opmgr.register_node("df1", df1); + opmgr.register_node("df2", df2); + opmgr.register_node("df3", df3); + + dfo1->init(cfgMgr); + dfo2->init(cfgMgr); + dfo3->init(cfgMgr); + df1->init(cfgMgr); + df2->init(cfgMgr); + df3->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + dfo1->execute_command("conf", null_data); + dfo2->execute_command("conf", null_data); + dfo3->execute_command("conf", null_data); + df1->execute_command("conf", null_data); + df2->execute_command("conf", null_data); + df3->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + df1_collector.reset(); + df2_collector.reset(); + df3_collector.reset(); + dfo_collector.reset(); + + auto df1_recv = iom->get_receiver("df1_trigdec_out"); + df1_recv->add_callback(df1_recv_trigdec); + auto df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + auto df3_recv = iom->get_receiver("df3_trigdec_out"); + df3_recv->add_callback(df3_recv_trigdec); + + auto dfo1_inh_recv = iom->get_receiver("dfo1_triginh_out"); + dfo1_inh_recv->add_callback(dfo1_recv_inhibit); + auto dfo2_inh_recv = iom->get_receiver("dfo2_triginh_out"); + dfo2_inh_recv->add_callback(dfo2_recv_inhibit); + auto dfo3_inh_recv = iom->get_receiver("dfo3_triginh_out"); + dfo3_inh_recv->add_callback(dfo3_recv_inhibit); + + // Start DFO1 and DFO2 first, delay DFO3 + dfo1->execute_command("start", start_data); + dfo2->execute_command("start", start_data); + df1->execute_command("start", start_data); + df2->execute_command("start", start_data); + df3->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send triggers to DFO1 and DFO2 only initially + auto send_to_dfo1_dfo2 = [&iom](dfmessages::trigger_number_t trigger_number) { + for (const auto& conn : { "dfo1_trigdec_in", "dfo2_trigdec_in" }) { + dfmessages::TriggerDecision td; + td.trigger_number = trigger_number; + td.run_number = 1; + td.trigger_timestamp = trigger_number * 1000; + td.trigger_type = 1; + td.readout_type = dfmessages::ReadoutType::kLocalized; + + auto sender = iom->get_sender(conn); + sender->send(std::move(td), iomanager::Sender::s_block); + } + }; + + for (dfmessages::trigger_number_t trig = 20; trig <= 25; ++trig) { + send_to_dfo1_dfo2(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Now start DFO3 (delayed) + dfo3->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // Send same triggers to DFO3 + for (dfmessages::trigger_number_t trig = 20; trig <= 25; ++trig) { + dfmessages::TriggerDecision td; + td.trigger_number = trig; + td.run_number = 1; + td.trigger_timestamp = trig * 1000; + td.trigger_type = 1; + td.readout_type = dfmessages::ReadoutType::kLocalized; + + auto sender = iom->get_sender("dfo3_trigdec_in"); + sender->send(std::move(td), iomanager::Sender::s_block); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // Verify that each trigger was still assigned to exactly one DF module + auto df1_decisions = df1_collector.get_decisions(); + auto df2_decisions = df2_collector.get_decisions(); + auto df3_decisions = df3_collector.get_decisions(); + + std::map trigger_counts; + for (const auto& dec : df1_decisions) + trigger_counts[dec.trigger_number]++; + for (const auto& dec : df2_decisions) + trigger_counts[dec.trigger_number]++; + for (const auto& dec : df3_decisions) + trigger_counts[dec.trigger_number]++; + + // Each trigger should have been received exactly once despite delayed DFO3 + for (dfmessages::trigger_number_t trig = 20; trig <= 25; ++trig) { + TLOG() << "Trigger " << trig << " received " << trigger_counts[trig] << " times"; + BOOST_REQUIRE_EQUAL(trigger_counts[trig], 1); + } + + // Complete triggers + std::set df1_triggers; + std::set df2_triggers; + std::set df3_triggers; + + for (const auto& dec : df1_decisions) + df1_triggers.insert(dec.trigger_number); + for (const auto& dec : df2_decisions) + df2_triggers.insert(dec.trigger_number); + for (const auto& dec : df3_decisions) + df3_triggers.insert(dec.trigger_number); + + collect_opmon_entries(); + auto dfo_infos = get_dfo_info(); + auto dfs_infos = get_dfs_info(); + auto dfo1_metrics = dfo_infos["dfo1"]; + auto dfo2_metrics = dfo_infos["dfo2"]; + auto dfo3_metrics = dfo_infos["dfo3"]; + auto df1_metrics = dfs_infos["df1"]; + auto df2_metrics = dfs_infos["df2"]; + auto df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 6); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 0); // DFO3 has status information for received triggers + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 0); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 6 * 2); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 6 * 2); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 6 * 2); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), df1_triggers.size() * 1); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), df2_triggers.size() * 1); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), df3_triggers.size() * 1); + + for (const auto& trig : df1_triggers) { + send_trb_completion("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df1", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df2_triggers) { + send_trb_completion("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df2", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + for (const auto& trig : df3_triggers) { + send_trb_completion("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + send_token("df3", trig); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + collect_opmon_entries(); + dfo_infos = get_dfo_info(); + dfs_infos = get_dfs_info(); + dfo1_metrics = dfo_infos["dfo1"]; + dfo2_metrics = dfo_infos["dfo2"]; + dfo3_metrics = dfo_infos["dfo3"]; + df1_metrics = dfs_infos["df1"]; + df2_metrics = dfs_infos["df2"]; + df3_metrics = dfs_infos["df3"]; + + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(dfo1_metrics.decisions_completed(), 6); + BOOST_REQUIRE_EQUAL(dfo2_metrics.decisions_completed(), 6); + BOOST_REQUIRE_EQUAL(dfo3_metrics.decisions_completed(), 6); + + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.tokens_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.tokens_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.tokens_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.trb_completions_received(), df1_triggers.size()); + BOOST_REQUIRE_EQUAL(df2_metrics.trb_completions_received(), df2_triggers.size()); + BOOST_REQUIRE_EQUAL(df3_metrics.trb_completions_received(), df3_triggers.size()); + BOOST_REQUIRE_EQUAL(df1_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.requests_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.requests_received(), 0); + BOOST_REQUIRE(df1_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df2_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE(df3_metrics.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(df1_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(df1_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df2_metrics.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(df3_metrics.duplicate_decisions_received(), 0); + + dfo1->execute_command("drain_dataflow", null_data); + dfo2->execute_command("drain_dataflow", null_data); + dfo3->execute_command("drain_dataflow", null_data); + df1->execute_command("stop", null_data); + df2->execute_command("stop", null_data); + df3->execute_command("stop", null_data); + + dfo1->execute_command("scrap", null_data); + dfo2->execute_command("scrap", null_data); + dfo3->execute_command("scrap", null_data); + df1->execute_command("scrap", null_data); + df2->execute_command("scrap", null_data); + df3->execute_command("scrap", null_data); + + df1_recv->remove_callback(); + df2_recv->remove_callback(); + df3_recv->remove_callback(); + dfo1_inh_recv->remove_callback(); + dfo2_inh_recv->remove_callback(); + dfo3_inh_recv->remove_callback(); + + TLOG() << "Test case DelayedDFOStillMatchesAssignments END"; +} + +BOOST_AUTO_TEST_CASE(DFOCrashRecovery) +{ + TLOG() << "Test case DFOCrashRecovery BEGIN"; + + // Create modules + auto dfo1 = appfwk::make_module("DFOModule", "dfo1"); + auto dfo2 = appfwk::make_module("DFOModule", "dfo2"); + auto dfo3 = appfwk::make_module("DFOModule", "dfo3"); + auto df1 = appfwk::make_module("DataflowStatusModule", "df1"); + auto df2 = appfwk::make_module("DataflowStatusModule", "df2"); + auto df3 = appfwk::make_module("DataflowStatusModule", "df3"); + + opmgr.register_node("dfo1", dfo1); + opmgr.register_node("dfo2", dfo2); + opmgr.register_node("dfo3", dfo3); + opmgr.register_node("df1", df1); + opmgr.register_node("df2", df2); + opmgr.register_node("df3", df3); + + dfo1->init(cfgMgr); + dfo2->init(cfgMgr); + dfo3->init(cfgMgr); + df1->init(cfgMgr); + df2->init(cfgMgr); + df3->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + dfo1->execute_command("conf", null_data); + dfo2->execute_command("conf", null_data); + dfo3->execute_command("conf", null_data); + df1->execute_command("conf", null_data); + df2->execute_command("conf", null_data); + df3->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + df1_collector.reset(); + df2_collector.reset(); + df3_collector.reset(); + + auto df1_recv = iom->get_receiver("df1_trigdec_out"); + df1_recv->add_callback(df1_recv_trigdec); + auto df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + auto df3_recv = iom->get_receiver("df3_trigdec_out"); + df3_recv->add_callback(df3_recv_trigdec); + + auto dfo1_inh_recv = iom->get_receiver("dfo1_triginh_out"); + dfo1_inh_recv->add_callback(dfo1_recv_inhibit); + auto dfo2_inh_recv = iom->get_receiver("dfo2_triginh_out"); + dfo2_inh_recv->add_callback(dfo2_recv_inhibit); + auto dfo3_inh_recv = iom->get_receiver("dfo3_triginh_out"); + dfo3_inh_recv->add_callback(dfo3_recv_inhibit); + + // Start all modules + dfo1->execute_command("start", start_data); + dfo2->execute_command("start", start_data); + dfo3->execute_command("start", start_data); + df1->execute_command("start", start_data); + df2->execute_command("start", start_data); + df3->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send a few triggers + for (dfmessages::trigger_number_t trig = 30; trig <= 32; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Simulate DFO2 crash by stopping it abruptly + TLOG() << "Simulating DFO2 crash..."; + dfo2->execute_command("scrap", null_data); + + // Continue sending triggers to remaining DFOs + for (dfmessages::trigger_number_t trig = 33; trig <= 35; ++trig) { + // Send only to DFO1 and DFO3 + for (const auto& conn : { "dfo1_trigdec_in", "dfo3_trigdec_in" }) { + dfmessages::TriggerDecision td; + td.trigger_number = trig; + td.run_number = 1; + td.trigger_timestamp = trig * 1000; + td.trigger_type = 1; + td.readout_type = dfmessages::ReadoutType::kLocalized; + + auto sender = iom->get_sender(conn); + sender->send(std::move(td), iomanager::Sender::s_block); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + // System should continue working with remaining DFOs + auto df1_decisions = df1_collector.get_decisions(); + auto df2_decisions = df2_collector.get_decisions(); + auto df3_decisions = df3_collector.get_decisions(); + + size_t total_decisions = df1_decisions.size() + df2_decisions.size() + df3_decisions.size(); + TLOG() << "Total decisions received: " << total_decisions; + + // Should have received 6 decisions total (triggers 30-35) + BOOST_REQUIRE_EQUAL(total_decisions, 6); + + // Complete triggers + std::set df1_triggers; + std::set df2_triggers; + std::set df3_triggers; + + for (const auto& dec : df1_decisions) + df1_triggers.insert(dec.trigger_number); + for (const auto& dec : df2_decisions) + df2_triggers.insert(dec.trigger_number); + for (const auto& dec : df3_decisions) + df3_triggers.insert(dec.trigger_number); + + for (const auto& trig : df1_triggers) { + send_trb_completion("df1", trig); + send_token("df1", trig); + } + for (const auto& trig : df2_triggers) { + send_trb_completion("df2", trig); + send_token("df2", trig); + } + for (const auto& trig : df3_triggers) { + send_trb_completion("df3", trig); + send_token("df3", trig); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + dfo1->execute_command("drain_dataflow", null_data); + dfo3->execute_command("drain_dataflow", null_data); + df1->execute_command("stop", null_data); + df2->execute_command("stop", null_data); + df3->execute_command("stop", null_data); + + dfo1->execute_command("scrap", null_data); + dfo3->execute_command("scrap", null_data); + df1->execute_command("scrap", null_data); + df2->execute_command("scrap", null_data); + df3->execute_command("scrap", null_data); + + df1_recv->remove_callback(); + df2_recv->remove_callback(); + df3_recv->remove_callback(); + dfo1_inh_recv->remove_callback(); + dfo2_inh_recv->remove_callback(); + dfo3_inh_recv->remove_callback(); + + TLOG() << "Test case DFOCrashRecovery END"; +} + +BOOST_AUTO_TEST_CASE(DataflowStatusModuleCrashAndRecovery) +{ + TLOG() << "Test case DataflowStatusModuleCrashAndRecovery BEGIN"; + + // Create modules + auto dfo1 = appfwk::make_module("DFOModule", "dfo1"); + auto dfo2 = appfwk::make_module("DFOModule", "dfo2"); + auto dfo3 = appfwk::make_module("DFOModule", "dfo3"); + auto df1 = appfwk::make_module("DataflowStatusModule", "df1"); + auto df2 = appfwk::make_module("DataflowStatusModule", "df2"); + auto df3 = appfwk::make_module("DataflowStatusModule", "df3"); + + opmgr.register_node("dfo1", dfo1); + opmgr.register_node("dfo2", dfo2); + opmgr.register_node("dfo3", dfo3); + opmgr.register_node("df1", df1); + opmgr.register_node("df2", df2); + opmgr.register_node("df3", df3); + + dfo1->init(cfgMgr); + dfo2->init(cfgMgr); + dfo3->init(cfgMgr); + df1->init(cfgMgr); + df2->init(cfgMgr); + df3->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + dfo1->execute_command("conf", null_data); + dfo2->execute_command("conf", null_data); + dfo3->execute_command("conf", null_data); + df1->execute_command("conf", null_data); + df2->execute_command("conf", null_data); + df3->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + df1_collector.reset(); + df2_collector.reset(); + df3_collector.reset(); + + auto df1_recv = iom->get_receiver("df1_trigdec_out"); + df1_recv->add_callback(df1_recv_trigdec); + auto df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + auto df3_recv = iom->get_receiver("df3_trigdec_out"); + df3_recv->add_callback(df3_recv_trigdec); + + auto dfo1_inh_recv = iom->get_receiver("dfo1_triginh_out"); + dfo1_inh_recv->add_callback(dfo1_recv_inhibit); + auto dfo2_inh_recv = iom->get_receiver("dfo2_triginh_out"); + dfo2_inh_recv->add_callback(dfo2_recv_inhibit); + auto dfo3_inh_recv = iom->get_receiver("dfo3_triginh_out"); + dfo3_inh_recv->add_callback(dfo3_recv_inhibit); + + // Start all modules + dfo1->execute_command("start", start_data); + dfo2->execute_command("start", start_data); + dfo3->execute_command("start", start_data); + df1->execute_command("start", start_data); + df2->execute_command("start", start_data); + df3->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send a few triggers before crash + for (dfmessages::trigger_number_t trig = 40; trig <= 42; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + auto pre_crash_df1_count = df1_collector.count_decisions(); + auto pre_crash_df2_count = df2_collector.count_decisions(); + auto pre_crash_df3_count = df3_collector.count_decisions(); + + TLOG() << "Before crash - DF1: " << pre_crash_df1_count << ", DF2: " << pre_crash_df2_count + << ", DF3: " << pre_crash_df3_count; + + // Simulate DF2 crash by stopping it + TLOG() << "Simulating DF2 crash..."; + df2->execute_command("stop", null_data); + df2->execute_command("scrap", null_data); + + // Wait for DF2 to timeout (5+ seconds) + TLOG() << "Waiting for DF2 heartbeat timeout..."; + std::this_thread::sleep_for(std::chrono::milliseconds(5500)); + + // Send more triggers - they should be redistributed to DF1 and DF3 only + for (dfmessages::trigger_number_t trig = 43; trig <= 45; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + auto post_crash_df1_count = df1_collector.count_decisions(); + auto post_crash_df2_count = df2_collector.count_decisions(); + auto post_crash_df3_count = df3_collector.count_decisions(); + + TLOG() << "After crash - DF1: " << post_crash_df1_count << ", DF2: " << post_crash_df2_count + << ", DF3: " << post_crash_df3_count; + + // DF2 should not have received any new triggers + BOOST_REQUIRE_EQUAL(post_crash_df2_count, pre_crash_df2_count); + + // DF1 and DF3 should have received the new triggers, plus triggers from df2 + BOOST_REQUIRE(post_crash_df1_count > pre_crash_df1_count || post_crash_df3_count > pre_crash_df3_count); + + // Total should be 6 (triggers 40-45) + BOOST_REQUIRE_EQUAL(post_crash_df1_count + post_crash_df3_count, 6); + + // Now simulate DF2 recovery + TLOG() << "Simulating DF2 recovery..."; + df2 = appfwk::make_module("DataflowStatusModule", "df2"); + opmgr.register_node("df2", df2); + df2->init(cfgMgr); + df2->execute_command("conf", null_data); + + df2_collector.reset(); + df2_recv = iom->get_receiver("df2_trigdec_out"); + df2_recv->add_callback(df2_recv_trigdec); + + df2->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + // Send more triggers - DF2 should start receiving them again + for (dfmessages::trigger_number_t trig = 46; trig <= 48; ++trig) { + send_trigdec_to_all_dfos(trig); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + + auto recovery_df2_count = df2_collector.count_decisions(); + TLOG() << "After recovery - DF2: " << recovery_df2_count; + + // DF2 should have received at least one trigger after recovery + BOOST_REQUIRE(recovery_df2_count > 0); + + // Complete all triggers + auto df1_decisions = df1_collector.get_decisions(); + auto df2_decisions = df2_collector.get_decisions(); + auto df3_decisions = df3_collector.get_decisions(); + + std::set df1_triggers; + std::set df2_triggers; + std::set df3_triggers; + + for (const auto& dec : df1_decisions) + df1_triggers.insert(dec.trigger_number); + for (const auto& dec : df2_decisions) + df2_triggers.insert(dec.trigger_number); + for (const auto& dec : df3_decisions) + df3_triggers.insert(dec.trigger_number); + + for (const auto& trig : df1_triggers) { + send_trb_completion("df1", trig); + send_token("df1", trig); + } + for (const auto& trig : df2_triggers) { + send_trb_completion("df2", trig); + send_token("df2", trig); + } + for (const auto& trig : df3_triggers) { + send_trb_completion("df3", trig); + send_token("df3", trig); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + + dfo1->execute_command("drain_dataflow", null_data); + dfo2->execute_command("drain_dataflow", null_data); + dfo3->execute_command("drain_dataflow", null_data); + df1->execute_command("stop", null_data); + df2->execute_command("stop", null_data); + df3->execute_command("stop", null_data); + + dfo1->execute_command("scrap", null_data); + dfo2->execute_command("scrap", null_data); + dfo3->execute_command("scrap", null_data); + df1->execute_command("scrap", null_data); + df2->execute_command("scrap", null_data); + df3->execute_command("scrap", null_data); + + df1_recv->remove_callback(); + df2_recv->remove_callback(); + df3_recv->remove_callback(); + dfo1_inh_recv->remove_callback(); + dfo2_inh_recv->remove_callback(); + dfo3_inh_recv->remove_callback(); + + TLOG() << "Test case DataflowStatusModuleCrashAndRecovery END"; +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace dunedaq diff --git a/unittest/DataflowStatusModule_test.cxx b/unittest/DataflowStatusModule_test.cxx new file mode 100755 index 00000000..34282b21 --- /dev/null +++ b/unittest/DataflowStatusModule_test.cxx @@ -0,0 +1,937 @@ +/** + * @file DataflowStatusModule_test.cxx Test application that tests and demonstrates + * the functionality of the DataflowStatusModule class. + * + * This is part of the DUNE DAQ Application Framework, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#include "DataflowStatusModule.hpp" + +#include "dfmessages/TriggerDecisionToken.hpp" +#include "dfmessages/TriggerInhibit.hpp" +#include "dfmodules/CommonIssues.hpp" +#include "dfmodules/opmon/DataflowStatusModule.pb.h" +#include "iomanager/IOManager.hpp" +#include "iomanager/Sender.hpp" +#include "opmonlib/TestOpMonManager.hpp" + +#define BOOST_TEST_MODULE DataflowStatusModule_test // NOLINT + +#include "boost/test/unit_test.hpp" + +#include +#include +#include +#include + +using namespace dunedaq::dfmodules; + +namespace dunedaq { + +struct EnvFixture +{ + EnvFixture() { setenv("DUNEDAQ_PARTITION", "partition_name", 0); } +}; +BOOST_TEST_GLOBAL_FIXTURE(EnvFixture); + +struct CfgFixture +{ + CfgFixture() + { + std::string oksConfig = "oksconflibs:test/config/dataflowstatus_test.data.xml"; + std::string appName = "TestApp"; + std::string sessionName = "partition_name"; + cfgMgr = std::make_shared(oksConfig, appName, sessionName); + get_iomanager()->configure(sessionName, cfgMgr->get_queues(), cfgMgr->get_networkconnections(), nullptr, opmgr); + } + ~CfgFixture() { get_iomanager()->reset(); } + + auto get_opmon_info() + { + + opmgr.collect(); + auto opmon_facility = opmgr.get_backend_facility(); + auto list = opmon_facility->get_entries(std::regex(".*DataflowStatusInfo")); + BOOST_REQUIRE_EQUAL(list.size(), 1); + const auto& entry = list.front(); + return opmonlib::from_entry(entry); + } + + opmonlib::TestOpMonManager opmgr; + std::shared_ptr cfgMgr; +}; + +struct ConnectionFixture +{ + ConnectionFixture() = default; + + static void send_trigdec(dfmessages::trigger_number_t trigger_number, bool different_run = false) + { + dfmessages::TriggerDecision td; + td.trigger_number = trigger_number; + td.run_number = different_run ? 2 : 1; + td.trigger_timestamp = 1; + td.trigger_type = 1; + td.readout_type = dfmessages::ReadoutType::kLocalized; + auto iom = iomanager::IOManager::get(); + TLOG() << "Sending TriggerDecision with trigger number " << trigger_number << " from \"DFO\""; + iom->get_sender("trigdec_dfo")->send(std::move(td), iomanager::Sender::s_block); + } + + static void send_token( + dfmessages::trigger_number_t trigger_number, + dfmessages::sequence_number_t sequence_number = 0, + bool different_run = false) + { + dfmessages::TriggerDecisionToken token; + token.trigger_id = dfmessages::TriggerId{ different_run ? 2U : 1U, trigger_number, sequence_number }; + token.writer_identifier = "test_writer"; + token.data_size = 1234; + auto iom = iomanager::IOManager::get(); + auto sender = iom->get_sender("token"); + sender->send(std::move(token), iomanager::Sender::s_block); + } + + static void send_dataflow_status_request(dfmessages::trigger_number_t trigger_number, bool different_run = false) + { + dfmessages::DataflowStatusRequest request; + request.trigger_number = trigger_number; + request.run_number = different_run ? 2 : 1; + request.reply_destination = "df_status"; + auto iom = iomanager::IOManager::get(); + auto sender = iom->get_sender("df_status_request"); + sender->send(std::move(request), iomanager::Sender::s_block); + } + + static void send_trb_completion( + dfmessages::trigger_number_t trigger_number, + dfmessages::sequence_number_t sequence_number = 0, + size_t trigger_record_max_sequence_number = 0, + bool different_run = false) + { + dfmessages::TRBCompletion trb_complete; + trb_complete.trigger_id = dfmessages::TriggerId{ different_run ? 2U : 1U, trigger_number, sequence_number }; + trb_complete.source_id = daqdataformats::SourceID(daqdataformats::SourceID::Subsystem::kTRBuilder, 1); + trb_complete.trigger_record_max_sequence_number = trigger_record_max_sequence_number; + auto iom = iomanager::IOManager::get(); + auto sender = iom->get_sender("trb_completion"); + sender->send(std::move(trb_complete), iomanager::Sender::s_block); + } + + static std::vector s_received_statuses; + static void receive_dataflow_status(const dfmessages::DataflowStatus& status) + { + TLOG() << "Received DataflowStatus with trigger number " << status.trigger_number; + s_received_statuses.push_back(status); + } + static std::vector s_received_decisions; + static void receive_trigger_decision(const dfmessages::TriggerDecision& decision) + { + TLOG() << "Received TriggerDecision with trigger number " << decision.trigger_number; + s_received_decisions.push_back(decision); + } + + static void reset() + { + s_received_statuses.clear(); + s_received_decisions.clear(); + } +}; +std::vector ConnectionFixture::s_received_statuses; +std::vector ConnectionFixture::s_received_decisions; + +BOOST_FIXTURE_TEST_SUITE(DataflowStatusModule_test, CfgFixture) + +BOOST_AUTO_TEST_CASE(CopyAndMoveSemantics) +{ + BOOST_REQUIRE(!std::is_copy_constructible_v); + BOOST_REQUIRE(!std::is_copy_assignable_v); + BOOST_REQUIRE(!std::is_move_constructible_v); + BOOST_REQUIRE(!std::is_move_assignable_v); +} + +BOOST_AUTO_TEST_CASE(Constructor) +{ + TLOG() << "Test case Constructor BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + TLOG() << "Test case Constructor END"; +} + +BOOST_AUTO_TEST_CASE(Init) +{ + TLOG() << "Test case Init BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + dfs->init(cfgMgr); + TLOG() << "Test case Init END"; +} + +BOOST_AUTO_TEST_CASE(Commands) +{ + TLOG() << "Test case Commands BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + TLOG() << "Executing conf command"; + dfs->execute_command("conf", null_data); + TLOG() << "Executing start command"; + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + TLOG() << "Executing stop command"; + dfs->execute_command("stop", null_data); + TLOG() << "Executing scrap command"; + dfs->execute_command("scrap", null_data); + + TLOG() << "Retrieving metrics from opmon"; + auto metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.requests_received(), 0); + BOOST_REQUIRE(metric.status_messages_sent() >= 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + BOOST_REQUIRE_EQUAL(ConnectionFixture::s_received_statuses.back().decision_destination, "trigdec_dfo"); + + TLOG() << "Test case Commands END"; +} + +BOOST_AUTO_TEST_CASE(DataFlow) +{ + TLOG() << "Test case DataFlow BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + + ConnectionFixture::send_trigdec(1, true); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + ConnectionFixture::send_token(999, true); + ConnectionFixture::send_token(9999, true); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + // Note: Counters are reset by calling get_opmon_info! + auto metric = get_opmon_info(); + + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + ConnectionFixture::send_trigdec(2); + ConnectionFixture::send_trigdec(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + ConnectionFixture::send_trigdec(4); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 2); + + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + + ConnectionFixture::send_trb_completion(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(4) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + ConnectionFixture::send_token(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(4) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 1); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 1); + + ConnectionFixture::send_trb_completion(2); + ConnectionFixture::send_trb_completion(4); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(4) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 1); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + ConnectionFixture::send_token(2); + ConnectionFixture::send_token(4); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 3); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(4) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 3); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 2); + + auto start_time = std::chrono::steady_clock::now(); + dfs->execute_command("stop", null_data); + auto stop_time = std::chrono::steady_clock::now(); + BOOST_REQUIRE(stop_time - start_time < std::chrono::milliseconds(100)); + + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case DataFlow END"; +} + +BOOST_AUTO_TEST_CASE(Requests) +{ + TLOG() << "Test case Requests BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + + // Note: Counters are reset by calling get_opmon_info! + auto metric = get_opmon_info(); + + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + // Requests for different run are ignored + ConnectionFixture::send_dataflow_status_request(1, true); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.requests_received(), 0); + + ConnectionFixture::send_dataflow_status_request(1); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + dunedaq::dfmessages::DataflowStatus expected_status; + for (const auto& status : ConnectionFixture::s_received_statuses) { + if (status.trigger_number == 1) { + expected_status = status; + break; + } + } + + BOOST_REQUIRE(expected_status.trigger_number != dfmessages::TypeDefaults::s_invalid_trigger_number); + BOOST_REQUIRE_EQUAL(expected_status.trigger_number, 1); + BOOST_REQUIRE_EQUAL(expected_status.run_number, 1); + BOOST_REQUIRE_EQUAL(expected_status.triggers_building.size(), 0); + BOOST_REQUIRE_EQUAL(expected_status.triggers_writing.size(), 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.requests_received(), 1); + + dfs->execute_command("stop", null_data); + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case Requests END"; +} + +BOOST_AUTO_TEST_CASE(OutOfOrder) +{ + TLOG() << "Test case OutOfOrder BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + + // Note: Counters are reset by calling get_opmon_info! + auto metric = get_opmon_info(); + + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + ConnectionFixture::send_trb_completion(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 1); + + ConnectionFixture::send_token(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_tokens_received(), 1); + + ConnectionFixture::send_trigdec(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(1) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_tokens_received(), 0); + + ConnectionFixture::send_trigdec(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(1) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + BOOST_REQUIRE_EQUAL(metric.duplicate_decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_tokens_received(), 0); + + ConnectionFixture::send_token(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(1) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 1); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 1); + + BOOST_REQUIRE_EQUAL(metric.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.early_tokens_received(), 1); + + ConnectionFixture::send_trb_completion(1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(1) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 1); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + BOOST_REQUIRE_EQUAL(metric.duplicate_decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.unexpected_trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.unexpected_tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.early_tokens_received(), 0); + + dfs->execute_command("stop", null_data); + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case OutOfOrder END"; +} + +BOOST_AUTO_TEST_CASE(StopTimeout_Building) +{ + TLOG() << "Test case StopTimeout_Building BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + auto metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + ConnectionFixture::send_trigdec(2); + ConnectionFixture::send_trigdec(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 2); + + auto start_time = std::chrono::steady_clock::now(); + dfs->execute_command("stop", null_data); + auto stop_time = std::chrono::steady_clock::now(); + BOOST_REQUIRE(stop_time - start_time > std::chrono::milliseconds(1000)); + + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case StopTimeout_Building END"; +} + +BOOST_AUTO_TEST_CASE(StopTimeout_Writing) +{ + TLOG() << "Test case StopTimeout_Writing BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + auto metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + ConnectionFixture::send_trigdec(2); + ConnectionFixture::send_trigdec(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 2); + + ConnectionFixture::send_trb_completion(2); + ConnectionFixture::send_trb_completion(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + auto start_time = std::chrono::steady_clock::now(); + dfs->execute_command("stop", null_data); + auto stop_time = std::chrono::steady_clock::now(); + BOOST_REQUIRE(stop_time - start_time > std::chrono::milliseconds(1000)); + + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case StopTimeout_Writing END"; +} + +BOOST_AUTO_TEST_CASE(StopTimeout_BuildingAndWriting) +{ + TLOG() << "Test case StopTimeout_BuildingAndWriting BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + auto metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + ConnectionFixture::send_trigdec(2); + ConnectionFixture::send_trigdec(3); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 2); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 2); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 2); + + ConnectionFixture::send_trb_completion(2); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(3) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(2) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + auto start_time = std::chrono::steady_clock::now(); + dfs->execute_command("stop", null_data); + auto stop_time = std::chrono::steady_clock::now(); + + // Stop transition timeout is shared between waiting for both building and writting triggers + BOOST_REQUIRE(stop_time - start_time > std::chrono::milliseconds(1000)); + BOOST_REQUIRE(stop_time - start_time < std::chrono::milliseconds(2000)); + + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case StopTimeout_BuildingAndWriting END"; +} + +BOOST_AUTO_TEST_CASE(MultipleSequences) +{ + TLOG() << "Test case MultipleSequences BEGIN"; + auto dfs = appfwk::make_module("DataflowStatusModule", "test"); + opmgr.register_node("dfs", dfs); + dfs->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + auto iom = iomanager::IOManager::get(); + ConnectionFixture::reset(); + auto dfs_recv = iom->get_receiver("df_status"); + dfs_recv->add_callback(ConnectionFixture::receive_dataflow_status); + auto dec_recv = iom->get_receiver("trigdec_trb"); + dec_recv->add_callback(ConnectionFixture::receive_trigger_decision); + + dfs->execute_command("conf", null_data); + dfs->execute_command("start", start_data); + + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + auto metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 0); + + // Send a trigger decision that will be split into multiple sequences (e.g., sequence 0, 1, 2) + ConnectionFixture::send_trigdec(100); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + BOOST_REQUIRE(ConnectionFixture::s_received_decisions.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.size() >= 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + BOOST_REQUIRE_EQUAL(metric.decisions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.decisions_sent(), 1); + + // Send first TRB completion for sequence 0, with max_sequence_number = 2 + ConnectionFixture::send_trb_completion(100, 0, 2); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Should still be building, waiting for all sequences to complete + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + // Send second TRB completion for sequence 1 + ConnectionFixture::send_trb_completion(100, 1, 2); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Still building, waiting for sequence 2 + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + // Send third TRB completion for sequence 2 - all sequences now built + ConnectionFixture::send_trb_completion(100, 2, 2); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Now should transition to writing state + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 1); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 0); + + // Send first token for sequence 0 + ConnectionFixture::send_token(100, 0); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Still writing, waiting for all tokens + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 1); + + // Send second token for sequence 1 + ConnectionFixture::send_token(100, 1); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Still writing, waiting for token for sequence 2 + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 0); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 1); + + // Send third token for sequence 2 - all sequences now written + ConnectionFixture::send_token(100, 2); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + + // Now should be completed + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_building.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().triggers_writing.size() == 0); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.size() == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().recently_completed_triggers.count(100) == 1); + BOOST_REQUIRE(ConnectionFixture::s_received_statuses.back().trigger_records_processed == 1); + + metric = get_opmon_info(); + BOOST_REQUIRE_EQUAL(metric.trb_completions_received(), 0); + BOOST_REQUIRE_EQUAL(metric.tokens_received(), 1); + + dfs->execute_command("stop", null_data); + dfs->execute_command("scrap", null_data); + + dec_recv->remove_callback(); + dfs_recv->remove_callback(); + TLOG() << "Test case MultipleSequences END"; +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace dunedaq diff --git a/unittest/TRBModule_test.cxx b/unittest/TRBModule_test.cxx new file mode 100755 index 00000000..0aacc2b7 --- /dev/null +++ b/unittest/TRBModule_test.cxx @@ -0,0 +1,626 @@ +/** + * @file TRBModule_test.cxx Test application that tests and demonstrates + * the functionality of the TRBModule class. + * + * This is part of the DUNE DAQ Application Framework, copyright 2020. + * Licensing/copyright details are in the COPYING file that you should have + * received with this code. + */ + +#include "TRBModule.hpp" + +#include "daqdataformats/SourceID.hpp" +#include "dfmessages/DataRequest.hpp" +#include "dfmessages/TriggerDecision.hpp" +#include "dfmessages/TRBCompletion.hpp" +#include "dfmessages/TriggerRecord_serialization.hpp" +#include "dfmessages/Fragment_serialization.hpp" +#include "dfmodules/CommonIssues.hpp" +#include "dfmodules/opmon/TRBModule.pb.h" +#include "iomanager/IOManager.hpp" +#include "iomanager/Sender.hpp" +#include "opmonlib/TestOpMonManager.hpp" + +#define BOOST_TEST_MODULE TRBModule_test // NOLINT + +#include "boost/test/unit_test.hpp" + +#include +#include +#include +#include + +using namespace dunedaq::dfmodules; + +namespace dunedaq { + +struct EnvFixture +{ + EnvFixture() { setenv("DUNEDAQ_PARTITION", "partition_name", 0); } +}; +BOOST_TEST_GLOBAL_FIXTURE(EnvFixture); + +struct CfgFixture +{ + CfgFixture() + { + std::string oksConfig = "oksconflibs:test/config/triggerrecordbuilder_test.data.xml"; + std::string appName = "TestApp"; + std::string sessionName = "partition_name"; + cfgMgr = std::make_shared(oksConfig, appName, sessionName); + get_iomanager()->configure(sessionName, cfgMgr->get_queues(), cfgMgr->get_networkconnections(), nullptr, opmgr); + } + ~CfgFixture() { get_iomanager()->reset(); } + + auto get_trb_info() + { + opmgr.collect(); + auto opmon_facility = opmgr.get_backend_facility(); + auto list = opmon_facility->get_entries(std::regex(".*TRBInfo")); + BOOST_REQUIRE_EQUAL(list.size(), 1); + const auto& entry = list.front(); + return opmonlib::from_entry(entry); + } + auto get_trb_errors() + { + opmgr.collect(); + auto opmon_facility = opmgr.get_backend_facility(); + auto list = opmon_facility->get_entries(std::regex(".*TRBErrors")); + BOOST_REQUIRE_EQUAL(list.size(), 1); + const auto& entry = list.front(); + return opmonlib::from_entry(entry); + } + + dunedaq::opmonlib::TestOpMonManager opmgr; + std::shared_ptr cfgMgr; +}; + +BOOST_FIXTURE_TEST_SUITE(TRBModule_test, CfgFixture) + +// Storage for received messages +std::vector> received_trigger_records; +std::vector received_data_requests; +std::vector received_trb_completes; + +void +recv_trigrecord(std::unique_ptr& tr) +{ + TLOG() << "Received TriggerRecord with trigger number " << tr->get_header_ref().get_trigger_number(); + received_trigger_records.push_back(std::move(tr)); +} + +void +recv_datareq(const dfmessages::DataRequest& req) +{ + TLOG() << "Received DataRequest for trigger number " << req.trigger_number; + received_data_requests.push_back(req); +} + +void +recv_trbcomplete(const dfmessages::TRBCompletion& complete) +{ + TLOG() << "Received TRBCompletion for trigger number " << complete.trigger_id.trigger_number; + received_trb_completes.push_back(complete); +} + +void +send_trigdec(dfmessages::trigger_number_t trigger_number, dfmessages::run_number_t run_number = 1, int window_size = 2000) +{ + dunedaq::dfmessages::TriggerDecision td; + td.trigger_number = trigger_number; + td.run_number = run_number; + td.trigger_timestamp = 50000 + trigger_number * 1000; + td.trigger_type = 1; + td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; + + // Add component request (simplified) + dfmessages::ComponentRequest comp_req; + comp_req.component.subsystem = daqdataformats::SourceID::Subsystem::kDetectorReadout; + comp_req.component.id = 0; + comp_req.window_begin = td.trigger_timestamp; + comp_req.window_end = td.trigger_timestamp + window_size; + td.components.push_back(comp_req); + + auto iom = iomanager::IOManager::get(); + TLOG() << "Sending TriggerDecision with trigger number " << trigger_number << " to TRB"; + iom->get_sender("trigdec_trb")->send(std::move(td), iomanager::Sender::s_block); +} + +void +send_fragment(dfmessages::trigger_number_t trigger_number, + dfmessages::run_number_t run_number = 1, + daqdataformats::sequence_number_t sequence_number = 0) +{ + daqdataformats::FragmentHeader hdr; + hdr.trigger_number = trigger_number; + hdr.run_number = run_number; + hdr.sequence_number = sequence_number; + hdr.trigger_timestamp = 50000 + trigger_number * 1000; + hdr.window_begin = hdr.trigger_timestamp + (sequence_number * 5000); + hdr.window_end = hdr.trigger_timestamp + ((sequence_number + 1) * 5000); + hdr.element_id = daqdataformats::SourceID(daqdataformats::SourceID::Subsystem::kDetectorReadout, 0); + hdr.fragment_type = static_cast(daqdataformats::FragmentType::kWIBEth); + hdr.size = sizeof(daqdataformats::FragmentHeader); + + auto frag = std::make_unique(&hdr, dunedaq::daqdataformats::Fragment::BufferAdoptionMode::kCopyFromBuffer); + + auto iom = iomanager::IOManager::get(); + TLOG() << "Sending Fragment for trigger number " << trigger_number << " to TRB"; + iom->get_sender>("fragment")->send(std::move(frag), iomanager::Sender::s_block); +} + +BOOST_AUTO_TEST_CASE(CopyAndMoveSemantics) +{ + BOOST_REQUIRE(!std::is_copy_constructible_v); + BOOST_REQUIRE(!std::is_copy_assignable_v); + BOOST_REQUIRE(!std::is_move_constructible_v); + BOOST_REQUIRE(!std::is_move_assignable_v); +} + +BOOST_AUTO_TEST_CASE(Constructor) +{ + TLOG() << "Test case Constructor BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + TLOG() << "Test case Constructor END"; +} + +BOOST_AUTO_TEST_CASE(Init) +{ + TLOG() << "Test case Init BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + trb->init(cfgMgr); + TLOG() << "Test case Init END"; +} + +BOOST_AUTO_TEST_CASE(Commands) +{ + TLOG() << "Test case Commands BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + trb->execute_command("start", start_data); + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + auto metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 0); + TLOG() << "Test case Commands END"; +} + +BOOST_AUTO_TEST_CASE(DataFlow) +{ + TLOG() << "Test case DataFlow BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 0); + + // Send a trigger decision + send_trigdec(1); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 1); + + // Should have received a data request + BOOST_REQUIRE_EQUAL(received_data_requests.size(), 1); + BOOST_REQUIRE_EQUAL(received_data_requests[0].trigger_number, 1); + + // Send corresponding fragment + send_fragment(1); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should have received a trigger record + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records[0]->get_header_ref().get_trigger_number(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records[0]->get_header_ref().get_run_number(), 1); + + // Should have recieved a TRBCompletion message + BOOST_REQUIRE_EQUAL(received_trb_completes.size(), 1); + BOOST_REQUIRE_EQUAL(received_trb_completes[0].trigger_id.trigger_number, 1); + + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 0); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case DataFlow END"; +} + +BOOST_AUTO_TEST_CASE(WrongRunNumber) +{ + TLOG() << "Test case WrongRunNumber BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Send trigger decision with wrong run number + send_trigdec(1, 999); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should not create any pending trigger decisions (rejected) + auto metric = get_trb_info(); + auto err_metric = get_trb_errors(); + BOOST_REQUIRE_EQUAL(metric.received_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(err_metric.unexpected_trigger_decisions(), 1); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case WrongRunNumber END"; +} + +BOOST_AUTO_TEST_CASE(FragmentTimeout) +{ + TLOG() << "Test case FragmentTimeout BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Send a trigger decision + send_trigdec(1); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 1); + + // Should have received a data request + BOOST_REQUIRE_EQUAL(received_data_requests.size(), 1); + BOOST_REQUIRE_EQUAL(received_data_requests[0].trigger_number, 1); + + // Send corresponding fragment + send_fragment(1); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should have received a trigger record + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records[0]->get_header_ref().get_trigger_number(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records[0]->get_header_ref().get_run_number(), 1); + + // Should have recieved a TRBCompletion message + BOOST_REQUIRE_EQUAL(received_trb_completes.size(), 1); + BOOST_REQUIRE_EQUAL(received_trb_completes[0].trigger_id.trigger_number, 1); + + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 0); + + // Send trigger decision + send_trigdec(2); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 1); + + // Wait for timeout (configured to 1000ms) + std::this_thread::sleep_for(std::chrono::milliseconds(1200)); + send_fragment(999); // Send fragment for a different trigger number to avoid completing the trigger record, but process for stale + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should have timed out and sent incomplete trigger record + metric = get_trb_info(); + auto err_metric = get_trb_errors(); + BOOST_REQUIRE_EQUAL(err_metric.timed_out_trigger_records(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 2); + + // Should have recieved a TRBCompletion message + BOOST_REQUIRE_EQUAL(received_trb_completes.size(), 2); + BOOST_REQUIRE_EQUAL(received_trb_completes[1].trigger_id.trigger_number, 2); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case FragmentTimeout END"; +} + +BOOST_AUTO_TEST_CASE(UnexpectedFragment) +{ + TLOG() << "Test case UnexpectedFragment BEGIN"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Send fragment without trigger decision + send_fragment(999); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should be counted as unexpected + auto metric = get_trb_info(); + auto err_metric = get_trb_errors(); + BOOST_REQUIRE_EQUAL(err_metric.unexpected_fragments(), 1); + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 0); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case UnexpectedFragment END"; +} + +BOOST_AUTO_TEST_CASE(MultipleTriggers) +{ + TLOG() << "Test case MultipleTriggers BEGIN"; + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Send multiple trigger decisions + for (int i = 1; i <= 5; ++i) { + send_trigdec(i); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + auto metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 5); + BOOST_REQUIRE_EQUAL(received_data_requests.size(), 5); + + // Send all fragments + for (int i = 1; i <= 5; ++i) { + send_fragment(i); + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // All should complete + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 5); + BOOST_REQUIRE_EQUAL(received_trb_completes.size(), 5); + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case MultipleTriggers END"; +} + +BOOST_AUTO_TEST_CASE(MultipleSequences) +{ + TLOG() << "Test case MultipleSequences START"; + auto trb = appfwk::make_module("TRBModule", "test"); + opmgr.register_node("trb", trb); + trb->init(cfgMgr); + + appfwk::DAQModule::CommandData_t null_data; + appfwk::DAQModule::CommandData_t start_data; + start_data.emplace("run", 1); + + trb->execute_command("conf", null_data); + + auto iom = iomanager::IOManager::get(); + auto tr_recv = iom->get_receiver>("trigrecord"); + tr_recv->add_callback(recv_trigrecord); + auto dr_recv = iom->get_receiver("datareq_ru0"); + dr_recv->add_callback(recv_datareq); + auto trbc_recv = iom->get_receiver("trb_complete"); + trbc_recv->add_callback(recv_trbcomplete); + + received_trb_completes.clear(); + received_trigger_records.clear(); + received_data_requests.clear(); + + trb->execute_command("start", start_data); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Create a trigger decision with a large time window that will be split into multiple sequences + // max_sequence_length_ticks is set to 5000 in config + // We'll create a window of 12000 ticks, which should result in 3 sequences: + // sequence 0: 0-5000, sequence 1: 5000-10000, sequence 2: 10000-12000 + send_trigdec(100, 1, 12000); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + // Should have received 3 data requests (one per sequence) + BOOST_REQUIRE_EQUAL(received_data_requests.size(), 3); + TLOG() << "Received " << received_data_requests.size() << " data requests (one per sequence)"; + + // Verify sequence numbers in data requests + std::set request_sequences; + for (const auto& req : received_data_requests) { + BOOST_REQUIRE_EQUAL(req.trigger_number, 100); + request_sequences.insert(req.sequence_number); + TLOG() << "DataRequest: trigger=" << req.trigger_number + << ", sequence=" << req.sequence_number + << ", window=" << req.request_information.window_begin + << "-" << req.request_information.window_end; + } + BOOST_REQUIRE_EQUAL(request_sequences.size(), 3); + BOOST_REQUIRE(request_sequences.count(0) > 0); + BOOST_REQUIRE(request_sequences.count(1) > 0); + BOOST_REQUIRE(request_sequences.count(2) > 0); + + auto metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 3); // 3 pending sequences + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 3); + + // Send fragments for all three sequences + TLOG() << "Sending fragments for all 3 sequences"; + for (daqdataformats::sequence_number_t seq = 0; seq < 3; ++seq) { + send_fragment(100, 1, seq); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + // Should have received 3 trigger records (one per sequence) + BOOST_REQUIRE_EQUAL(received_trigger_records.size(), 3); + TLOG() << "Received " << received_trigger_records.size() << " TriggerRecords (one per sequence)"; + + // Verify the trigger records + std::set tr_sequences; + for (const auto& tr : received_trigger_records) { + auto& hdr = tr->get_header_ref(); + BOOST_REQUIRE_EQUAL(hdr.get_trigger_number(), 100); + BOOST_REQUIRE_EQUAL(hdr.get_run_number(), 1); + BOOST_REQUIRE_EQUAL(hdr.get_max_sequence_number(), 2); // max sequence is 2 (0, 1, 2) + + tr_sequences.insert(hdr.get_sequence_number()); + + TLOG() << "TriggerRecord: trigger=" << hdr.get_trigger_number() + << ", sequence=" << hdr.get_sequence_number() + << ", max_sequence=" << hdr.get_max_sequence_number() + << ", fragments=" << tr->get_fragments_ref().size(); + + // Each sequence should have one fragment + BOOST_REQUIRE_EQUAL(tr->get_fragments_ref().size(), 1); + } + + // Verify we got all three sequences + BOOST_REQUIRE_EQUAL(tr_sequences.size(), 3); + BOOST_REQUIRE(tr_sequences.count(0) > 0); + BOOST_REQUIRE(tr_sequences.count(1) > 0); + BOOST_REQUIRE(tr_sequences.count(2) > 0); + + // Should have received 1 TRBCompletion (after all sequences complete) + BOOST_REQUIRE_EQUAL(received_trb_completes.size(), 3); + BOOST_REQUIRE_EQUAL(received_trb_completes[0].trigger_id.trigger_number, 100); + BOOST_REQUIRE_EQUAL(received_trb_completes[0].trigger_record_max_sequence_number, 2); + TLOG() << "Received TRBCompletion with max_sequence_number=" + << received_trb_completes[0].trigger_record_max_sequence_number; + + metric = get_trb_info(); + BOOST_REQUIRE_EQUAL(metric.pending_trigger_decisions(), 0); + BOOST_REQUIRE_EQUAL(metric.pending_fragments(), 0); + + trb->execute_command("stop", null_data); + trb->execute_command("scrap", null_data); + + trbc_recv->remove_callback(); + tr_recv->remove_callback(); + dr_recv->remove_callback(); + TLOG() << "Test case MultipleSequences END"; +} + +BOOST_AUTO_TEST_SUITE_END() +} // namespace dunedaq diff --git a/unittest/TriggerRecordBuilderData_test.cxx b/unittest/TriggerRecordBuilderData_test.cxx deleted file mode 100644 index f381ee6b..00000000 --- a/unittest/TriggerRecordBuilderData_test.cxx +++ /dev/null @@ -1,190 +0,0 @@ -/** - * @file TriggerRecordBuilderData_test.cxx Test application that tests and demonstrates - * the functionality of the TriggerRecordBuilderData class. - * - * This is part of the DUNE DAQ Application Framework, copyright 2020. - * Licensing/copyright details are in the COPYING file that you should have - * received with this code. - */ - -#include "opmonlib/TestOpMonManager.hpp" -#include "dfmodules/TriggerRecordBuilderData.hpp" - -#define BOOST_TEST_MODULE TriggerRecordBuilderData_test // NOLINT - -#include "boost/test/unit_test.hpp" - -#include -#include -#include - -using namespace dunedaq::dfmodules; - -BOOST_AUTO_TEST_SUITE(TriggerRecordBuilderData_Test) - -BOOST_AUTO_TEST_CASE(CopyAndMoveSemantics) -{ - BOOST_REQUIRE(!std::is_copy_constructible_v); - BOOST_REQUIRE(!std::is_copy_assignable_v); - BOOST_REQUIRE(!std::is_move_constructible_v); - BOOST_REQUIRE(!std::is_move_assignable_v); -} - -BOOST_AUTO_TEST_CASE(Constructors) -{ - dunedaq::dfmessages::TriggerDecision td; - td.trigger_number = 1; - td.run_number = 2; - td.trigger_timestamp = 3; - td.trigger_type = 4; - td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - - AssignedTriggerDecision atd(td, "test"); - - BOOST_REQUIRE_EQUAL(atd.decision.trigger_number, td.trigger_number); - BOOST_REQUIRE_EQUAL(atd.connection_name, "test"); - - // TRBD must have a default constructor so that it can be used in a std::map, but a default-constructed TRBD is - // invalid. - TriggerRecordBuilderData trbd; - BOOST_REQUIRE(trbd.is_in_error()); - - TriggerRecordBuilderData trbd2("test", 10); - - BOOST_REQUIRE_EQUAL(trbd2.used_slots(), 0); - BOOST_REQUIRE_EQUAL(trbd2.is_busy(), false); - BOOST_REQUIRE(!trbd2.is_in_error()); - - trbd2.set_in_error(true); - BOOST_REQUIRE(trbd2.is_in_error()); - BOOST_REQUIRE_EQUAL(trbd2.is_busy(), true); - - trbd2.set_in_error(false); - BOOST_REQUIRE_EQUAL(trbd2.used_slots(), 0); - BOOST_REQUIRE_EQUAL(trbd2.is_busy(), false); - BOOST_REQUIRE(!trbd2.is_in_error()); - - BOOST_REQUIRE_EXCEPTION(TriggerRecordBuilderData("test", 10, 15), - DFOThresholdsNotConsistent, - [](DFOThresholdsNotConsistent const&) { return true; }); -} - -BOOST_AUTO_TEST_CASE(Assignments) -{ - auto start_time = std::chrono::steady_clock::now(); - dunedaq::dfmessages::TriggerDecision td; - td.trigger_number = 1; - td.run_number = 2; - td.trigger_timestamp = 3; - td.trigger_type = 4; - td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - - dunedaq::opmonlib::TestOpMonManager opmgr; - auto trbd_p = std::make_shared("test", 2); - opmgr.register_node("trbd", trbd_p); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 0); - BOOST_REQUIRE(!trbd_p->is_busy()); - - auto assignment = trbd_p->make_assignment(td); - BOOST_REQUIRE_EQUAL(assignment->connection_name, "test"); - trbd_p->add_assignment(assignment); - - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 1); - auto got_assignment = trbd_p->get_assignment(1); - BOOST_REQUIRE_EQUAL(got_assignment->decision.trigger_number, assignment->decision.trigger_number); - BOOST_REQUIRE_EQUAL(got_assignment->decision.trigger_timestamp, assignment->decision.trigger_timestamp); - BOOST_REQUIRE_EQUAL(got_assignment.get(), assignment.get()); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 1); - - auto extracted_assignment = trbd_p->extract_assignment(1); - BOOST_REQUIRE_EQUAL(extracted_assignment.get(), assignment.get()); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 0); - trbd_p->add_assignment(extracted_assignment); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 1); - - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - - std::chrono::steady_clock::time_point complete_time; - trbd_p->complete_assignment(1, - [&complete_time](nlohmann::json&) { complete_time = std::chrono::steady_clock::now(); }); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 0); - - auto latency = - std::chrono::duration_cast(complete_time - assignment->assigned_time) - .count(); - - BOOST_REQUIRE_CLOSE(static_cast(trbd_p->average_latency(start_time).count()), static_cast(latency), 5); - - auto null_got_assignment = trbd_p->get_assignment(2); - BOOST_REQUIRE_EQUAL(null_got_assignment, nullptr); - auto null_extracted_assignment = trbd_p->extract_assignment(3); - BOOST_REQUIRE_EQUAL(null_extracted_assignment, nullptr); - - trbd_p->add_assignment(assignment); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 1); - auto remnants = trbd_p->flush(); - BOOST_REQUIRE_EQUAL(trbd_p->used_slots(), 0); - BOOST_REQUIRE_EQUAL(remnants.size(), 1); - -} - -BOOST_AUTO_TEST_CASE(Exceptions) -{ - dunedaq::dfmessages::TriggerDecision td; - td.trigger_number = 1; - td.run_number = 2; - td.trigger_timestamp = 3; - td.trigger_type = 4; - td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - dunedaq::dfmessages::TriggerDecision another_td; - another_td.trigger_number = 2; - another_td.run_number = 2; - another_td.trigger_timestamp = 5; - another_td.trigger_type = 4; - another_td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - dunedaq::dfmessages::TriggerDecision yet_another_td; - yet_another_td.trigger_number = 3; - yet_another_td.run_number = 2; - yet_another_td.trigger_timestamp = 7; - yet_another_td.trigger_type = 4; - yet_another_td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - - TriggerRecordBuilderData trbd("test", 2); - BOOST_REQUIRE_EQUAL(trbd.used_slots(), 0); - BOOST_REQUIRE(!trbd.is_busy()); - - auto assignment = trbd.make_assignment(td); - BOOST_REQUIRE_EQUAL(assignment->connection_name, "test"); - trbd.add_assignment(assignment); - - auto another_assignment = trbd.make_assignment(another_td); - trbd.add_assignment(another_assignment); - - BOOST_REQUIRE_EQUAL(trbd.used_slots(), 2); - BOOST_REQUIRE(trbd.is_busy()); - - BOOST_REQUIRE_EXCEPTION(trbd.complete_assignment(3), - AssignedTriggerDecisionNotFound, - [](AssignedTriggerDecisionNotFound const&) { return true; }); - - auto yet_another_assignment = trbd.make_assignment(yet_another_td); - BOOST_CHECK_NO_THROW(trbd.add_assignment(yet_another_assignment)); - // we are now above threshold but we can accept new assigments anyway because we are not in error - BOOST_REQUIRE(trbd.is_busy()); - - trbd.set_in_error(true); - dunedaq::dfmessages::TriggerDecision err_td; - err_td.trigger_number = 4; - err_td.run_number = 2; - err_td.trigger_timestamp = 10; - err_td.trigger_type = 4; - err_td.readout_type = dunedaq::dfmessages::ReadoutType::kLocalized; - auto err_assignment = trbd.make_assignment(err_td); - - BOOST_REQUIRE_EXCEPTION( - trbd.add_assignment(err_assignment), NoSlotsAvailable, [](NoSlotsAvailable const&) { return true; }); -} - - - -BOOST_AUTO_TEST_SUITE_END()