diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 2e65698578f..2739b4ac1e3 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -603,6 +603,10 @@ jobs: options: --gpus all env: MODEL_STORAGE: ${{ github.workspace }}/ci_models + # TF32 GEMMs on the pool's A4000 (absent on its T4) drift the GNN + # classifier output and the ntuple_finding_gnn.root hash; disabling + # TF32 makes both cards agree. + NVIDIA_TF32_OVERRIDE: "0" steps: - name: Fix up PATH diff --git a/Python/Examples/tests/root_file_hashes.txt b/Python/Examples/tests/root_file_hashes.txt index b86f5f69388..e5012e8d767 100644 --- a/Python/Examples/tests/root_file_hashes.txt +++ b/Python/Examples/tests/root_file_hashes.txt @@ -78,6 +78,12 @@ test_root_material_writer__material.root: 6835480e29fb110f5c6f954fa9e84144e7606e test_root_clusters_writer[configPosConstructor]__clusters.root: e842df4fe04eefff3df5f32cd1026e93286be62b8040dc700a2aff557c56dec8 test_root_clusters_writer[configKwConstructor]__clusters.root: e842df4fe04eefff3df5f32cd1026e93286be62b8040dc700a2aff557c56dec8 test_root_clusters_writer[kwargsConstructor]__clusters.root: e842df4fe04eefff3df5f32cd1026e93286be62b8040dc700a2aff557c56dec8 +test_gnn_metric_learning[gpu]__performance_finding_gnn.root: c17fb877bb165e28db0a2b99881763093e7fc9a707c045feb6a6a6b68e0dd660 +test_gnn_metric_learning[gpu]__ntuple_finding_gnn.root: 30f11c53318ce63ba9416858641050ba1a75e820af8138ccdbac10aad756d401 +test_gnn_module_map[gpu-torch]__ntuple_finding_gnn.root: 1df2fb0dda8bad7aeca7d676c2f13f5d8a1ddbb7e8d41010b6d66a11bd579f4f +test_gnn_module_map[gpu-torch]__performance_finding_gnn.root: c17fb877bb165e28db0a2b99881763093e7fc9a707c045feb6a6a6b68e0dd660 +test_gnn_module_map[gpu-onnx]__ntuple_finding_gnn.root: 1df2fb0dda8bad7aeca7d676c2f13f5d8a1ddbb7e8d41010b6d66a11bd579f4f +test_gnn_module_map[gpu-onnx]__performance_finding_gnn.root: c17fb877bb165e28db0a2b99881763093e7fc9a707c045feb6a6a6b68e0dd660 test_ML_Ambiguity_Solver__performance_finding_ambiML.root: c17fb877bb165e28db0a2b99881763093e7fc9a707c045feb6a6a6b68e0dd660 test_truth_tracking_kalman[generic-0.0]__trackstates_kf.root: 2fb04500df18f48aeee9b8aca755f4485e5e8bbf4e27cc29d25dab918c771bd2 test_truth_tracking_kalman[generic-0.0]__tracksummary_kf.root: 77905a97daec655e115a5f5056349d1a19d44c283d38b06660778b0a474e865f diff --git a/Python/Examples/tests/test_examples.py b/Python/Examples/tests/test_examples.py index f7e3d0e3719..607fe086ad8 100644 --- a/Python/Examples/tests/test_examples.py +++ b/Python/Examples/tests/test_examples.py @@ -973,7 +973,7 @@ def assert_gnn_output(output_dir: Path): @pytest.mark.parametrize("hardware", ["cpu", "gpu"]) @pytest.mark.skipif(not gnnEnabled, reason="Gnn environment not set up") -def test_gnn_metric_learning(tmp_path, trk_geo, field, hardware): +def test_gnn_metric_learning(tmp_path, trk_geo, field, assert_root_hash, hardware): """Test GNN track finding with metric learning graph construction""" if hardware == "cpu": pytest.skip("CPU not yet supported") @@ -1018,6 +1018,8 @@ def test_gnn_metric_learning(tmp_path, trk_geo, field, hardware): raise assert_gnn_output(tmp_path) + for f in root_files: + assert_root_hash(f, tmp_path / f) @pytest.mark.odd @@ -1083,7 +1085,7 @@ def test_gnn_shrink_nodes_same_output(tmp_path, hardware): @pytest.mark.skipif(not gnnEnabled, reason="Gnn environment not set up") @pytest.mark.parametrize("backend", ["torch", "onnx"]) @pytest.mark.parametrize("hardware", ["gpu"]) -def test_gnn_module_map(tmp_path, backend, hardware): +def test_gnn_module_map(tmp_path, assert_root_hash, backend, hardware): """Test GNN track finding with module map graph construction on ODD""" from gnn_module_map_odd import runGnnModuleMap from acts.examples.odd import getOpenDataDetector @@ -1137,6 +1139,10 @@ def test_gnn_module_map(tmp_path, backend, hardware): # Verify output assert_gnn_output(tmp_path) + assert_root_hash("ntuple_finding_gnn.root", tmp_path / "ntuple_finding_gnn.root") + assert_root_hash( + "performance_finding_gnn.root", tmp_path / "performance_finding_gnn.root" + ) @pytest.mark.odd