From 8c590317e18aa969b0cc2e5a1b8710c4db9dacf0 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 11 Feb 2022 14:50:50 -0800 Subject: [PATCH 1/6] add SCR to HDF path --- src/CheckPoint.C | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index 75baed0c..3d5e7070 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -1119,9 +1119,24 @@ void CheckPoint::write_checkpoint_hdf5(float_sw4 a_time, int a_cycle, s << mEW->getPath() << "/"; s << fileSuffix.str(); +#ifndef SW4_USE_SCR // Keep track of the number of files, save previous file name, and delete the // second last. cycle_checkpoints(s.str()); +#else + // SCR can delete older checkpoints, e.g., SCR_PREFIX_SIZE=3 + + // Inform SCR that a new checkpoint is starting + std::string cycle_num; + cycle_num << "cycle=" << a_cycle; + SCR_Start_output(cycle_num.str().c_str(), SCR_FLAG_CHECKPOINT); + + // Ask SCR for the path to write our checkpoint file + // This could be moved just before each H5FCreate call if need to preserve original file name + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + s = scr_file; +#endif hid_t fid, fapl, dxpl, dspace, mspace, mydspace, dtype, dcpl; int myrank, nrank; @@ -1343,6 +1358,12 @@ void CheckPoint::write_checkpoint_hdf5(float_sw4 a_time, int a_cycle, #else H5Fclose(fid); #endif + +#ifdef SW4_USE_SCR + int valid = 1; + SCR_Complete_output(valid); +#endif + etime = MPI_Wtime(); if (myrank == 0) @@ -1360,12 +1381,39 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle, MPI_Comm_rank(MPI_COMM_WORLD, &myrank); MPI_Comm_size(MPI_COMM_WORLD, &nrank); +#ifdef SW4_USE_SCR + // Double check that SCR has a checkpoint loaded + int have_restart = 0; + char cycle_num[SCR_MAX_FILENAME]; + SCR_Have_restart(&have_restart, cycle_num); + if (! have_restart) { + // We expected SCR to have something to get to this point + abort(); + } + + // Get checkpoint name from SCR (we use application cycle number) + SCR_Start_restart(cycle_num); +#endif + std::stringstream s; if (mRestartPathSet) s << mRestartPath << "/"; else if (mEW->getPath() != "./") s << mEW->getPath(); + +#ifndef SW4_USE_SCR s << mRestartFile; +#else + // TODO: this is not right, but you get the idea... + std::stringstream fileSuffix; + compute_file_suffix(cycle_num, fileSuffix); + s << fileSuffix; + + // Ask SCR for the path to open our checkpoint file + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + s = scr_file; +#endif if (myrank == 0) std::cout << "Reading checkpoint from file " << s.str() << std::endl; @@ -1474,6 +1522,11 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle, H5Pclose(fapl); H5Pclose(dxpl); H5Fclose(fid); + +#ifdef SW4_USE_SCR + int valid = 1; + SCR_Complete_restart(valid); +#endif } #endif // End USE_HDF5 //----------------------------------------------------------------------- From 461a4331c80dc758321bde2cdcf4493dde56b56b Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 11 Feb 2022 15:05:30 -0800 Subject: [PATCH 2/6] add SCR to getDt for HDF --- src/CheckPoint.C | 67 +++++++++++++++++++----------------------------- 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index 3d5e7070..e6c95a1b 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -578,15 +578,41 @@ void CheckPoint::read_checkpoint(float_sw4& a_time, int& a_cycle, //----------------------------------------------------------------------- float_sw4 CheckPoint::getDt() { -#ifndef SW4_USE_SCR float_sw4 dt; + +#ifdef SW4_USE_SCR + // Get checkpoint name (cycle number) from SCR + int have_restart = 0; + char cycle_num[SCR_MAX_FILENAME]; + SCR_Have_restart(&have_restart, cycle_num); + if (! have_restart) { + // We expected SCR to have something to get to this point + abort(); + } + + SCR_Start_restart(cycle_num); +#endif + if (mEW->getRank() == 0) { std::stringstream s; if (mRestartPathSet) s << mRestartPath << "/"; else if (mEW->getPath() != "./") s << mEW->getPath(); + +#ifndef SW4_USE_SCR s << mRestartFile; // string 's' is the file name including path +#else + // TODO: this is not right, but you get the idea... + std::stringstream fileSuffix; + compute_file_suffix(cycle_num, fileSuffix); + s << fileSuffix; + + // Ask SCR for the path to open our checkpoint file + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + s = scr_file; +#endif if (mUseHDF5) { #ifdef USE_HDF5 @@ -620,45 +646,6 @@ float_sw4 CheckPoint::getDt() { } MPI_Bcast(&dt, 1, mEW->m_mpifloat, 0, mEW->m_cartesian_communicator); return dt; -#else - - int have_restart = 0; - char checkpoint_dir[SCR_MAX_FILENAME]; - SCR_Have_restart(&have_restart, checkpoint_dir); - if (! have_restart) { - std::cerr<<"Error :: SCR found no checkpoints ! \n"<getRank()<<".bin"; - char scr_file[SCR_MAX_FILENAME]; - SCR_Route_file(s.str().c_str(), scr_file); - int valid=1; - if (std::FILE *file=std::fopen(scr_file,"rb")){ - float_sw4 dt; - if ( std::fread(&dt,sizeof dt,1,file)==1) { - scr_file_handle=file; - return dt; - } else { - std::cerr<<"ERROR:: Read of SCR checkpoint file failed in getDt \n"< Date: Fri, 11 Feb 2022 15:08:05 -0800 Subject: [PATCH 3/6] drop extra have and start restart calls --- src/CheckPoint.C | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index e6c95a1b..565492fd 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -1368,20 +1368,6 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle, MPI_Comm_rank(MPI_COMM_WORLD, &myrank); MPI_Comm_size(MPI_COMM_WORLD, &nrank); -#ifdef SW4_USE_SCR - // Double check that SCR has a checkpoint loaded - int have_restart = 0; - char cycle_num[SCR_MAX_FILENAME]; - SCR_Have_restart(&have_restart, cycle_num); - if (! have_restart) { - // We expected SCR to have something to get to this point - abort(); - } - - // Get checkpoint name from SCR (we use application cycle number) - SCR_Start_restart(cycle_num); -#endif - std::stringstream s; if (mRestartPathSet) s << mRestartPath << "/"; @@ -1391,6 +1377,8 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle, #ifndef SW4_USE_SCR s << mRestartFile; #else + // TODO: need to get cycle_num from earlier call to Have_restart + // TODO: this is not right, but you get the idea... std::stringstream fileSuffix; compute_file_suffix(cycle_num, fileSuffix); From 7beb75c6589ada122cf4be0a04a3a9085860d53f Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 11 Feb 2022 15:17:23 -0800 Subject: [PATCH 4/6] add compute_file_suffix given a string --- src/CheckPoint.C | 8 ++++++-- src/CheckPoint.h | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index 565492fd..aeb78e64 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -299,6 +299,10 @@ void CheckPoint::compute_file_suffix(int cycle, std::stringstream& fileSuffix) { fileSuffix << ".sw4checkpoint"; } +void CheckPoint::compute_file_suffix(const char* cycle, std::stringstream& fileSuffix) { + fileSuffix << mCheckPointFile << "." << cycle << ".sw4checkpoint"; +} + //----------------------------------------------------------------------- void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle, vector& a_Um, vector& a_U, @@ -606,7 +610,7 @@ float_sw4 CheckPoint::getDt() { // TODO: this is not right, but you get the idea... std::stringstream fileSuffix; compute_file_suffix(cycle_num, fileSuffix); - s << fileSuffix; + s << fileSuffix.str(); // Ask SCR for the path to open our checkpoint file char scr_file[SCR_MAX_FILENAME]; @@ -1382,7 +1386,7 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle, // TODO: this is not right, but you get the idea... std::stringstream fileSuffix; compute_file_suffix(cycle_num, fileSuffix); - s << fileSuffix; + s << fileSuffix.str(); // Ask SCR for the path to open our checkpoint file char scr_file[SCR_MAX_FILENAME]; diff --git a/src/CheckPoint.h b/src/CheckPoint.h index 5068c97b..36b2df09 100644 --- a/src/CheckPoint.h +++ b/src/CheckPoint.h @@ -75,6 +75,7 @@ void write_checkpoint_scr(float_sw4 a_time, int a_cycle, void define_pio(); void setSteps(int a_steps); void compute_file_suffix(int cycle, std::stringstream& fileSuffix); + void compute_file_suffix(const char* cycle, std::stringstream& fileSuffix); void cycle_checkpoints(string fname); void write_header(int& fid, float_sw4 a_time, int a_cycle, int& hsize); void read_header(int& fid, float_sw4& a_time, int& a_cycle, int& hsize); From a40421614cff923784d21f580c5b8a915b8c1b94 Mon Sep 17 00:00:00 2001 From: Adam Moody Date: Fri, 11 Feb 2022 16:25:27 -0800 Subject: [PATCH 5/6] keep previous SCR code in getDt --- src/CheckPoint.C | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index aeb78e64..3d41a283 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -650,6 +650,45 @@ float_sw4 CheckPoint::getDt() { } MPI_Bcast(&dt, 1, mEW->m_mpifloat, 0, mEW->m_cartesian_communicator); return dt; + +#if 0 + int have_restart = 0; + char checkpoint_dir[SCR_MAX_FILENAME]; + SCR_Have_restart(&have_restart, checkpoint_dir); + if (! have_restart) { + std::cerr<<"Error :: SCR found no checkpoints ! \n"<getRank()<<".bin"; + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + int valid=1; + if (std::FILE *file=std::fopen(scr_file,"rb")){ + float_sw4 dt; + if ( std::fread(&dt,sizeof dt,1,file)==1) { + scr_file_handle=file; + return dt; + } else { + std::cerr<<"ERROR:: Read of SCR checkpoint file failed in getDt \n"< Date: Fri, 11 Feb 2022 17:06:58 -0800 Subject: [PATCH 6/6] add scr to non-hdf path --- src/CheckPoint.C | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/CheckPoint.C b/src/CheckPoint.C index 3d41a283..c06d6233 100644 --- a/src/CheckPoint.C +++ b/src/CheckPoint.C @@ -342,9 +342,25 @@ void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle, s << fileSuffix.str(); } +#ifndef SW4_USE_SCR // Keep track of the number of files, save previous file name, and delete the // second last. cycle_checkpoints(s.str()); +#else + // SCR can delete older checkpoints, + // e.g., SCR_PREFIX_SIZE=3 to keep a sliding window of the last 3 + + // Inform SCR that a new checkpoint is starting + std::string cycle_num; + cycle_num << "cycle=" << a_cycle; + SCR_Start_output(cycle_num.str().c_str(), SCR_FLAG_CHECKPOINT); + + // Ask SCR for the path to write our checkpoint file + // This could be moved just before each H5FCreate call if need to preserve original file name + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + s = scr_file; +#endif // Open file from processor zero and write header. int hsize; @@ -445,6 +461,11 @@ void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle, delete[] doubleField; } if (iwrite) close(fid); + +#ifdef SW4_USE_SCR + int valid = 1; + SCR_Complete_output(valid); +#endif } // end write_checkpoint() //----------------------------------------------------------------------- @@ -468,7 +489,22 @@ void CheckPoint::read_checkpoint(float_sw4& a_time, int& a_cycle, s << mRestartPath << "/"; else if (mEW->getPath() != "./") s << mEW->getPath(); + +#ifndef SW4_USE_SCR s << mRestartFile; +#else + // TODO: need to get cycle_num from earlier call to Have_restart + + // TODO: this is not right, but you get the idea... + std::stringstream fileSuffix; + compute_file_suffix(cycle_num, fileSuffix); + s << fileSuffix.str(); + + // Ask SCR for the path to open our checkpoint file + char scr_file[SCR_MAX_FILENAME]; + SCR_Route_file(s.str().c_str(), scr_file); + s = scr_file; +#endif } // Open file from processor zero and read header. @@ -578,6 +614,11 @@ void CheckPoint::read_checkpoint(float_sw4& a_time, int& a_cycle, delete[] doubleField; } if (iread) close(fid); + +#ifdef SW4_USE_SCR + int valid = 1; + SCR_Complete_restart(valid); +#endif } //-----------------------------------------------------------------------