Merge pull request #3564 from heplesser/fix-update-timer

Fix data race in mechanism preventing excessive update times

Author: heplesser

nestkernel/simulation_manager.cpp

@@ -831,8 +831,10 @@ nest::SimulationManager::update_()
   bool done_all = true;
   long old_to_step;
 
+  // These variables will be updated only by the master thread below
   double start_current_update = sw_simulate_.elapsed();
   bool update_time_limit_exceeded = false;
+  // End of variables updated by master thread
 
   std::vector< std::shared_ptr< WrappedThreadException > > exceptions_raised( kernel().vp_manager.get_num_threads() );
 
@@ -1084,17 +1086,29 @@ nest::SimulationManager::update_()
             print_progress_();
           }
 
-          // We cannot throw exception inside master, would not get caught.
+          // Track time needed for single update cycle
           const double end_current_update = sw_simulate_.elapsed();
           const double update_time = end_current_update - start_current_update;
-          update_time_limit_exceeded = update_time > update_time_limit_;
+          start_current_update = end_current_update;
+
           min_update_time_ = std::min( min_update_time_, update_time );
           max_update_time_ = std::max( max_update_time_, update_time );
-          start_current_update = end_current_update;
+
+          // If the simulation slowed down excessively, we cannot throw an exception here
+          // in the master section, as it will not be caught by our mechanism for handling
+          // exceptions in parallel context. So we set a flag and process it immediately
+          // after the master section.
+          update_time_limit_exceeded = update_time > update_time_limit_;
         }
 // end of master section, all threads have to synchronize at this point
 #pragma omp barrier
 
+        if ( update_time_limit_exceeded )
+        {
+          LOG( M_ERROR, "SimulationManager::update", "Update time limit exceeded." );
+          throw KernelException();
+        }
+
         // if block to avoid omp barrier if SIONLIB is not used
 #ifdef HAVE_SIONLIB
         kernel().io_manager.post_step_hook();
@@ -1104,15 +1118,7 @@ nest::SimulationManager::update_()
         kernel().get_omp_synchronization_simulation_stopwatch().stop();
 #endif
 
-        const double end_current_update = sw_simulate_.elapsed();
-        if ( end_current_update - start_current_update > update_time_limit_ )
-        {
-          LOG( M_ERROR, "SimulationManager::update", "Update time limit exceeded." );
-          throw KernelException();
-        }
-        start_current_update = end_current_update;
-
-      } while ( to_do_ > 0 and not update_time_limit_exceeded and not exceptions_raised.at( tid ) );
+      } while ( to_do_ > 0 );
 
       // End of the slice, we update the number of synaptic elements
       for ( SparseNodeArray::const_iterator i = kernel().node_manager.get_local_nodes( tid ).begin();

testsuite/summarize_tests.py

@@ -122,6 +122,7 @@ def parse_result_file(fname):
             results[ph_name] = {"Tests": 0, "Skipped": 0, "Failures": 0, "Errors": 0, "Time": 0, "Failed tests": [msg]}
             totals["Failed tests"].append(msg)
 
+    missing_phases = []
     cols = ["Tests", "Skipped", "Failures", "Errors", "Time"]
 
     col_w = max(len(c) for c in cols) + 2
@@ -143,7 +144,10 @@ def parse_result_file(fname):
     print(tline)
     for pn, pr in results.items():
         print(f"{pn:<{first_col_w}s}", end="")
-        if pr["Tests"] == 0 and pr["Failed tests"]:
+        if pr is None:
+            print(f"{'--- RESULTS MISSING FOR PHASE ---':^{len(cols) * col_w}}")
+            missing_phases.append(pn)
+        elif pr["Tests"] == 0 and pr["Failed tests"]:
             print(f"{'--- XML PARSING FAILURE ---':^{len(cols) * col_w}}")
         else:
             for c in cols: