Dev dlrm offline eval

RecommenderSystems/dlrm/config.py

@@ -13,8 +13,10 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import os
 import argparse
 import oneflow as flow
+from datetime import datetime
 
 
 def get_args(print_args=True):
@@ -127,6 +129,11 @@ def str_list(x):
         flow.env.get_world_size() > 1 and not args.ddp
     ) or args.execution_mode == "graph"
 
+    if args.eval_save_dir != '':
+        time_str = str(datetime.now().strftime("%Y%m%d%H%M%S"))
+        args.eval_save_dir = os.path.join(args.eval_save_dir, f'eval_results-{time_str}')
+        if not os.path.exists(args.eval_save_dir):
+            os.makedirs(args.eval_save_dir)
     if print_args and flow.env.get_rank() == 0:
         _print_args(args)
     return args

RecommenderSystems/dlrm/train.py

@@ -168,9 +168,8 @@ def train(self):
                 self.eval(self.save_model_after_each_eval)
         if self.eval_after_training:
             self.eval(True)
-
-        if self.args.eval_save_dir != '' and self.eval_after_training:
-            calculate_auc_from_dir(self.args.eval_save_dir)
+            if self.args.eval_save_dir != '' and self.rank == 0:
+                calculate_auc_from_dir(self.args.eval_save_dir)
 
     def eval(self, save_model=False):
         if self.eval_batchs <= 0:
@@ -187,12 +186,13 @@ def eval(self, save_model=False):
             labels.append(label_)
             preds.append(pred.numpy())
         if self.args.eval_save_dir != '':
-            pf = os.path.join(self.args.eval_save_dir, f'eval_results_iter_{self.cur_iter}.pkl')
-            with open(pf, 'wb') as f:
-                obj = {'labels': labels, 'preds': preds, 'iter': self.cur_iter}
-                pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL)
-            # auc = roc_auc_score(label_, pred.numpy())
-            auc = 'nc'
+            if self.rank == 0:
+                pf = os.path.join(self.args.eval_save_dir, f'iter_{self.cur_iter}.pkl')
+                with open(pf, 'wb') as f:
+                    obj = {'labels': labels, 'preds': preds, 'iter': self.cur_iter}
+                    pickle.dump(obj, f, protocol=pickle.HIGHEST_PROTOCOL)
+            auc = roc_auc_score(label_, pred.numpy())
+            # auc = 'nc'
         else:
             labels = np.concatenate(labels, axis=0)
             preds = np.concatenate(preds, axis=0)

RecommenderSystems/dlrm/train_one_embedding_graph.sh

@@ -49,5 +49,5 @@ python3 -m oneflow.distributed.launch \
     --blocked_based_path $BLOCK_BASED_PATH \
     --column_size_array '227605432,39060,17295,7424,20265,3,7122,1543,63,130229467,3067956,405282,10,2209,11938,155,4,976,14,292775614,40790948,187188510,590152,12973,108,36' \
     --test_name 'train_one_embedding_graph_'$DEVICE_NUM_PER_NODE'gpu' | tee 'train_one_embedding_graph_'$DEVICE_NUM_PER_NODE'gpu'.log
-    #--eval_save_dir '/NVME0/guoran/auc/' \
+    #--eval_save_dir '/NVME0/guoran/auc' \
     #--eval_after_training \

RecommenderSystems/dlrm/utils/auc_calculater.py

@@ -19,7 +19,7 @@ def calculate_auc_from_file(pkl):
     print(f'Iter {iter} AUC: {auc:0.4f}, Num of Evaluation: {labels.shape[0]}, time:{duration:0.3f}')
 
 
-def calculate_auc_from_dir(directory, startswith='eval_results_iter'):
+def calculate_auc_from_dir(directory, startswith='iter'):
     print('calculate AUC from folder:', directory)
     for file in os.listdir(directory):
         filename = os.fsdecode(file)