Add a loss comparison script

ghstack-source-id: ec9e87d
Pull-Request: #2029

Author: fegin

.github/workflows/integration_test_8gpu_features.yaml

@@ -76,5 +76,10 @@ jobs:
         export TEST_WITH_ROCM=$([[ "${{ matrix.gpu-arch-type }}" == "rocm" ]] && echo 1 || echo 0)
         python -m tests.integration_tests.run_tests --test_suite features $RUNNER_TEMP/artifacts-to-be-uploaded --ngpu 8
 
+        # Verify the accuracy.
+        export baseline_cmd='CONFIG_FILE="./torchtitan/models/llama3/train_configs/debug_model.toml" ./run_train.sh'
+        export baseline_cmd='CONFIG_FILE="./torchtitan/models/llama3/train_configs/debug_model.toml" ./run_train.sh --parallelism.data_parallel_replicate_degree=2'
+        python3 scripts/loss_compare.py . .  --baseline-cmd=${baseline_cmd} --test-cmd=${test_cmd} --no-seed-checkpoint --steps=10
+
         rm -rf $RUNNER_TEMP/artifacts-to-be-uploaded/*/checkpoint
         rm -rf artifacts-to-be-uploaded/*/checkpoint