Add captured actual outputs to CI artifacts (#40965)

* fix

* fix

* Remove `# TODO: ???` as it make me `???`

* fix

* fix

* fix

---------

Co-authored-by: ydshieh <[email protected]>

Author: ydshieh

.github/workflows/model_jobs.yml

@@ -128,28 +128,35 @@ jobs:
           echo "machine_type=$machine_type" >> $GITHUB_ENV
           echo "machine_type=$machine_type" >> $GITHUB_OUTPUT
 
+      - name: Create report directory if it doesn't exist
+        shell: bash
+        run: |
+          mkdir -p /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+          echo "dummy" > /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/dummy.txt
+          ls -la /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
+
       - name: Run all tests on GPU
         working-directory: /transformers
-        run: python3 -m pytest -rsfE -v --make-reports=${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
+        run: |
+          PATCH_TESTING_METHODS_TO_COLLECT_OUTPUTS=yes _PATCHED_TESTING_METHODS_OUTPUT_DIR=/transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports python3 -m pytest -rsfE -v --make-reports=${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports tests/${{ matrix.folders }}
 
       - name: Failure short reports
         if: ${{ failure() }}
         continue-on-error: true
-        run: cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports/failures_short.txt
+        run: cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/failures_short.txt
 
-      - name: Run test
-        shell: bash
+      - name: Captured information
+        if: ${{ failure() }}
+        continue-on-error: true
         run: |
-          mkdir -p /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports
-          echo "hello" > /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports/hello.txt
-          echo "${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports"
+          cat /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports/captured_info.txt
 
       - name: "Test suite reports artifacts: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports"
         if: ${{ always() }}
         uses: actions/upload-artifact@v4
         with:
           name: ${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
-          path: /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ matrix.folders }}_test_reports
+          path: /transformers/reports/${{ env.machine_type }}_${{ inputs.report_name_prefix }}_${{ env.matrix_folders }}_test_reports
 
   collated_reports:
     name: Collated Reports

utils/notification_service.py

@@ -1196,6 +1196,15 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
             "time_spent": [],
             "failures": {},
             "job_link": {},
+            "captured_info": {},
+        }
+        for matrix_name in job_matrix
+        if f"{report_name_prefix}_{matrix_name}_test_reports" in available_artifacts
+    }
+
+    matrix_job_results_extra = {
+        matrix_name: {
+            "captured_info": {},
         }
         for matrix_name in job_matrix
         if f"{report_name_prefix}_{matrix_name}_test_reports" in available_artifacts
@@ -1225,7 +1234,21 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
 
                 stacktraces = handle_stacktraces(artifact["failures_line"])
 
-                # TODO: ???
+                # Add the captured actual outputs for patched methods (`torch.testing.assert_close`, `assertEqual` etc.)
+                if "captured_info" in artifact:
+                    step_number = None
+                    for step in job.get("steps", []):
+                        if step["name"] == "Captured information":
+                            step_number = step["number"]
+                            break
+                    if step_number is not None:
+                        step_link = f"{job['html_url']}#step:{step_number}:1"
+                        matrix_job_results[matrix_name]["captured_info"][artifact_gpu] = step_link
+                        matrix_job_results_extra[matrix_name]["captured_info"][artifact_gpu] = {
+                            "link": step_link,
+                            "captured_info": artifact["captured_info"],
+                        }
+
                 for line in artifact["summary_short"].split("\n"):
                     if line.startswith("FAILED "):
                         # Avoid the extra `FAILED` entry given by `run_test_using_subprocess` causing issue when calling
@@ -1432,6 +1455,20 @@ def pop_default(l: list[Any], i: int, default: Any) -> Any:
             token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
         )
 
+    if len(matrix_job_results_extra) > 0:
+        with open(
+            f"ci_results_{job_name}/{test_to_result_name[test_name]}_results_extra.json", "w", encoding="UTF-8"
+        ) as fp:
+            json.dump(matrix_job_results_extra, fp, indent=4, ensure_ascii=False)
+
+        api.upload_file(
+            path_or_fileobj=f"ci_results_{job_name}/{test_to_result_name[test_name]}_results_extra.json",
+            path_in_repo=f"{report_repo_folder}/ci_results_{job_name}/{test_to_result_name[test_name]}_results_extra.json",
+            repo_id=report_repo_id,
+            repo_type="dataset",
+            token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None),
+        )
+
     # Let's create a file contain job --> job link
     if len(matrix_job_results) > 0:
         target_results = matrix_job_results