fix rclone install

fix re-run passed tests to not fail the ci
add file cleanup after execution

Author: btc-fan

.github/workflows/build_and_run_chain_simulator_and_execute_system_test.yml

@@ -142,7 +142,17 @@ jobs:
           TS=$(date +'%Y_%^B_%d__%H_%M_%S'); 
           echo "TS=$TS" >> "$GITHUB_ENV"
           echo "TIMESTAMP=$TS" >> "$GITHUB_ENV"
+
+          # Fix any interrupted dpkg operations first
+          sudo dpkg --configure -a || true
+
+          # Clean up any partial packages
+          sudo apt-get clean
+          sudo apt-get autoclean
+
+          # Update package lists and install rclone
           sudo apt-get update -y && sudo apt-get install -y rclone
+
           mkdir -p ~/.config/rclone
           cat > ~/.config/rclone/rclone.conf <<EOF
           [r2]
@@ -276,17 +286,55 @@ jobs:
         run: |
           set +e
           pytest mx-chain-testing-suite/scenarios/ --html=report.html --self-contained-html --continue-on-collection-errors
-          PYTEST_EXIT_CODE=$?
+          INITIAL_PYTEST_EXIT_CODE=$?
           set -e
-          echo "PYTEST_EXIT_CODE=$PYTEST_EXIT_CODE" >> $GITHUB_ENV
-          echo "Pytest exit code: $PYTEST_EXIT_CODE"
+
+          echo "Initial pytest exit code: $INITIAL_PYTEST_EXIT_CODE"
+
+          # Parse the HTML report to determine actual test status
+          FINAL_PYTEST_EXIT_CODE=$INITIAL_PYTEST_EXIT_CODE
+
           if [ -f "report.html" ]; then
             echo "Report generated successfully."
+
+            # Extract test summary from HTML report using multiple parsing approaches
+            FAILED_TESTS=$(grep -E '[0-9]+ Failed|Failed.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+            ERROR_TESTS=$(grep -E '[0-9]+ Errors|Errors.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+            RERUN_TESTS=$(grep -E '[0-9]+ Reruns|Reruns.*[0-9]+' report.html | head -1 | grep -o '[0-9]\+' | head -1 || echo "0")
+
+            # Fallback: check for "0 Failed" pattern directly
+            if grep -q "0 Failed" report.html && [ "$INITIAL_PYTEST_EXIT_CODE" -ne 0 ]; then
+              echo "Detected '0 Failed' pattern in report despite non-zero exit code - likely successful reruns"
+              FAILED_TESTS=0
+            fi
+
+            echo "Failed tests: $FAILED_TESTS"
+            echo "Error tests: $ERROR_TESTS" 
+            echo "Rerun tests: $RERUN_TESTS"
+
+            # Determine final status based on actual test results
+            if [ "$FAILED_TESTS" -eq 0 ] && [ "$ERROR_TESTS" -eq 0 ]; then
+              if [ "$RERUN_TESTS" -gt 0 ]; then
+                echo "✅ All tests passed after $RERUN_TESTS reruns - considering as SUCCESS"
+              else
+                echo "✅ All tests passed on first attempt"
+              fi
+              FINAL_PYTEST_EXIT_CODE=0
+            else
+              echo "❌ Tests have genuine failures: $FAILED_TESTS failed, $ERROR_TESTS errors"
+              FINAL_PYTEST_EXIT_CODE=1
+            fi
+
+            # Move report to reports directory
             mkdir -p ./reports
             mv report.html ./reports/
           else
-            echo "Report not found."
+            echo "❌ Report not found - using original exit code"
+            FINAL_PYTEST_EXIT_CODE=$INITIAL_PYTEST_EXIT_CODE
           fi
+
+          echo "PYTEST_EXIT_CODE=$FINAL_PYTEST_EXIT_CODE" >> $GITHUB_ENV
+          echo "Final pytest exit code: $FINAL_PYTEST_EXIT_CODE"
           
       - name: Stage report for R2
         run: |
@@ -482,4 +530,80 @@ jobs:
             exit 1
           else
             echo "Tests passed successfully."
-          fi
+          fi
+          
+
+      - name: Cleanup Workspace and Processes
+        if: always()
+        run: |
+          echo "🧹 Starting SAFE cleanup for job ${{ github.run_id }}..."
+          
+          # Get current working directory to limit cleanup scope
+          WORKSPACE_DIR=$(pwd)
+          echo "Workspace directory: $WORKSPACE_DIR"
+          
+          # 1. Kill only OUR ChainSimulator processes (by checking working directory)
+          echo "Killing ChainSimulator processes from this workspace..."
+          for pid in $(pgrep -f "chainsimulator"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "Killing chainsimulator process $pid from our workspace"
+                kill -TERM $pid 2>/dev/null || true
+                sleep 2
+                kill -9 $pid 2>/dev/null || true
+              fi
+            fi
+          done
+          
+          # 2. Kill only OUR pytest processes (by checking if they're in our workspace)
+          echo "Killing pytest processes from this workspace..."
+          for pid in $(pgrep -f "pytest.*scenarios"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "Killing pytest process $pid from our workspace"
+                kill -TERM $pid 2>/dev/null || true
+              fi
+            fi
+          done
+          
+          # 3. Remove only OUR temporary files (with unique identifiers)
+          echo "Removing temporary files from this job..."
+          rm -f /tmp/chainsim_init_${{ github.run_id }}.log || true
+          rm -f /tmp/chain_simulator_initialized_${{ github.run_id }}.lock || true
+          # Only remove temp files that might be from this job (be very specific)
+          rm -f /tmp/chainsim_init.log || true  # Only if we created it without job ID
+          rm -f /tmp/chain_simulator_initialized.lock || true  # Only if we created it without job ID
+          
+          # 4. Clean up ONLY our workspace directories
+          echo "Cleaning up workspace directories..."
+          rm -rf "$WORKSPACE_DIR/mx-chain-go" || true
+          rm -rf "$WORKSPACE_DIR/mx-chain-simulator-go" || true
+          rm -rf "$WORKSPACE_DIR/mx-chain-testing-suite" || true
+          rm -rf "$WORKSPACE_DIR/reports" || true
+          rm -rf "$WORKSPACE_DIR/r2_upload" || true
+          rm -f "$WORKSPACE_DIR/report.html" || true
+          
+          # 5. Clean up Python cache ONLY in our workspace
+          echo "Cleaning Python cache in workspace..."
+          find "$WORKSPACE_DIR" -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true
+          find "$WORKSPACE_DIR" -type d -name "*.egg-info" -exec rm -rf {} + 2>/dev/null || true
+          find "$WORKSPACE_DIR" -type f -name "*.pyc" -delete 2>/dev/null || true
+
+          # 6. Check disk space (informational only)
+          echo "Checking remaining disk space..."
+          df -h "$WORKSPACE_DIR" || true
+          
+          # 7. Show what processes are still running FROM OUR WORKSPACE (for debugging)
+          echo "Checking for remaining processes from our workspace:"
+          for pid in $(pgrep -f "chainsimulator\|pytest.*scenarios"); do
+            if [ -n "$pid" ]; then
+              CWD=$(pwdx $pid 2>/dev/null | awk '{print $2}' || echo "unknown")
+              if [[ "$CWD" == *"$WORKSPACE_DIR"* ]]; then
+                echo "⚠️ Process $pid still running from our workspace: $CWD"
+              fi
+            fi
+          done
+          
+          echo "✅ Safe cleanup completed for job ${{ github.run_id }}!"