Nightly #80
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly | |
| on: | |
| schedule: | |
| - cron: '0 6 * * *' # 6am UTC daily | |
| workflow_dispatch: # Allow manual trigger | |
| permissions: | |
| issues: write | |
| contents: read | |
| jobs: | |
| cli-tests: | |
| name: Go CLI Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.26' | |
| cache-dependency-path: cli/go.sum | |
| - name: Run CLI tests with coverage | |
| working-directory: cli | |
| run: | | |
| go test -race -coverprofile=coverage.out -covermode=atomic ./... -v | |
| echo "" | |
| echo "=== Coverage Summary ===" | |
| COVERAGE=$(go tool cover -func=coverage.out | tail -1 | awk '{print $3}') | |
| echo "Total coverage: $COVERAGE" | |
| echo "COVERAGE=$COVERAGE" >> "$GITHUB_ENV" | |
| - name: Test CLI builds | |
| working-directory: cli | |
| run: make build | |
| static-validation: | |
| name: Static Validation | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Run smoke tests | |
| run: | | |
| chmod +x tests/smoke-test.sh | |
| ./tests/smoke-test.sh --verbose | |
| - name: Run docs release gate | |
| run: | | |
| chmod +x tests/docs/validate-doc-release.sh | |
| ./tests/docs/validate-doc-release.sh | |
| - name: Validate hooks/docs parity | |
| run: | | |
| chmod +x scripts/validate-hooks-doc-parity.sh | |
| ./scripts/validate-hooks-doc-parity.sh | |
| retrieval-bench: | |
| name: Retrieval Quality Bench | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.26' | |
| cache-dependency-path: cli/go.sum | |
| - name: Run retrieval bench (synthetic corpus) | |
| working-directory: cli | |
| run: | | |
| go run ./cmd/ao retrieval-bench --json 2>&1 | tee bench-results.json | |
| # Fail if P@3 drops below 0.67 | |
| P3=$(go run ./cmd/ao retrieval-bench 2>&1 | grep "Precision@3" | awk '{print $2}') | |
| echo "Precision@3: $P3" | |
| if awk "BEGIN{exit !($P3 < 0.67)}"; then | |
| echo "FAIL: Retrieval precision dropped below 0.67" | |
| exit 1 | |
| fi | |
| - name: Run retrieval bench (representative live corpus) | |
| working-directory: cli | |
| run: | | |
| go run ./cmd/ao retrieval-bench --live --corpus cmd/ao/testdata/retrieval-bench-live --json > bench-live.json | |
| python3 - <<'PY' | |
| import json | |
| import sys | |
| with open("bench-live.json", "r", encoding="utf-8") as fh: | |
| data = json.load(fh) | |
| coverage = float(data.get("coverage", 0.0)) | |
| total = int(data.get("total_learnings", 0)) | |
| empty = [item["query"] for item in data.get("results", []) if int(item.get("count", 0)) == 0] | |
| print(f"Live coverage: {coverage:.2f}") | |
| print(f"Live total learnings: {total}") | |
| if total < 8: | |
| print("FAIL: representative live corpus is too small to be meaningful", file=sys.stderr) | |
| sys.exit(1) | |
| if coverage < 0.8: | |
| print(f"FAIL: live retrieval coverage dropped below 0.80 (empty queries: {empty})", file=sys.stderr) | |
| sys.exit(1) | |
| PY | |
| security-toolchain: | |
| name: Security Toolchain | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.26' | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - name: Install scanner tools | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install semgrep ruff radon pytest | |
| GOBIN=/usr/local/bin go install github.com/securego/gosec/v2/cmd/gosec@latest | |
| GOBIN=/usr/local/bin go install github.com/zricethezav/gitleaks/v8@latest | |
| # golangci-lint | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b /usr/local/bin | |
| # trivy | |
| curl -sSfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin | |
| # hadolint | |
| curl -sSfL -o /usr/local/bin/hadolint "https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-x86_64" | |
| chmod +x /usr/local/bin/hadolint | |
| - name: Run security gate (full) | |
| run: | | |
| chmod +x scripts/security-gate.sh | |
| ./scripts/security-gate.sh --mode full | |
| env: | |
| SECURITY_GATE_OUTPUT_DIR: ${{ runner.temp }}/agentops-security | |
| TOOLCHAIN_OUTPUT_DIR: ${{ runner.temp }}/agentops-tooling | |
| - name: Upload security gate artifacts | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: nightly-security-gate | |
| path: ${{ runner.temp }}/agentops-security/ | |
| retention-days: 7 | |
| compile: | |
| name: Compile Knowledge Cycle | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 # full git history for mining | |
| - name: Set up Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.26' | |
| cache-dependency-path: cli/go.sum | |
| - name: Build ao | |
| run: cd cli && make build | |
| - name: Mine — extract signal from git and .agents/ | |
| run: ./cli/bin/ao mine --since 26h --sources git,agents --output-dir /tmp/compile/mine | |
| continue-on-error: true # mine failure does not block defrag | |
| - name: Defrag — prune orphans, dedup, oscillation sweep | |
| run: | | |
| ./cli/bin/ao defrag \ | |
| --prune --dedup --oscillation-sweep \ | |
| --output-dir /tmp/compile/defrag | |
| - name: Check Compile health gates | |
| run: bash scripts/check-compile-health.sh | |
| env: | |
| COMPILE_OUTPUT_DIR: /tmp/compile | |
| COMPILE_MAX_AGE_HOURS: 30 | |
| - name: Upload Compile report | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: compile-report | |
| path: /tmp/compile/ | |
| retention-days: 14 | |
| summary: | |
| name: Summary | |
| needs: [cli-tests, static-validation, security-toolchain, compile] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Summary | |
| run: | | |
| echo "## Nightly Test Results" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| CLI tests | ${{ needs.cli-tests.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Static validation | ${{ needs.static-validation.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Security toolchain | ${{ needs.security-toolchain.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Compile knowledge cycle | ${{ needs.compile.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Run completed at $(date -u)" >> $GITHUB_STEP_SUMMARY | |
| - name: Create issue on failure | |
| if: needs.cli-tests.result == 'failure' || needs.static-validation.result == 'failure' || needs.security-toolchain.result == 'failure' || needs.compile.result == 'failure' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| TITLE="Nightly build failed: $(date -u +%Y-%m-%d)" | |
| BODY="## Nightly Build Failure | |
| | Job | Status | | |
| |-----|--------| | |
| | CLI tests | ${{ needs.cli-tests.result }} | | |
| | Static validation | ${{ needs.static-validation.result }} | | |
| | Security toolchain | ${{ needs.security-toolchain.result }} | | |
| | Compile knowledge cycle | ${{ needs.compile.result }} | | |
| **Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| Please investigate and fix before the next release." | |
| LABEL_ARGS=(--label bug) | |
| if gh label list --limit 200 --json name --jq '.[].name' | grep -Fxq "nightly-failure"; then | |
| LABEL_ARGS+=(--label nightly-failure) | |
| fi | |
| # Check if issue already exists for today | |
| EXISTING=$(gh issue list --state open --search "\"$TITLE\" in:title" --json number,title --jq 'map(select(.title == "'"$TITLE"'")) | .[0].number' 2>/dev/null || true) | |
| if [[ -z "$EXISTING" || "$EXISTING" == "null" ]]; then | |
| gh issue create --title "$TITLE" --body "$BODY" "${LABEL_ARGS[@]}" | |
| else | |
| echo "Issue #$EXISTING already exists for today's failure" | |
| fi | |
| - name: Check for failures | |
| if: needs.cli-tests.result == 'failure' || needs.static-validation.result == 'failure' || needs.security-toolchain.result == 'failure' || needs.compile.result == 'failure' | |
| run: exit 1 |