Skip to content

Commit fe5598b

Browse files
oocxoocx
committed
fix(uat): block minimal artifacts and add validation helper + tests
1 parent ce179d5 commit fe5598b

7 files changed

Lines changed: 165 additions & 65 deletions

File tree

‎.github/agents/uat-tester.agent.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ Validate that generated markdown renders correctly in real-world PR environments
3232
- GitHub: `artifacts/comprehensive-demo-standard-diff.md` (standard diff format)
3333
- Azure DevOps: `artifacts/comprehensive-demo.md` (inline diff format, default)
3434
- For simulations: use `artifacts/uat-simulation-*.md` (requires `UAT_SIMULATE=true`)
35+
- **Do NOT use minimal artifacts for real UAT**: files with `minimal` or `uat-minimal` in their name are rejected unless `UAT_FORCE=true` is set
3536
- Before creating any PR, post the **exact Title and Description** in chat using the standard template (Problem / Change / Verification)
3637
- Post markdown as **PR comments** (not PR description)
3738
- Prefix comments with agent identifier (scripts do this automatically)

‎docs/features/consistent-value-formatting/retrospective.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ This section tracks follow-up implementation progress from this retrospective.
9999
| Skill authoring guidance (approval minimization) | ✅ Done | Added explicit guidance to prefer stable wrapper commands to reduce approvals ([#86](https://github.com/oocx/tfplan2md/pull/86)). |
100100
| UAT run/simulate skills foundation | ✅ Done | Initial UAT-related skills and scripts registered in workflow docs ([#85](https://github.com/oocx/tfplan2md/pull/85)). |
101101
| Standard metrics section in retrospectives | ✅ Done | Added/standardized metrics guidance in workflow docs/retrospective artifacts ([#84](https://github.com/oocx/tfplan2md/pull/84)). |
102-
| Guardrails against wrong UAT artifact | ✅ Partial | Added guardrails in UAT scripts and updated UAT guidance to prefer the UAT wrapper ([#95](https://github.com/oocx/tfplan2md/pull/95)). Canonical artifact enforcement still pending. |
102+
| Guardrails against wrong UAT artifact | ✅ Done | Added guardrails in UAT scripts and updated UAT guidance to prefer the UAT wrapper ([#95](https://github.com/oocx/tfplan2md/pull/95)). Filename-based canonical artifact enforcement implemented in `scripts/uat-helpers.sh` and tests added. |
103103
| Doc alignment gate in Code Review | ✅ Done | Added documentation alignment checklist gate to Code Reviewer ([#113](https://github.com/oocx/tfplan2md/pull/113)). |
104104
| Role boundaries + handoff/status templates | ✅ Done | Enforced role boundaries (Architect/Task Planner), added handoff template, added Developer status template ([#114](https://github.com/oocx/tfplan2md/pull/114)). |
105105
| Wire report style guide into agents | ✅ Done | Added report style guide references to Requirements Engineer, Developer, Code Reviewer, Technical Writer, UAT Tester ([#112](https://github.com/oocx/tfplan2md/pull/112)). |

‎scripts/uat-azdo.sh‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -30,36 +30,11 @@ log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
3030
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
3131
log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
3232

33-
# Validate and set default artifact path
34-
# Args: $1=artifact path (or empty for default), $2=simulate flag, $3=force flag
35-
validate_artifact() {
36-
local artifact="${1:-}"
37-
local simulate="${2:-false}"
38-
local force="${3:-false}"
39-
40-
# Default to inline-diff variant for Azure DevOps (comprehensive-demo.md is inline-diff)
41-
if [[ -z "$artifact" ]]; then
42-
artifact="artifacts/comprehensive-demo.md"
43-
log_info "No artifact specified, using Azure DevOps default: $artifact"
44-
fi
45-
46-
# Check if artifact exists
47-
if [[ ! -f "$artifact" ]]; then
48-
log_error "Artifact not found: $artifact"
49-
exit 1
50-
fi
51-
52-
# Block simulation artifacts in real UAT runs
53-
if [[ "$artifact" =~ simulation ]] && [[ "$simulate" != "true" ]] && [[ "$force" != "true" ]]; then
54-
log_error "Simulation artifact detected: $artifact"
55-
log_error "Simulation artifacts should not be used for real UAT."
56-
log_error "Use --simulate flag for simulation mode, or --force to override."
57-
exit 1
58-
fi
59-
60-
log_info "✓ Using artifact: $artifact"
61-
echo "$artifact"
62-
}
33+
# Artifact validation implemented in shared helper
34+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
35+
# shellcheck source=/dev/null
36+
source "$script_dir/uat-helpers.sh"
37+
6338

6439
cmd_setup() {
6540
log_info "Checking Azure CLI authentication..."
@@ -85,8 +60,8 @@ cmd_create() {
8560
local simulate="${UAT_SIMULATE:-false}"
8661
local force="${UAT_FORCE:-false}"
8762

88-
# Validate and potentially set default artifact
89-
file="$(validate_artifact "$file" "$simulate" "$force")"
63+
# Validate and potentially set default artifact (platform-aware)
64+
file="$(validate_artifact azdo "$file" "$simulate" "$force")"
9065

9166
local branch
9267
branch=$(git branch --show-current)

‎scripts/uat-github.sh‎

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,44 +20,19 @@ log_info() { echo -e "${GREEN}[INFO]${NC} $*"; }
2020
log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
2121
log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
2222

23-
# Validate and set default artifact path
24-
# Args: $1=artifact path (or empty for default), $2=simulate flag, $3=force flag
25-
validate_artifact() {
26-
local artifact="${1:-}"
27-
local simulate="${2:-false}"
28-
local force="${3:-false}"
29-
30-
# Default to standard-diff variant for GitHub
31-
if [[ -z "$artifact" ]]; then
32-
artifact="artifacts/comprehensive-demo-standard-diff.md"
33-
log_info "No artifact specified, using GitHub default: $artifact"
34-
fi
35-
36-
# Check if artifact exists
37-
if [[ ! -f "$artifact" ]]; then
38-
log_error "Artifact not found: $artifact"
39-
exit 1
40-
fi
41-
42-
# Block simulation artifacts in real UAT runs
43-
if [[ "$artifact" =~ simulation ]] && [[ "$simulate" != "true" ]] && [[ "$force" != "true" ]]; then
44-
log_error "Simulation artifact detected: $artifact"
45-
log_error "Simulation artifacts should not be used for real UAT."
46-
log_error "Use --simulate flag for simulation mode, or --force to override."
47-
exit 1
48-
fi
49-
50-
log_info "✓ Using artifact: $artifact"
51-
echo "$artifact"
52-
}
23+
# Artifact validation implemented in shared helper
24+
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
25+
# shellcheck source=/dev/null
26+
source "$script_dir/uat-helpers.sh"
27+
5328

5429
cmd_create() {
5530
local file="${1:-}"
5631
local simulate="${UAT_SIMULATE:-false}"
5732
local force="${UAT_FORCE:-false}"
5833

59-
# Validate and potentially set default artifact
60-
file="$(validate_artifact "$file" "$simulate" "$force")"
34+
# Validate and potentially set default artifact (platform-aware)
35+
file="$(validate_artifact github "$file" "$simulate" "$force")"
6136

6237
local branch
6338
branch=$(git branch --show-current)

‎scripts/uat-helpers.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/usr/bin/env bash
2+
# Shared UAT helper functions (artifact validation, etc.)
3+
# Intended to be sourced from scripts/uat-*.sh
4+
5+
set -euo pipefail
6+
7+
# Reuse log helpers defined in calling scripts; if they don't exist, provide no-op functions
8+
if ! declare -F log_info >/dev/null 2>&1; then
9+
log_info() { :; }
10+
fi
11+
if ! declare -F log_warn >/dev/null 2>&1; then
12+
log_warn() { :; }
13+
fi
14+
if ! declare -F log_error >/dev/null 2>&1; then
15+
# log_error should at least write to stderr
16+
log_error() { echo "$*" >&2; }
17+
fi
18+
19+
# validate_artifact <platform> <artifact-path-or-empty> <simulate:false|true> <force:false|true>
20+
# Returns: echoes the resolved artifact path on success; returns non-zero on failure
21+
validate_artifact() {
22+
local platform="${1:-}"
23+
local artifact="${2:-}"
24+
local simulate="${3:-false}"
25+
local force="${4:-false}"
26+
27+
if [[ -z "$platform" ]]; then
28+
log_error "validate_artifact: missing platform argument (github|azdo)"
29+
return 2
30+
fi
31+
32+
# Apply platform-specific defaults
33+
case "$platform" in
34+
github)
35+
if [[ -z "$artifact" ]]; then
36+
artifact="artifacts/comprehensive-demo-standard-diff.md"
37+
log_info "No artifact specified, using GitHub default: $artifact"
38+
fi
39+
;;
40+
azdo)
41+
if [[ -z "$artifact" ]]; then
42+
artifact="artifacts/comprehensive-demo.md"
43+
log_info "No artifact specified, using Azure DevOps default: $artifact"
44+
fi
45+
;;
46+
*)
47+
log_error "Unknown platform: $platform"
48+
return 2
49+
;;
50+
esac
51+
52+
# Check file existence
53+
if [[ ! -f "$artifact" ]]; then
54+
log_error "Artifact not found: $artifact"
55+
return 1
56+
fi
57+
58+
# Block known simulation/minimal artifacts unless simulate or force is set
59+
if [[ "$artifact" =~ (simulation|uat-simulation|minimal|uat-minimal) ]] && [[ "$simulate" != "true" ]] && [[ "$force" != "true" ]]; then
60+
log_error "Artifact appears to be a simulation or minimal artifact: $artifact"
61+
log_error "Simulation/minimal artifacts should not be used for real UAT."
62+
log_error "Use --simulate (UAT_SIMULATE=true) for simulation mode, or --force (UAT_FORCE=true) to override."
63+
return 1
64+
fi
65+
66+
log_info "✓ Using artifact: $artifact"
67+
echo "$artifact"
68+
}

‎scripts/uat-run.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,15 @@ if [[ -n "$artifact_arg" ]]; then
6565
artifact_azdo="$artifact_arg"
6666
fi
6767

68+
# Apply user-facing defaults for visibility, and summarize chosen artifacts before creating PRs
69+
if [[ -z "$artifact_github" ]]; then
70+
artifact_github="artifacts/comprehensive-demo-standard-diff.md"
71+
fi
72+
if [[ -z "$artifact_azdo" ]]; then
73+
artifact_azdo="artifacts/comprehensive-demo.md"
74+
fi
75+
log_info "Artifacts to be used: GitHub: $artifact_github, AzDO: $artifact_azdo"
76+
6877
# Note: Artifact existence checks moved to individual scripts
6978
# which will also apply smart defaults if artifact is empty
7079

‎tests/shell/uat_validate_test.sh‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
5+
cd "$REPO_ROOT"
6+
7+
# Ensure artifacts dir exists and defaults are present
8+
mkdir -p artifacts
9+
echo "# default github" > artifacts/comprehensive-demo-standard-diff.md
10+
echo "# default azdo" > artifacts/comprehensive-demo.md
11+
12+
# Source helper
13+
# shellcheck source=scripts/uat-helpers.sh
14+
source scripts/uat-helpers.sh
15+
16+
fail=0
17+
18+
# Test: minimal artifact should be rejected
19+
echo "# minimal" > artifacts/uat-minimal.md
20+
if validate_artifact github "artifacts/uat-minimal.md" false false >/dev/null 2>&1; then
21+
echo "ERROR: minimal artifact should have been rejected"
22+
fail=1
23+
else
24+
echo "OK: minimal artifact rejected"
25+
fi
26+
27+
# Test: force override must accept
28+
if artifact=$(validate_artifact github "artifacts/uat-minimal.md" false true); then
29+
echo "OK: minimal artifact accepted with force: $artifact"
30+
else
31+
echo "ERROR: minimal artifact should be accepted with force"
32+
fail=1
33+
fi
34+
35+
# Test: simulation artifact rejected without simulate
36+
echo "# sim" > artifacts/uat-simulation-2025-12-26.md
37+
if validate_artifact github "artifacts/uat-simulation-2025-12-26.md" false false >/dev/null 2>&1; then
38+
echo "ERROR: simulation artifact should have been rejected"
39+
fail=1
40+
else
41+
echo "OK: simulation artifact rejected"
42+
fi
43+
44+
# Test: simulation artifact accepted with simulate flag
45+
if artifact=$(validate_artifact github "artifacts/uat-simulation-2025-12-26.md" true false); then
46+
echo "OK: simulation artifact accepted with simulate flag: $artifact"
47+
else
48+
echo "ERROR: simulation artifact should be accepted with simulate"
49+
fail=1
50+
fi
51+
52+
# Test: defaults select existing files
53+
if artifact=$(validate_artifact github "" false false); then
54+
echo "OK: default github artifact selected: $artifact"
55+
else
56+
echo "ERROR: default github artifact selection failed"
57+
fail=1
58+
fi
59+
60+
if artifact=$(validate_artifact azdo "" false false); then
61+
echo "OK: default azdo artifact selected: $artifact"
62+
else
63+
echo "ERROR: default azdo artifact selection failed"
64+
fail=1
65+
fi
66+
67+
if [[ $fail -ne 0 ]]; then
68+
echo "One or more tests failed." >&2
69+
exit 1
70+
fi
71+
72+
echo "All tests passed."

0 commit comments

Comments
 (0)