Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 41 additions & 14 deletions claudecode/evals/eval_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ def __init__(self, work_dir: str = None, verbose: bool = False):
self.github_token = os.environ.get('GITHUB_TOKEN', '')
if not self.github_token:
try:
result = subprocess.run(['gh', 'auth', 'token'],
result = subprocess.run(['gh', 'auth', 'token'],
capture_output=True, text=True, timeout=TIMEOUT_GIT_OPERATION)
if result.returncode == 0:
self.github_token = result.stdout.strip()
Expand All @@ -92,6 +92,20 @@ def __init__(self, work_dir: str = None, verbose: bool = False):
except (subprocess.SubprocessError, FileNotFoundError) as e:
self.log(f"Could not retrieve GitHub token from gh CLI: {e}")

# Get GitHub host for Enterprise support
# GITHUB_API_URL is set by GitHub Actions for Enterprise instances
self.github_host = 'github.com'
github_api_url = os.environ.get('GITHUB_API_URL', '')
if github_api_url:
try:
from urllib.parse import urlparse
parsed = urlparse(github_api_url)
if parsed.hostname and parsed.hostname != 'api.github.com':
self.github_host = parsed.hostname
self.log(f"Using GitHub Enterprise host: {self.github_host}")
except Exception as e:
self.log(f"Could not parse GITHUB_API_URL: {e}")


def log(self, message: str, prefix: str = "[EVAL]") -> None:
"""Log a message if verbose mode is enabled."""
Expand Down Expand Up @@ -232,40 +246,50 @@ def _setup_repository(self, test_case: EvalCase) -> Tuple[bool, str, str]:
# Clone or update the base repository
if not os.path.exists(base_repo_path):
self.log(f"Cloning {repo_name} to {base_repo_path}")
clone_url = f"https://github.com/{repo_name}.git"
# Use x-access-token as username for PAT authentication (works with GitHub and GHE)
if self.github_token:
clone_url = f"https://{self.github_token}@github.com/{repo_name}.git"

clone_url = f"https://x-access-token:{self.github_token}@{self.github_host}/{repo_name}.git"
else:
clone_url = f"https://{self.github_host}/{repo_name}.git"

# Set GIT_TERMINAL_PROMPT=0 to prevent git from prompting for credentials
clone_env = os.environ.copy()
clone_env['GIT_TERMINAL_PROMPT'] = '0'

try:
subprocess.run(['git', 'clone', '--filter=blob:none', clone_url, base_repo_path],
check=True, capture_output=True, timeout=TIMEOUT_CLONE)
check=True, capture_output=True, timeout=TIMEOUT_CLONE, env=clone_env)
except subprocess.CalledProcessError as e:
error_msg = f"Failed to clone repository: {e.stderr.decode()}"
self.log(error_msg)
return False, "", error_msg

# Clean up any stale worktrees for this evaluation
eval_branch_prefix = f"eval-pr-{safe_repo_name}-{pr_number}"
self._clean_worktrees(base_repo_path, eval_branch_prefix)

# Create worktree for this specific evaluation
eval_branch = self._get_eval_branch_name(test_case)
worktree_path = os.path.join(self.work_dir, f"{safe_repo_name}_pr{pr_number}_{int(time.time())}")


# Environment to prevent git from prompting for credentials
git_env = os.environ.copy()
git_env['GIT_TERMINAL_PROMPT'] = '0'

try:
# Fetch the PR
self.log(f"Fetching PR #{pr_number} from {repo_name}")
subprocess.run(['git', '-C', base_repo_path, 'fetch', 'origin', f'pull/{pr_number}/head'],
check=True, capture_output=True, timeout=TIMEOUT_FETCH)
check=True, capture_output=True, timeout=TIMEOUT_FETCH, env=git_env)

# Create new worktree with PR changes
self.log(f"Creating worktree at {worktree_path}")
subprocess.run(['git', '-C', base_repo_path, 'worktree', 'add', '-b', eval_branch,
subprocess.run(['git', '-C', base_repo_path, 'worktree', 'add', '-b', eval_branch,
worktree_path, 'FETCH_HEAD'],
check=True, capture_output=True, timeout=TIMEOUT_WORKTREE_CREATE)
check=True, capture_output=True, timeout=TIMEOUT_WORKTREE_CREATE, env=git_env)

return True, worktree_path, ""

except subprocess.CalledProcessError as e:
error_msg = f"Failed to set up worktree: {e.stderr.decode()}"
self.log(error_msg)
Expand Down Expand Up @@ -410,6 +434,9 @@ def _run_sast_audit(self, test_case: EvalCase, repo_path: str) -> Tuple[bool, st
env['ANTHROPIC_API_KEY'] = self.claude_api_key
if self.github_token:
env['GITHUB_TOKEN'] = self.github_token
# Pass through GitHub API URL for Enterprise support
if os.environ.get('GITHUB_API_URL'):
env['GITHUB_API_URL'] = os.environ['GITHUB_API_URL']
env['EVAL_MODE'] = '1' # Enable eval mode

# Run the audit script
Expand Down
21 changes: 14 additions & 7 deletions claudecode/github_action_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,26 @@ class AuditError(ValueError):

class GitHubActionClient:
"""Simplified GitHub API client for GitHub Actions environment."""

def __init__(self):
"""Initialize GitHub client using environment variables."""
self.github_token = os.environ.get('GITHUB_TOKEN')
if not self.github_token:
raise ValueError("GITHUB_TOKEN environment variable required")

self.headers = {
'Authorization': f'Bearer {self.github_token}',
'Accept': 'application/vnd.github.v3+json',
'X-GitHub-Api-Version': '2022-11-28'
}


# Get GitHub API URL - supports GitHub Enterprise
# GITHUB_API_URL is automatically set by GitHub Actions for Enterprise instances
# Falls back to public GitHub API if not set
self.api_base_url = os.environ.get('GITHUB_API_URL', 'https://api.github.com').rstrip('/')
if self.api_base_url != 'https://api.github.com':
print(f"[Info] Using GitHub Enterprise API: {self.api_base_url}", file=sys.stderr)

# Get excluded directories from environment
exclude_dirs = os.environ.get('EXCLUDE_DIRECTORIES', '')
self.excluded_dirs = [d.strip() for d in exclude_dirs.split(',') if d.strip()] if exclude_dirs else []
Expand All @@ -69,13 +76,13 @@ def get_pr_data(self, repo_name: str, pr_number: int) -> Dict[str, Any]:
Dictionary containing PR data
"""
# Get PR metadata
pr_url = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
pr_url = f"{self.api_base_url}/repos/{repo_name}/pulls/{pr_number}"
response = requests.get(pr_url, headers=self.headers)
response.raise_for_status()
pr_data = response.json()

# Get PR files with pagination support
files_url = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}/files?per_page=100"
files_url = f"{self.api_base_url}/repos/{repo_name}/pulls/{pr_number}/files?per_page=100"
response = requests.get(files_url, headers=self.headers)
response.raise_for_status()
files_data = response.json()
Expand Down Expand Up @@ -126,7 +133,7 @@ def get_pr_diff(self, repo_name: str, pr_number: int) -> str:
Returns:
Complete PR diff in unified format
"""
url = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
url = f"{self.api_base_url}/repos/{repo_name}/pulls/{pr_number}"
headers = dict(self.headers)
headers['Accept'] = 'application/vnd.github.diff'

Expand Down
31 changes: 25 additions & 6 deletions claudecode/test_github_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,26 @@ def test_init_with_token(self):
assert client.headers['Authorization'] == 'Bearer test-token'
assert 'Accept' in client.headers
assert 'X-GitHub-Api-Version' in client.headers

assert client.api_base_url == 'https://api.github.com'

def test_init_with_github_enterprise(self):
"""Test initialization with GitHub Enterprise URL."""
with patch.dict(os.environ, {
'GITHUB_TOKEN': 'test-token',
'GITHUB_API_URL': 'https://github.mycompany.com/api/v3'
}):
client = GitHubActionClient()
assert client.api_base_url == 'https://github.mycompany.com/api/v3'

def test_init_with_github_enterprise_trailing_slash(self):
"""Test initialization strips trailing slash from Enterprise URL."""
with patch.dict(os.environ, {
'GITHUB_TOKEN': 'test-token',
'GITHUB_API_URL': 'https://github.mycompany.com/api/v3/'
}):
client = GitHubActionClient()
assert client.api_base_url == 'https://github.mycompany.com/api/v3'

@patch('requests.get')
def test_get_pr_data_success(self, mock_get):
"""Test successful PR data retrieval."""
Expand Down Expand Up @@ -90,14 +109,14 @@ def test_get_pr_data_success(self, mock_get):
client = GitHubActionClient()
result = client.get_pr_data('owner/repo', 123)

# Verify API calls
# Verify API calls use the configured base URL
assert mock_get.call_count == 2
mock_get.assert_any_call(
'https://api.github.com/repos/owner/repo/pulls/123',
f'{client.api_base_url}/repos/owner/repo/pulls/123',
headers=client.headers
)
mock_get.assert_any_call(
'https://api.github.com/repos/owner/repo/pulls/123/files?per_page=100',
f'{client.api_base_url}/repos/owner/repo/pulls/123/files?per_page=100',
headers=client.headers
)

Expand Down Expand Up @@ -185,10 +204,10 @@ def main():
client = GitHubActionClient()
result = client.get_pr_diff('owner/repo', 123)

# Verify API call
# Verify API call uses the configured base URL
mock_get.assert_called_once()
call_args = mock_get.call_args
assert call_args[0][0] == 'https://api.github.com/repos/owner/repo/pulls/123'
assert call_args[0][0] == f'{client.api_base_url}/repos/owner/repo/pulls/123'
assert call_args[1]['headers']['Accept'] == 'application/vnd.github.diff'

# Verify result
Expand Down
28 changes: 25 additions & 3 deletions scripts/comment-pr-findings.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,38 @@ const context = {
function ghApi(endpoint, method = 'GET', data = null) {
// Build arguments array safely to prevent command injection
const args = ['api', endpoint, '--method', method];

if (data) {
args.push('--input', '-');
}


// Set up environment for gh CLI
// GH_TOKEN is the preferred way to authenticate gh CLI
// For GitHub Enterprise, GH_HOST should be set to the enterprise hostname
const env = { ...process.env };
if (process.env.GITHUB_TOKEN && !process.env.GH_TOKEN) {
env.GH_TOKEN = process.env.GITHUB_TOKEN;
}

// Extract hostname from GITHUB_API_URL for Enterprise support
if (process.env.GITHUB_API_URL && !process.env.GH_HOST) {
try {
const apiUrl = new URL(process.env.GITHUB_API_URL);
if (apiUrl.hostname !== 'api.github.com') {
env.GH_HOST = apiUrl.hostname;
console.log(`Using GitHub Enterprise host: ${apiUrl.hostname}`);
}
} catch (e) {
console.error(`Failed to parse GITHUB_API_URL: ${e.message}`);
}
}

try {
const result = spawnSync('gh', args, {
encoding: 'utf8',
input: data ? JSON.stringify(data) : undefined,
stdio: ['pipe', 'pipe', 'pipe']
stdio: ['pipe', 'pipe', 'pipe'],
env: env
});

if (result.error) {
Expand Down