feat: add example key pattern detection (3 new patterns)

raye-deng · raye-deng · commit b1e6c79c8ccc · 2026-03-28T14:10:59.000+08:00
Add high-confidence detection for AI-generated example/placeholder API keys
that traditional linters miss:

- example-api-key: OpenAI keys with 'example' suffix (confidence 0.95)
- example-github-pat: GitHub PATs with 'example' suffix (confidence 0.95)
- placeholder-secret-basic: Obvious placeholder secrets (confidence 0.8)

Also improve:
- example-stripe-key: now covers pk_test_ and pk_live_ prefix formats
- Split github-pat-general (general detection) from example-github-pat
  (example-specific, higher confidence)

8 new tests added, all 101 security tests passing.

Refs: product-audit-5-scenarios.md Quick Wins - Example Key 模式检测
diff --git a/packages/core/src/detectors/v4/security-pattern.ts b/packages/core/src/detectors/v4/security-pattern.ts
@@ -212,6 +212,15 @@ const SECURITY_PATTERNS: SecurityPattern[] = [
   // These patterns are commonly produced by AI from training data
   // and represent AI-specific security risks that traditional tools miss.
 
+  {
+    id: 'example-api-key',
+    pattern: /sk-(?:proj-|svcacct-)[a-zA-Z0-9]+-[a-zA-Z0-9]*example[a-zA-Z0-9]*/i,
+    severity: 'error',
+    confidence: 0.95,
+    message: 'Possible example OpenAI API key detected. AI often copies documentation examples with placeholder values. Use environment variables instead.',
+    languages: [],
+    excludeContextPatterns: [/process\.env/i, /import\.meta\.env/i, /\bgetenv\b/i],
+  },
   {
     id: 'example-openai-key',
     pattern: /sk-(?:proj-|svcacct-)[a-zA-Z0-9]{4,24}(-[a-zA-Z0-9]{4,24}){1,3}/,
@@ -229,6 +238,14 @@ const SECURITY_PATTERNS: SecurityPattern[] = [
     message: 'Placeholder secret value detected. AI often copies example values from documentation. Replace with proper secret management.',
     languages: [],
   },
+  {
+    id: 'placeholder-secret-basic',
+    pattern: /(?:password|secret)\s*[:=]\s*['"][^'"]*(?:example|test|demo|sample)[^'"]*['"]/i,
+    severity: 'warning',
+    confidence: 0.8,
+    message: 'Placeholder secret detected. AI commonly uses obvious placeholder values like "example" or "test" for secrets. Use environment variables instead.',
+    languages: [],
+  },
   {
     id: 'dynamic-cors-origin-reflection',
     pattern: /Access-Control-Allow-Origin.*?(?:req|request|headers\.origin|\$\{.*?origin)/i,
@@ -422,17 +439,25 @@ const SECURITY_PATTERNS: SecurityPattern[] = [
 
   {
     id: 'example-stripe-key',
-    pattern: /\bsk_(?:test|live)_[a-zA-Z0-9]{20,}/,
+    pattern: /\b(?:pk|sk)_(?:test|live)_[a-zA-Z0-9]{20,}(?:example)?/i,
     severity: 'error',
     confidence: 0.9,
     message: 'Possible Stripe API key detected. AI often copies example keys from Stripe documentation. Use environment variables or a secrets manager.',
     languages: [],
   },
   {
     id: 'example-github-pat',
+    pattern: /\bghp_[a-zA-Z0-9]{4,}example\b/i,
+    severity: 'error',
+    confidence: 0.95,
+    message: 'Possible example GitHub Personal Access Token detected. AI often copies documentation examples with "example" suffix. Use environment variables instead.',
+    languages: [],
+  },
+  {
+    id: 'github-pat-general',
     pattern: /\bghp_[a-zA-Z0-9]{36}\b/,
     severity: 'error',
-    confidence: 0.9,
+    confidence: 0.8,
     message: 'Possible GitHub Personal Access Token detected. Verify this is not a real token and use environment variables.',
     languages: [],
   },
diff --git a/packages/core/tests/v4/example-key-demo.ts b/packages/core/tests/v4/example-key-demo.ts
@@ -0,0 +1,85 @@
+import { SecurityPatternDetector } from '../../src/detectors/v4/security-pattern.js';
+import type { DetectorContext } from '../../src/detectors/v4/types.js';
+import type { CodeUnit } from '../../src/ir/types.js';
+import { createCodeUnit } from '../../src/ir/types.js';
+
+// Demo file with example/placeholder keys that AI commonly generates
+const demoSource = `// AI-generated code with example/placeholder keys
+import OpenAI from 'openai';
+
+// Scenario 1: OpenAI example key (AI copied from docs)
+const openaiKey = "sk-proj-abc123-example";
+const openai = new OpenAI({ apiKey: openaiKey });
+
+// Scenario 2: GitHub PAT with example suffix
+const githubToken = "ghp_aabbccddeeff00112233445566778899example";
+
+// Scenario 3: Stripe secret key (from Stripe docs)
+const stripeSecretKey = "sk_test_abcdefghijklmnopqrstuvwxyzexample";
+
+// Scenario 4: Placeholder secret value
+const config = {
+  password: "example123",
+  apiSecret: "changeme",
+  authToken: "your_api_key_here",
+};
+
+// Scenario 5: Environment variable with hardcoded fallback
+const dbPassword = process.env.DB_PASSWORD || "some-secret-value-here";
+
+// This should NOT be flagged (proper env var usage)
+const productionKey = process.env.PRODUCTION_API_KEY;
+
+async function main() {
+  const completion = await openai.chat.completions.create({
+    model: "gpt-4",
+    messages: [{ role: "user", content: "Hello" }],
+  });
+  console.log(completion.choices[0].message);
+}
+
+main();`;
+
+function makeUnit(source: string): CodeUnit {
+  return createCodeUnit({
+    id: 'func:demo.ts:main',
+    file: 'demo.ts',
+    language: 'typescript',
+    kind: 'function',
+    location: { startLine: 0, startColumn: 0, endLine: source.split('\n').length, endColumn: 0 },
+    source,
+  });
+}
+
+const detector = new SecurityPatternDetector();
+const context: DetectorContext = { projectRoot: '/project', allFiles: ['demo.ts'] };
+
+async function run() {
+  const unit = makeUnit(demoSource);
+  const results = await detector.detect([unit], context);
+  
+  console.log('=== Example Key Detection Demo Report ===\n');
+  console.log(`File: demo.ts`);
+  console.log(`Total detections: ${results.length}\n`);
+  
+  for (const result of results) {
+    console.log(`[${result.severity.toUpperCase()}] ${result.message}`);
+    console.log(`  Line: ${result.line}`);
+    console.log(`  Pattern: ${result.metadata.patternId}`);
+    console.log(`  Confidence: ${result.confidence}`);
+    console.log(`  Matched: ${result.metadata.matchedLine}`);
+    console.log();
+  }
+  
+  // Summary
+  const newPatterns = ['example-api-key', 'example-github-pat', 'placeholder-secret-basic'];
+  const newDetections = results.filter(r => newPatterns.includes(r.metadata.patternId));
+  
+  console.log('=== New Pattern Detections (this update) ===');
+  console.log(`Count: ${newDetections.length}`);
+  for (const result of newDetections) {
+    console.log(`  - ${result.metadata.patternId}: Line ${result.line}`);
+  }
+}
+
+run().catch(console.error);
diff --git a/packages/core/tests/v4/example-key-pattern-detector.test.ts b/packages/core/tests/v4/example-key-pattern-detector.test.ts
@@ -0,0 +1,161 @@
+/**
+ * Example Key Pattern Detection Tests
+ *
+ * Tests for detecting example/placeholder keys that AI often
+ * copies from documentation but accidentally leaves in code.
+ *
+ * @since 0.5.0
+ */
+
+import { describe, it, expect } from 'vitest';
+import { SecurityPatternDetector } from '../../src/detectors/v4/security-pattern.js';
+import type { DetectorContext } from '../../src/detectors/v4/types.js';
+import type { CodeUnit } from '../../src/ir/types.js';
+import { createCodeUnit } from '../../src/ir/types.js';
+
+// ─── Helpers ───────────────────────────────────────────────────────
+
+function makeUnit(
+  source: string,
+  language: CodeUnit['language'] = 'typescript',
+  file: string = 'test.ts',
+): CodeUnit {
+  return createCodeUnit({
+    id: `func:${file}:fn`,
+    file,
+    language,
+    kind: 'function',
+    location: { startLine: 0, startColumn: 0, endLine: source.split('\n').length, endColumn: 0 },
+    source,
+  });
+}
+
+function createContext(): DetectorContext {
+  return {
+    projectRoot: '/project',
+    allFiles: ['test.ts'],
+  };
+}
+
+// ─── Tests ─────────────────────────────────────────────────────────
+
+describe('Example Key Pattern Detection', () => {
+  const detector = new SecurityPatternDetector();
+
+  it('should detect OpenAI example keys with example suffix', async () => {
+    const unit = makeUnit('const openaiKey = "sk-proj-abc123-example";');
+    const results = await detector.detect([unit], createContext());
+    
+    // Should be detected as example API key (both example-api-key and example-openai-key may match)
+    const hasExampleApiKey = results.some(r => r.metadata.patternId === 'example-api-key');
+    expect(hasExampleApiKey).toBe(true);
+  });
+
+  it('should detect GitHub PATs with example suffix', async () => {
+    const unit = makeUnit('const githubToken = "ghp_1234567890abcdef1234567890abcdefexample";');
+    const results = await detector.detect([unit], createContext());
+    
+    // Should be detected as example GitHub PAT (placeholder-secret-value may also trigger, which is fine)
+    const hasExampleGithubPat = results.some(r => r.metadata.patternId === 'example-github-pat');
+    expect(hasExampleGithubPat).toBe(true);
+  });
+
+  it('should detect placeholder secrets', async () => {
+    const testCases = [
+      'const secret = "password=example123";',
+      'const apikey = "demo-key-12345";',
+      'const token = "sample_token_abc123";',
+      'const password = "changeme123";',
+    ];
+
+    for (const code of testCases) {
+      const unit = makeUnit(code);
+      const results = await detector.detect([unit], createContext());
+      
+      // These should be detected as placeholder secrets
+      const hasPlaceholderSecret = results.some(r => 
+        r.metadata.patternId === 'placeholder-secret-value'
+      );
+      
+      if (!hasPlaceholderSecret) {
+        console.log(`Code not detected: ${code}`);
+        console.log('Results:', results);
+      }
+      
+      expect(hasPlaceholderSecret).toBe(true);
+    }
+  });
+
+  it('should detect AWS example keys', async () => {
+    const unit = makeUnit('const awsKey = "AKIAIOSFODNN7EXAMPLE";');
+    const results = await detector.detect([unit], createContext());
+    
+    expect(results).toHaveLength(1);
+    expect(results[0].message).toContain('AWS access key ID');
+    expect(results[0].metadata.patternId).toBe('aws-access-key');
+  });
+
+  it('should detect Stripe example keys', async () => {
+    const unit = makeUnit('const stripeKey = "sk_test_abcdefghijklmnopqrstuvwxyzexample";');
+    const results = await detector.detect([unit], createContext());
+    
+    const hasStripeDetection = results.some(r => 
+      r.metadata.patternId === 'example-stripe-key'
+    );
+    expect(hasStripeDetection).toBe(true);
+  });
+
+  it('should NOT detect real-looking keys', async () => {
+    const testCases = [
+      { 
+        code: 'const token = "ghp_1234567890abcdef1234567890abcdef";', 
+        description: 'GitHub PAT without example suffix',
+        // Note: example-openai-key is a general security check for ANY OpenAI key format,
+        // so we only test non-OpenAI patterns here
+        shouldMatchPatterns: ['github-pat-general'],
+      },
+    ];
+
+    for (const testCase of testCases) {
+      const unit = makeUnit(testCase.code);
+      const results = await detector.detect([unit], createContext());
+      
+      // These should NOT trigger example-specific patterns (those with "example" in the ID)
+      const hasExampleSpecificDetection = results.some(r => 
+        r.metadata.patternId === 'example-github-pat' ||
+        r.metadata.patternId === 'example-api-key' ||
+        r.metadata.patternId === 'placeholder-secret-basic'
+      );
+      
+      expect(hasExampleSpecificDetection).toBe(false);
+    }
+  });
+
+  it('should detect environment variable fallback secrets', async () => {
+    const unit = makeUnit(`
+      const apiKey = process.env.MY_API_KEY || "some-secret-value-here";
+    `);
+    const results = await detector.detect([unit], createContext());
+    
+    const hasFallbackDetection = results.some(r => 
+      r.metadata.patternId === 'env-var-fallback-secret-js'
+    );
+    
+    expect(hasFallbackDetection).toBe(true);
+  });
+
+  it('should ignore environment variables that are properly used', async () => {
+    const unit = makeUnit(`
+      const apiKey = process.env.MY_API_KEY;
+      if (!apiKey) throw new Error('API key required');
+    `);
+    const results = await detector.detect([unit], createContext());
+    
+    // Should not detect as hardcoded secret when using process.env properly
+    const hasHardcodedDetection = results.some(r => 
+      r.metadata.patternId === 'hardcoded-api-key'
+    );
+    
+    expect(hasHardcodedDetection).toBe(false);
+  });
+});
diff --git a/packages/core/tests/v4/security-pattern-detector.test.ts b/packages/core/tests/v4/security-pattern-detector.test.ts
@@ -608,7 +608,7 @@ function handleRequest(req: Request) {
     // Test detection of ghp_ prefix + 36 alphanum chars (fake value)
     const unit = makeUnit('const token = "ghp_000000000000000000000000000000000000";');
     const results = await detector.detect([unit], createContext());
-    const ghResult = results.find(r => r.metadata?.patternId === 'example-github-pat');
+    const ghResult = results.find(r => r.metadata?.patternId === 'github-pat-general');
     expect(ghResult).toBeDefined();
   });
 
