Skip to content

Commit 47244a7

Browse files
committed
fix: ruff format compliance for CI
1 parent 0e01a14 commit 47244a7

8 files changed

Lines changed: 480 additions & 351 deletions

File tree

mcp_server/config.py

Lines changed: 216 additions & 131 deletions
Large diffs are not rendered by default.

mcp_server/ingestion.py

Lines changed: 41 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,29 @@
1515
# PDF support (optional)
1616
try:
1717
import fitz # PyMuPDF
18+
1819
HAS_PYMUPDF = True
1920
except ImportError:
2021
HAS_PYMUPDF = False
2122

2223
# Office formats (optional)
2324
try:
2425
import docx # python-docx
26+
2527
HAS_DOCX = True
2628
except ImportError:
2729
HAS_DOCX = False
2830

2931
try:
3032
import openpyxl
33+
3134
HAS_XLSX = True
3235
except ImportError:
3336
HAS_XLSX = False
3437

3538
try:
3639
from pptx import Presentation
40+
3741
HAS_PPTX = True
3842
except ImportError:
3943
HAS_PPTX = False
@@ -47,6 +51,7 @@
4751
@dataclass
4852
class Chunk:
4953
"""A chunk of text from a document"""
54+
5055
content: str
5156
index: int
5257
start_char: int
@@ -57,6 +62,7 @@ class Chunk:
5762
@dataclass
5863
class Document:
5964
"""Parsed document with metadata and chunks"""
65+
6066
id: str
6167
content: str
6268
source: Path
@@ -175,7 +181,7 @@ def _parse_markdown(self, filepath: Path) -> tuple[str, Dict]:
175181
}
176182

177183
# Extract headers hierarchy
178-
header_pattern = r'^(#{1,6})\s+(.+)$'
184+
header_pattern = r"^(#{1,6})\s+(.+)$"
179185
for match in re.finditer(header_pattern, content, re.MULTILINE):
180186
level = len(match.group(1))
181187
title = match.group(2).strip()
@@ -189,20 +195,18 @@ def _parse_markdown(self, filepath: Path) -> tuple[str, Dict]:
189195
metadata["title"] = filepath.stem
190196

191197
# Extract frontmatter if present (YAML between ---)
192-
frontmatter_match = re.match(r'^---\n(.*?)\n---\n', content, re.DOTALL)
198+
frontmatter_match = re.match(r"^---\n(.*?)\n---\n", content, re.DOTALL)
193199
if frontmatter_match:
194200
metadata["has_frontmatter"] = True
195201
# Remove frontmatter from content for cleaner indexing
196-
content = content[frontmatter_match.end():]
202+
content = content[frontmatter_match.end() :]
197203

198204
return content, metadata
199205

200206
def _parse_pdf(self, filepath: Path) -> tuple[str, Dict]:
201207
"""Parse PDF file using PyMuPDF (text extraction, no markdown conversion)."""
202208
if not HAS_PYMUPDF:
203-
raise ImportError(
204-
"PyMuPDF (fitz) not installed. Install with: pip install pymupdf"
205-
)
209+
raise ImportError("PyMuPDF (fitz) not installed. Install with: pip install pymupdf")
206210

207211
metadata = {
208212
"type": "pdf",
@@ -258,15 +262,15 @@ def _parse_code(self, filepath: Path) -> tuple[str, Dict]:
258262
metadata["docstring"] = docstring_match.group(1).strip()
259263

260264
# Extract function names
261-
func_pattern = r'^def\s+(\w+)\s*\('
265+
func_pattern = r"^def\s+(\w+)\s*\("
262266
metadata["functions"] = re.findall(func_pattern, content, re.MULTILINE)
263267

264268
# Extract class names
265-
class_pattern = r'^class\s+(\w+)\s*[:\(]'
269+
class_pattern = r"^class\s+(\w+)\s*[:\(]"
266270
metadata["classes"] = re.findall(class_pattern, content, re.MULTILINE)
267271

268272
# Extract imports
269-
import_pattern = r'^(?:from\s+[\w.]+\s+)?import\s+[\w.,\s]+'
273+
import_pattern = r"^(?:from\s+[\w.]+\s+)?import\s+[\w.,\s]+"
270274
metadata["imports"] = re.findall(import_pattern, content, re.MULTILINE)[:10]
271275

272276
return content, metadata
@@ -320,7 +324,7 @@ def _parse_docx(self, filepath: Path) -> tuple[str, Dict]:
320324
text = para.text.strip()
321325
if text:
322326
# Preserve heading structure as markdown
323-
if para.style and para.style.name.startswith('Heading'):
327+
if para.style and para.style.name.startswith("Heading"):
324328
try:
325329
level = int(para.style.name.split()[-1])
326330
parts.append(f"{'#' * level} {text}")
@@ -468,7 +472,7 @@ def _chunk_text(self, text: str, metadata: Dict) -> List[Chunk]:
468472
metadata={
469473
"title": metadata.get("title", ""),
470474
"type": metadata.get("type", ""),
471-
}
475+
},
472476
)
473477
chunks.append(chunk)
474478
index += 1
@@ -508,14 +512,15 @@ def _chunk_markdown(self, text: str, metadata: Dict) -> List[Chunk]:
508512

509513
# Step 1: Mask code blocks to prevent splitting on # inside them
510514
code_blocks = []
515+
511516
def mask_code(match):
512517
code_blocks.append(match.group(0))
513518
return f"__CODE_BLOCK_{len(code_blocks) - 1}__"
514519

515-
masked_text = re.sub(r'```.*?```', mask_code, text, flags=re.DOTALL)
520+
masked_text = re.sub(r"```.*?```", mask_code, text, flags=re.DOTALL)
516521

517522
# Step 2: Split by ## and ### headers only (not # which catches code comments)
518-
sections = re.split(r'(?=^#{2,3}\s+)', masked_text, flags=re.MULTILINE)
523+
sections = re.split(r"(?=^#{2,3}\s+)", masked_text, flags=re.MULTILINE)
519524

520525
# Filter empty sections
521526
sections = [s for s in sections if s.strip()]
@@ -565,7 +570,7 @@ def restore_code(section_text):
565570
char_offset += len(section)
566571
continue
567572

568-
header_match = re.match(r'^(#{2,3}\s+.+)$', section_stripped, re.MULTILINE)
573+
header_match = re.match(r"^(#{2,3}\s+.+)$", section_stripped, re.MULTILINE)
569574
header_context = header_match.group(1) if header_match else ""
570575

571576
if len(section_stripped) <= self.chunk_size:
@@ -578,7 +583,7 @@ def restore_code(section_text):
578583
"title": metadata.get("title", ""),
579584
"type": metadata.get("type", ""),
580585
"section_header": header_context,
581-
}
586+
},
582587
)
583588
chunks.append(chunk)
584589
global_index += 1
@@ -611,11 +616,7 @@ def _detect_category(self, filepath: Path) -> str:
611616
path_str = str(filepath).replace("\\", "/").lower()
612617

613618
# Check category mappings in order (more specific first)
614-
for path_pattern, category in sorted(
615-
config.category_mappings.items(),
616-
key=lambda x: len(x[0]),
617-
reverse=True
618-
):
619+
for path_pattern, category in sorted(config.category_mappings.items(), key=lambda x: len(x[0]), reverse=True):
619620
if path_pattern in path_str:
620621
return category
621622

@@ -638,24 +639,38 @@ def _extract_keywords(self, content: str, category: str) -> List[str]:
638639

639640
# Extract additional technical terms
640641
# CVE patterns
641-
cve_pattern = r'CVE-\d{4}-\d{4,}'
642+
cve_pattern = r"CVE-\d{4}-\d{4,}"
642643
keywords.update(re.findall(cve_pattern, content, re.IGNORECASE))
643644

644645
# MITRE ATT&CK patterns
645-
mitre_pattern = r'T\d{4}(?:\.\d{3})?'
646+
mitre_pattern = r"T\d{4}(?:\.\d{3})?"
646647
keywords.update(re.findall(mitre_pattern, content))
647648

648649
# IP addresses
649-
ip_pattern = r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'
650+
ip_pattern = r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"
650651
ips = re.findall(ip_pattern, content)
651652
if len(ips) <= 5: # Only add if not too many (likely real targets)
652653
keywords.update(ips)
653654

654655
# Common security tools mentioned
655656
security_tools = [
656-
"nmap", "burp", "metasploit", "wireshark", "hydra", "john",
657-
"hashcat", "gobuster", "nikto", "sqlmap", "nuclei", "ffuf",
658-
"bloodhound", "mimikatz", "responder", "crackmapexec", "impacket"
657+
"nmap",
658+
"burp",
659+
"metasploit",
660+
"wireshark",
661+
"hydra",
662+
"john",
663+
"hashcat",
664+
"gobuster",
665+
"nikto",
666+
"sqlmap",
667+
"nuclei",
668+
"ffuf",
669+
"bloodhound",
670+
"mimikatz",
671+
"responder",
672+
"crackmapexec",
673+
"impacket",
659674
]
660675
for tool in security_tools:
661676
if tool in content_lower:

0 commit comments

Comments
 (0)