1515# PDF support (optional)
1616try :
1717 import fitz # PyMuPDF
18+
1819 HAS_PYMUPDF = True
1920except ImportError :
2021 HAS_PYMUPDF = False
2122
2223# Office formats (optional)
2324try :
2425 import docx # python-docx
26+
2527 HAS_DOCX = True
2628except ImportError :
2729 HAS_DOCX = False
2830
2931try :
3032 import openpyxl
33+
3134 HAS_XLSX = True
3235except ImportError :
3336 HAS_XLSX = False
3437
3538try :
3639 from pptx import Presentation
40+
3741 HAS_PPTX = True
3842except ImportError :
3943 HAS_PPTX = False
4751@dataclass
4852class Chunk :
4953 """A chunk of text from a document"""
54+
5055 content : str
5156 index : int
5257 start_char : int
@@ -57,6 +62,7 @@ class Chunk:
5762@dataclass
5863class Document :
5964 """Parsed document with metadata and chunks"""
65+
6066 id : str
6167 content : str
6268 source : Path
@@ -175,7 +181,7 @@ def _parse_markdown(self, filepath: Path) -> tuple[str, Dict]:
175181 }
176182
177183 # Extract headers hierarchy
178- header_pattern = r' ^(#{1,6})\s+(.+)$'
184+ header_pattern = r" ^(#{1,6})\s+(.+)$"
179185 for match in re .finditer (header_pattern , content , re .MULTILINE ):
180186 level = len (match .group (1 ))
181187 title = match .group (2 ).strip ()
@@ -189,20 +195,18 @@ def _parse_markdown(self, filepath: Path) -> tuple[str, Dict]:
189195 metadata ["title" ] = filepath .stem
190196
191197 # Extract frontmatter if present (YAML between ---)
192- frontmatter_match = re .match (r' ^---\n(.*?)\n---\n' , content , re .DOTALL )
198+ frontmatter_match = re .match (r" ^---\n(.*?)\n---\n" , content , re .DOTALL )
193199 if frontmatter_match :
194200 metadata ["has_frontmatter" ] = True
195201 # Remove frontmatter from content for cleaner indexing
196- content = content [frontmatter_match .end ():]
202+ content = content [frontmatter_match .end () :]
197203
198204 return content , metadata
199205
200206 def _parse_pdf (self , filepath : Path ) -> tuple [str , Dict ]:
201207 """Parse PDF file using PyMuPDF (text extraction, no markdown conversion)."""
202208 if not HAS_PYMUPDF :
203- raise ImportError (
204- "PyMuPDF (fitz) not installed. Install with: pip install pymupdf"
205- )
209+ raise ImportError ("PyMuPDF (fitz) not installed. Install with: pip install pymupdf" )
206210
207211 metadata = {
208212 "type" : "pdf" ,
@@ -258,15 +262,15 @@ def _parse_code(self, filepath: Path) -> tuple[str, Dict]:
258262 metadata ["docstring" ] = docstring_match .group (1 ).strip ()
259263
260264 # Extract function names
261- func_pattern = r' ^def\s+(\w+)\s*\('
265+ func_pattern = r" ^def\s+(\w+)\s*\("
262266 metadata ["functions" ] = re .findall (func_pattern , content , re .MULTILINE )
263267
264268 # Extract class names
265- class_pattern = r' ^class\s+(\w+)\s*[:\(]'
269+ class_pattern = r" ^class\s+(\w+)\s*[:\(]"
266270 metadata ["classes" ] = re .findall (class_pattern , content , re .MULTILINE )
267271
268272 # Extract imports
269- import_pattern = r' ^(?:from\s+[\w.]+\s+)?import\s+[\w.,\s]+'
273+ import_pattern = r" ^(?:from\s+[\w.]+\s+)?import\s+[\w.,\s]+"
270274 metadata ["imports" ] = re .findall (import_pattern , content , re .MULTILINE )[:10 ]
271275
272276 return content , metadata
@@ -320,7 +324,7 @@ def _parse_docx(self, filepath: Path) -> tuple[str, Dict]:
320324 text = para .text .strip ()
321325 if text :
322326 # Preserve heading structure as markdown
323- if para .style and para .style .name .startswith (' Heading' ):
327+ if para .style and para .style .name .startswith (" Heading" ):
324328 try :
325329 level = int (para .style .name .split ()[- 1 ])
326330 parts .append (f"{ '#' * level } { text } " )
@@ -468,7 +472,7 @@ def _chunk_text(self, text: str, metadata: Dict) -> List[Chunk]:
468472 metadata = {
469473 "title" : metadata .get ("title" , "" ),
470474 "type" : metadata .get ("type" , "" ),
471- }
475+ },
472476 )
473477 chunks .append (chunk )
474478 index += 1
@@ -508,14 +512,15 @@ def _chunk_markdown(self, text: str, metadata: Dict) -> List[Chunk]:
508512
509513 # Step 1: Mask code blocks to prevent splitting on # inside them
510514 code_blocks = []
515+
511516 def mask_code (match ):
512517 code_blocks .append (match .group (0 ))
513518 return f"__CODE_BLOCK_{ len (code_blocks ) - 1 } __"
514519
515- masked_text = re .sub (r' ```.*?```' , mask_code , text , flags = re .DOTALL )
520+ masked_text = re .sub (r" ```.*?```" , mask_code , text , flags = re .DOTALL )
516521
517522 # Step 2: Split by ## and ### headers only (not # which catches code comments)
518- sections = re .split (r' (?=^#{2,3}\s+)' , masked_text , flags = re .MULTILINE )
523+ sections = re .split (r" (?=^#{2,3}\s+)" , masked_text , flags = re .MULTILINE )
519524
520525 # Filter empty sections
521526 sections = [s for s in sections if s .strip ()]
@@ -565,7 +570,7 @@ def restore_code(section_text):
565570 char_offset += len (section )
566571 continue
567572
568- header_match = re .match (r' ^(#{2,3}\s+.+)$' , section_stripped , re .MULTILINE )
573+ header_match = re .match (r" ^(#{2,3}\s+.+)$" , section_stripped , re .MULTILINE )
569574 header_context = header_match .group (1 ) if header_match else ""
570575
571576 if len (section_stripped ) <= self .chunk_size :
@@ -578,7 +583,7 @@ def restore_code(section_text):
578583 "title" : metadata .get ("title" , "" ),
579584 "type" : metadata .get ("type" , "" ),
580585 "section_header" : header_context ,
581- }
586+ },
582587 )
583588 chunks .append (chunk )
584589 global_index += 1
@@ -611,11 +616,7 @@ def _detect_category(self, filepath: Path) -> str:
611616 path_str = str (filepath ).replace ("\\ " , "/" ).lower ()
612617
613618 # Check category mappings in order (more specific first)
614- for path_pattern , category in sorted (
615- config .category_mappings .items (),
616- key = lambda x : len (x [0 ]),
617- reverse = True
618- ):
619+ for path_pattern , category in sorted (config .category_mappings .items (), key = lambda x : len (x [0 ]), reverse = True ):
619620 if path_pattern in path_str :
620621 return category
621622
@@ -638,24 +639,38 @@ def _extract_keywords(self, content: str, category: str) -> List[str]:
638639
639640 # Extract additional technical terms
640641 # CVE patterns
641- cve_pattern = r' CVE-\d{4}-\d{4,}'
642+ cve_pattern = r" CVE-\d{4}-\d{4,}"
642643 keywords .update (re .findall (cve_pattern , content , re .IGNORECASE ))
643644
644645 # MITRE ATT&CK patterns
645- mitre_pattern = r' T\d{4}(?:\.\d{3})?'
646+ mitre_pattern = r" T\d{4}(?:\.\d{3})?"
646647 keywords .update (re .findall (mitre_pattern , content ))
647648
648649 # IP addresses
649- ip_pattern = r' \b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'
650+ ip_pattern = r" \b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b"
650651 ips = re .findall (ip_pattern , content )
651652 if len (ips ) <= 5 : # Only add if not too many (likely real targets)
652653 keywords .update (ips )
653654
654655 # Common security tools mentioned
655656 security_tools = [
656- "nmap" , "burp" , "metasploit" , "wireshark" , "hydra" , "john" ,
657- "hashcat" , "gobuster" , "nikto" , "sqlmap" , "nuclei" , "ffuf" ,
658- "bloodhound" , "mimikatz" , "responder" , "crackmapexec" , "impacket"
657+ "nmap" ,
658+ "burp" ,
659+ "metasploit" ,
660+ "wireshark" ,
661+ "hydra" ,
662+ "john" ,
663+ "hashcat" ,
664+ "gobuster" ,
665+ "nikto" ,
666+ "sqlmap" ,
667+ "nuclei" ,
668+ "ffuf" ,
669+ "bloodhound" ,
670+ "mimikatz" ,
671+ "responder" ,
672+ "crackmapexec" ,
673+ "impacket" ,
659674 ]
660675 for tool in security_tools :
661676 if tool in content_lower :
0 commit comments