salim0986/graph-bug-ai
0
1"""2Universal Parser — M1 upgrade3Extracts declarations AND structural relationships (calls, imports, inheritance, type-use)4from source files using tree-sitter AST queries.5 6Backward-compatible: parse_file() / parse_code() still return (captures, code_bytes)7New: extract_relationships() returns a ParseReport with typed records.8"""9 10import os11from dataclasses import dataclass, field12from typing import List, Optional, Tuple, Any13from tree_sitter_languages import get_language, get_parser14import tree_sitter15from .logger import setup_logger16 17logger = setup_logger(__name__)18 19# ---------------------------------------------------------------------------20# Language → file-extension map (unchanged from original)21# ---------------------------------------------------------------------------22 23EXTENSION_MAP = {24 # Web / JS25 ".js": "javascript",26 ".jsx": "javascript",27 ".ts": "typescript",28 ".tsx": "tsx",29 ".vue": "vue",30 ".html": "html",31 ".css": "css",32 ".json": "json",33 # Backend / Core34 ".py": "python",35 ".go": "go",36 ".rs": "rust",37 ".java": "java",38 ".rb": "ruby",39 ".php": "php",40 ".cs": "c_sharp",41 ".cpp": "cpp",42 ".c": "c",43 ".h": "c",44 ".hpp": "cpp",45 # Systems / Scripting46 ".sh": "bash",47 ".lua": "lua",48 ".yaml": "yaml",49 ".yml": "yaml",50 ".toml": "toml",51 # Config / Infra52 ".tf": "hcl",53 ".dockerfile": "dockerfile",54 ".make": "make",55 # Docs56 ".md": "markdown",57}58 59 60def detect_language_from_filename(filename: str) -> Optional[str]:61 """Return language name from file extension, or None if unsupported."""62 _, ext = os.path.splitext(filename)63 return EXTENSION_MAP.get(ext.lower())64 65 66# ---------------------------------------------------------------------------67# Typed relationship records68# ---------------------------------------------------------------------------69 70@dataclass71class Declaration:72 """A named declaration (function, class, method, interface, struct …)."""73 name: str74 type: str # "function" | "class" | "interface" | "method" | "struct" | "other"75 file: str76 start_line: int77 end_line: int78 code: str = "" # first 2000 chars of the declaration body79 80 81@dataclass82class Call:83 """A function/method call site."""84 caller: str # enclosing function name, or "module_level"85 callee: str # name of the function/method being called86 file: str87 line: int88 89 90@dataclass91class Import:92 """An import / include / use statement."""93 file: str94 module: str # module or path being imported95 alias: Optional[str] # specific name imported (e.g. `from os import path`)96 line: int97 98 99@dataclass100class Inheritance:101 """A class-extends or class-implements relationship."""102 child: str # the subclass / implementing class103 parent: str # the superclass or interface104 file: str105 line: int106 107 108@dataclass109class TypeUse:110 """A type annotation reference inside a function or class."""111 context: str # enclosing function / class name112 type_name: str113 file: str114 line: int115 116 117@dataclass118class ParseReport:119 """Full extraction result for one file."""120 file: str121 language: Optional[str]122 status: str = "failed" # "success" | "partial" | "failed"123 declarations: List[Declaration] = field(default_factory=list)124 calls: List[Call] = field(default_factory=list)125 imports: List[Import] = field(default_factory=list)126 inheritances: List[Inheritance] = field(default_factory=list)127 type_uses: List[TypeUse] = field(default_factory=list)128 errors: List[str] = field(default_factory=list)129 130 @property131 def nodes_extracted(self) -> int:132 return len(self.declarations)133 134 @property135 def relationships_extracted(self) -> int:136 return len(self.calls) + len(self.imports) + len(self.inheritances)137 138 139# ---------------------------------------------------------------------------140# Node-type → declaration-type mapping141# ---------------------------------------------------------------------------142 143_DECL_TYPE_MAP = {144 "function_definition": "function",145 "function_declaration": "function",146 "function_item": "function",147 "method_definition": "method",148 "method_declaration": "method",149 "class_definition": "class",150 "class_declaration": "class",151 "class_specifier": "class",152 "struct_item": "struct",153 "struct_specifier": "struct",154 "interface_declaration": "interface",155 "trait_item": "interface",156 "impl_item": "class",157 "enum_declaration": "class",158 "enum_item": "class",159 "abstract_class_declaration": "class",160 "namespace_declaration": "other",161 "type_alias_declaration": "other",162}163 164# Node types that represent a function/method scope (for enclosing-name lookup)165_ENCLOSING_FUNC_TYPES = frozenset({166 "function_definition",167 "function_declaration",168 "function_item",169 "method_definition",170 "method_declaration",171 "arrow_function",172 "function_expression",173 "closure_expression",174 "lambda",175})176 177# Class-scope node types across supported languages.178# Used to detect when a function_definition is actually a method.179_CLASS_SCOPE_TYPES = frozenset({180 "class_definition", # Python181 "class_declaration", # JS / TS / Java / C#182 "class_specifier", # C++183 "impl_item", # Rust184 "class_body", # Java (body of a class)185 "declaration_list", # C# class body186})187 188 189def _is_inside_class(func_node: Any) -> bool:190 """191 Return True when *func_node* is a method — i.e. its nearest enclosing192 structural scope is a class, not another function.193 194 Stops walking upward the moment it hits either:195 - a class-scope node → True (it's a method)196 - another function node → False (it's a nested function, not a method)197 """198 current = func_node.parent199 while current is not None:200 t = current.type201 if t in _CLASS_SCOPE_TYPES:202 return True203 if t in _ENCLOSING_FUNC_TYPES:204 return False # Crossed a function boundary before any class205 current = current.parent206 return False207 208 209# ---------------------------------------------------------------------------210# Main parser class211# ---------------------------------------------------------------------------212 213class UniversalParser:214 def __init__(self):215 current_dir = os.path.dirname(os.path.abspath(__file__))216 self.base_query_path = os.path.join(current_dir, "queries")217 218 # ------------------------------------------------------------------219 # BACKWARD-COMPATIBLE API220 # ------------------------------------------------------------------221 222 def parse_file(self, file_path: str) -> Tuple[Optional[Any], Optional[bytes]]:223 """224 Original interface: (captures, code_bytes) or (None, None).225 Still used by graph_builder.process_file_nodes().226 """227 _, ext = os.path.splitext(file_path)228 lang_name = EXTENSION_MAP.get(ext)229 if not lang_name:230 return None, None231 try:232 language = get_language(lang_name)233 parser = get_parser(lang_name)234 with open(file_path, "rb") as f:235 code_bytes = f.read()236 tree = parser.parse(code_bytes)237 query_file = os.path.join(self.base_query_path, lang_name, "tags.scm")238 if not os.path.exists(query_file):239 return None, None240 with open(query_file) as f:241 query_scm = f.read()242 query = language.query(query_scm)243 captures = query.captures(tree.root_node)244 return captures, code_bytes245 except FileNotFoundError:246 logger.warning(f"File not found: {file_path}")247 return None, None248 except Exception as e:249 logger.warning(f"Parser error in {file_path}: {e}")250 return None, None251 252 def parse_file_ast(self, file_path: str) -> Tuple[Optional[Any], Optional[bytes], Optional[str]]:253 """254 Returns (tree, code_bytes, language_name).255 Used by the new SemanticChunker in M2.256 """257 _, ext = os.path.splitext(file_path)258 lang_name = EXTENSION_MAP.get(ext)259 if not lang_name:260 return None, None, None261 try:262 parser = get_parser(lang_name)263 with open(file_path, "rb") as f:264 code_bytes = f.read()265 tree = parser.parse(code_bytes)266 return tree, code_bytes, lang_name267 except FileNotFoundError:268 logger.warning(f"File not found: {file_path}")269 return None, None, None270 except Exception as e:271 logger.warning(f"Parser error in {file_path}: {e}")272 return None, None, None273 274 def parse_code(self, code_bytes: bytes, lang_name: str) -> Tuple[Optional[Any], Optional[bytes]]:275 """276 Original interface: (captures, code_bytes) or (None, None).277 """278 if not lang_name or lang_name == "text":279 return None, None280 try:281 language = get_language(lang_name)282 parser = get_parser(lang_name)283 tree = parser.parse(code_bytes)284 query_file = os.path.join(self.base_query_path, lang_name, "tags.scm")285 if not os.path.exists(query_file):286 return None, None287 with open(query_file) as f:288 query_scm = f.read()289 query = language.query(query_scm)290 captures = query.captures(tree.root_node)291 return captures, code_bytes292 except Exception as e:293 logger.warning(f"Parser error for {lang_name}: {e}")294 return None, None295 296 # ------------------------------------------------------------------297 # NEW M1 API — relationship extraction298 # ------------------------------------------------------------------299 300 def extract_relationships(301 self,302 code_bytes: bytes,303 lang_name: str,304 file_path: str = "",305 ) -> ParseReport:306 """307 Extract declarations + structural relationships from code bytes.308 309 Returns a ParseReport with:310 - declarations (functions, classes, interfaces …)311 - calls (caller → callee)312 - imports (module dependencies)313 - inheritances (extends / implements)314 - type_uses (type annotations)315 - status success | partial | failed316 - errors human-readable error list317 """318 report = ParseReport(file=file_path, language=lang_name)319 320 if not lang_name or lang_name == "text":321 report.errors.append(f"Unsupported language: {lang_name!r}")322 return report323 324 try:325 language = get_language(lang_name)326 ts_parser = get_parser(lang_name)327 tree = ts_parser.parse(code_bytes)328 except Exception as e:329 report.errors.append(f"Tree-sitter init error: {e}")330 return report331 332 # --- Step 1: declarations from tags.scm ---333 tags_ok = self._extract_declarations(tree, language, code_bytes, file_path, report)334 335 # --- Step 2: relationships from relationships.scm ---336 rels_ok = self._extract_rels(tree, language, code_bytes, file_path, report)337 338 # --- Step 3: detect AST syntax errors ---339 # tree-sitter is error-tolerant; it inserts ERROR/MISSING nodes for invalid340 # syntax rather than refusing to parse. These nodes produce garbage captures.341 has_syntax_errors = tree.root_node.has_error342 if has_syntax_errors:343 report.errors.append("Source contains syntax errors — AST ERROR nodes present; results may be incomplete")344 345 if tags_ok and rels_ok and not has_syntax_errors:346 report.status = "success"347 elif tags_ok or rels_ok:348 report.status = "partial"349 else:350 report.status = "failed"351 352 logger.debug(353 f"[Parser] {file_path} ({lang_name}): "354 f"status={report.status} decls={report.nodes_extracted} "355 f"calls={len(report.calls)} imports={len(report.imports)} "356 f"inherit={len(report.inheritances)}"357 )358 return report359 360 def extract_relationships_from_file(self, file_path: str) -> ParseReport:361 """Convenience wrapper: read file then call extract_relationships()."""362 _, ext = os.path.splitext(file_path)363 lang_name = EXTENSION_MAP.get(ext.lower())364 report = ParseReport(file=file_path, language=lang_name)365 366 if not lang_name:367 report.errors.append(f"Unsupported extension: {ext!r}")368 return report369 try:370 with open(file_path, "rb") as f:371 code_bytes = f.read()372 return self.extract_relationships(code_bytes, lang_name, file_path)373 except FileNotFoundError:374 report.errors.append(f"File not found: {file_path}")375 return report376 except Exception as e:377 report.errors.append(f"Read error: {e}")378 return report379 380 # ------------------------------------------------------------------381 # Internal helpers382 # ------------------------------------------------------------------383 384 def _extract_declarations(385 self,386 tree: Any,387 language: Any,388 code_bytes: bytes,389 file_path: str,390 report: ParseReport,391 ) -> bool:392 """Populate report.declarations from tags.scm."""393 tags_file = os.path.join(self.base_query_path, language.__class__.__name__.lower(), "tags.scm")394 # language object doesn't expose name directly; use a workaround395 lang_name = report.language or ""396 tags_file = os.path.join(self.base_query_path, lang_name, "tags.scm")397 398 if not os.path.exists(tags_file):399 report.errors.append(f"No tags.scm for {lang_name}")400 return False401 try:402 with open(tags_file) as f:403 tags_scm = f.read()404 query = language.query(tags_scm)405 captures = query.captures(tree.root_node)406 except Exception as e:407 report.errors.append(f"tags.scm compile/run error: {e}")408 return False409 410 seen: set = set()411 for node, cap_name in captures:412 if cap_name != "name":413 continue414 parent = node.parent415 if parent is None:416 continue417 parent_type = parent.type418 # Avoid duplicate decls (same start byte)419 key = (parent.start_byte, parent_type)420 if key in seen:421 continue422 seen.add(key)423 424 decl_type = _DECL_TYPE_MAP.get(parent_type, "other")425 # Promote function_definition → "method" when the nearest enclosing426 # structural scope is a class. Python methods are syntactically427 # function_definitions; only context distinguishes them.428 if decl_type == "function" and _is_inside_class(parent):429 decl_type = "method"430 name_text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore")431 body = code_bytes[parent.start_byte:parent.end_byte].decode("utf-8", errors="ignore")[:2000]432 433 report.declarations.append(Declaration(434 name=name_text,435 type=decl_type,436 file=file_path,437 start_line=parent.start_point[0],438 end_line=parent.end_point[0],439 code=body,440 ))441 return True442 443 def _extract_rels(444 self,445 tree: Any,446 language: Any,447 code_bytes: bytes,448 file_path: str,449 report: ParseReport,450 ) -> bool:451 """Populate calls/imports/inheritances/type_uses from relationships.scm."""452 lang_name = report.language or ""453 rels_file = os.path.join(self.base_query_path, lang_name, "relationships.scm")454 455 if not os.path.exists(rels_file):456 report.errors.append(f"No relationships.scm for {lang_name}")457 return False458 try:459 with open(rels_file) as f:460 rels_scm = f.read()461 query = language.query(rels_scm)462 captures = query.captures(tree.root_node)463 except Exception as e:464 report.errors.append(f"relationships.scm error: {e}")465 return False466 467 # Process captures in document order (tree-sitter guarantees this)468 # We track "current inherit child" while walking the list so that469 # subsequent inherit.parent captures can reference the right child.470 current_inherit_child: Optional[Tuple[str, int]] = None # (name, line)471 current_implement_child: Optional[Tuple[str, int]] = None472 current_implement_interface: Optional[Tuple[str, int]] = None473 pending_import_module: Optional[Tuple[str, int]] = None # (module_text, line)474 # Dedup set: prevent the same call expression from being recorded twice475 # when multiple SCM patterns match the same AST node.476 seen_calls: set = set()477 478 for node, cap_name in captures:479 text = code_bytes[node.start_byte:node.end_byte].decode("utf-8", errors="ignore").strip()480 line = node.start_point[0]481 482 if cap_name == "call.callee":483 caller = self._find_enclosing_name(node, code_bytes)484 if text:485 call_key = (caller, text, line)486 if call_key not in seen_calls:487 seen_calls.add(call_key)488 report.calls.append(Call(caller=caller, callee=text, file=file_path, line=line))489 490 elif cap_name == "import.module":491 # Strip surrounding quotes (string literals in some languages)492 module = text.strip("\"'`")493 pending_import_module = (module, line)494 report.imports.append(Import(file=file_path, module=module, alias=None, line=line))495 496 elif cap_name == "import.name":497 name_clean = text.strip("\"'`")498 if pending_import_module and report.imports:499 # Attach the specific name to the most recent import500 last = report.imports[-1]501 if last.alias is None:502 report.imports[-1] = Import(503 file=last.file,504 module=last.module,505 alias=name_clean,506 line=last.line,507 )508 else:509 report.imports.append(Import(file=file_path, module=name_clean, alias=None, line=line))510 511 elif cap_name == "inherit.child":512 current_inherit_child = (text, line)513 514 elif cap_name == "inherit.parent":515 if current_inherit_child and text:516 child_name, child_line = current_inherit_child517 report.inheritances.append(Inheritance(518 child=child_name,519 parent=text,520 file=file_path,521 line=child_line,522 ))523 524 elif cap_name == "implement.child":525 current_implement_child = (text, line)526 if current_implement_interface:527 interface_name, _ = current_implement_interface528 report.inheritances.append(Inheritance(529 child=text,530 parent=interface_name,531 file=file_path,532 line=line,533 ))534 current_implement_child = None535 current_implement_interface = None536 537 elif cap_name == "implement.interface":538 current_implement_interface = (text, line)539 if current_implement_child and text:540 child_name, child_line = current_implement_child541 report.inheritances.append(Inheritance(542 child=child_name,543 parent=text,544 file=file_path,545 line=child_line,546 ))547 current_implement_child = None548 current_implement_interface = None549 550 elif cap_name == "type.use":551 context = self._find_enclosing_name(node, code_bytes)552 if text and text not in ("None", "void", "bool", "int", "str", "float"):553 report.type_uses.append(TypeUse(554 context=context,555 type_name=text,556 file=file_path,557 line=line,558 ))559 560 return True561 562 def _find_enclosing_name(self, node: Any, code_bytes: bytes) -> str:563 """564 Walk up the AST to find the nearest enclosing function/method name.565 Returns "module_level" if no enclosing function is found.566 """567 current = node.parent568 while current is not None:569 if current.type in _ENCLOSING_FUNC_TYPES:570 name_node = current.child_by_field_name("name")571 if name_node:572 return code_bytes[name_node.start_byte:name_node.end_byte].decode("utf-8", errors="ignore")573 current = current.parent574 return "module_level"575