diff --git a/graphify/extractors/base.py b/graphify/extractors/base.py index 17d4ce600c..d69d328b1b 100644 --- a/graphify/extractors/base.py +++ b/graphify/extractors/base.py @@ -110,6 +110,22 @@ def _file_stem(path: Path) -> str: return path.with_suffix("").as_posix() +def file_id_stem(path: "str | Path", root: "str | Path | None" = None) -> str: + """Canonical node-id prefix of ``path``: ``make_id(_file_stem(...))`` of its + root-relative form (#4432). Both sides are resolved first, so an absolute path + from detect and a relative root such as ``.`` still line up. "" when no + root-relative form exists, so a checkout location is never baked into an id.""" + p = Path(path) + if root is not None: + try: + p = p.resolve().relative_to(Path(root).resolve()) + except (OSError, RuntimeError, ValueError): + pass + if p.is_absolute(): + return "" + return make_id(_file_stem(p)) + + def _read_text(node, source: bytes) -> str: return source[node.start_byte:node.end_byte].decode("utf-8", errors="replace") diff --git a/graphify/llm.py b/graphify/llm.py index 9cc83267b6..c05eee05fd 100644 --- a/graphify/llm.py +++ b/graphify/llm.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, replace from pathlib import Path -from graphify.extractors.base import _read_source_text +from graphify.extractors.base import _read_source_text, file_id_stem from graphify.file_slice import ( FileSlice, bisect_slice, @@ -527,8 +527,8 @@ def _thinking_disabled_via_env() -> bool: attributed to the enclosing file and link to it with a `references` or `cites` edge. Never invent, shorten or rewrite a path, and never omit source_file. -Node ID format: lowercase, only [a-z0-9_], no dots or slashes. -Format: {stem}_{entity} where stem = full repo-relative path with the extension dropped, every segment joined with _ (e.g. src/auth/session.py -> src_auth_session); entity = symbol name (both normalised). Top-level files use just the filename stem (setup.py -> setup). +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one _. +Format: {stem}_{entity} where stem = full repo-relative path with the extension dropped, every segment joined with _ (e.g. src/auth/session.py -> src_auth_session; docs/отчёт.md -> docs_отчёт); entity = symbol name (both normalised). Top-level files use just the filename stem (setup.py -> setup). Each block carries a stem="..." attribute: use it character for character as the id prefix of every node from that file; never respell, shorten or transliterate it. Edge direction rule — source is always the ACTOR, target is the ACTED-UPON: - calls: source = the function/method that CONTAINS the call site; target = the function/method BEING CALLED. Never reverse this. @@ -613,17 +613,20 @@ def _neutralise_injection_sentinels(text: str) -> str: return _INJECTION_SENTINELS.sub(lambda m: m.group(0)[0] + "​" + m.group(0)[1:], text) -def _wrap_untrusted(rel: str, content: str) -> str: +def _wrap_untrusted(rel: str, content: str, stem: str = "") -> str: """Wrap one file's content in a labelled, hash-stamped untrusted-data block. The model's system prompt instructs it to treat everything inside as inert data, never as instructions. The sha256 lets a reviewer correlate a suspicious node back to the exact bytes that produced it. + ``stem`` is the file's canonical node-id prefix, computed in code, so the model + copies it instead of inventing its own (#4432). """ sha = hashlib.sha256(content.encode("utf-8", errors="replace")).hexdigest() safe = _neutralise_injection_sentinels(content) + stem_attr = f' stem="{stem}"' if stem else "" return ( - f'\n' + f'\n' f"{safe}\n" f"" ) @@ -672,7 +675,7 @@ def _read_files(units: "list[Path | FileSlice]", root: Path) -> str: f"[graphify] WARNING: file {rel} exceeds {_FILE_CHAR_CAP} characters and was truncated", file=sys.stderr, ) - parts.append(_wrap_untrusted(rel, content[:_FILE_CHAR_CAP])) + parts.append(_wrap_untrusted(rel, content[:_FILE_CHAR_CAP], file_id_stem(rel))) return "\n\n".join(parts) diff --git a/graphify/skill-agents.md b/graphify/skill-agents.md index f4fee6921d..f9b6940520 100644 --- a/graphify/skill-agents.md +++ b/graphify/skill-agents.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-amp.md b/graphify/skill-amp.md index f4fee6921d..f9b6940520 100644 --- a/graphify/skill-amp.md +++ b/graphify/skill-amp.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-claw.md b/graphify/skill-claw.md index d09ad219b2..b168c1d221 100644 --- a/graphify/skill-claw.md +++ b/graphify/skill-claw.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-codex.md b/graphify/skill-codex.md index 7a3c0f3989..7246a17410 100644 --- a/graphify/skill-codex.md +++ b/graphify/skill-codex.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -415,7 +420,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-copilot.md b/graphify/skill-copilot.md index d09ad219b2..b168c1d221 100644 --- a/graphify/skill-copilot.md +++ b/graphify/skill-copilot.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-droid.md b/graphify/skill-droid.md index 819a47f88e..04cc93d0ec 100644 --- a/graphify/skill-droid.md +++ b/graphify/skill-droid.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-kilo.md b/graphify/skill-kilo.md index 6fe81c4a6d..72dcd21516 100644 --- a/graphify/skill-kilo.md +++ b/graphify/skill-kilo.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-kiro.md b/graphify/skill-kiro.md index d09ad219b2..b168c1d221 100644 --- a/graphify/skill-kiro.md +++ b/graphify/skill-kiro.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-opencode.md b/graphify/skill-opencode.md index 332c01c810..6f47ad3cd0 100644 --- a/graphify/skill-opencode.md +++ b/graphify/skill-opencode.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -406,7 +411,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-pi.md b/graphify/skill-pi.md index 46b7819feb..ab5a1fc957 100644 --- a/graphify/skill-pi.md +++ b/graphify/skill-pi.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -408,7 +413,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-trae.md b/graphify/skill-trae.md index 69fcf1538b..f690dd0caf 100644 --- a/graphify/skill-trae.md +++ b/graphify/skill-trae.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -412,7 +417,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-vscode.md b/graphify/skill-vscode.md index 89ef00e794..ea02213b1c 100644 --- a/graphify/skill-vscode.md +++ b/graphify/skill-vscode.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -410,7 +415,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill-windows.md b/graphify/skill-windows.md index 0219a32435..95120739e7 100644 --- a/graphify/skill-windows.md +++ b/graphify/skill-windows.md @@ -270,7 +270,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -279,6 +280,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding="utf-8") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -291,7 +296,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -442,7 +447,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes",[]))} new)') '@ | & (Get-Content graphify-out\.graphify_python) - ``` -Clean up temp files: `Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_cached.json, graphify-out\.graphify_uncached.txt, graphify-out\.graphify_semantic_new.json` +Clean up temp files: `Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_cached.json, graphify-out\.graphify_uncached.txt, graphify-out\.graphify_uncached_stems.txt, graphify-out\.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skill.md b/graphify/skill.md index d09ad219b2..b168c1d221 100644 --- a/graphify/skill.md +++ b/graphify/skill.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/graphify/skills/agents/references/extraction-spec.md b/graphify/skills/agents/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/agents/references/extraction-spec.md +++ b/graphify/skills/agents/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/amp/references/extraction-spec.md b/graphify/skills/amp/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/amp/references/extraction-spec.md +++ b/graphify/skills/amp/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/claude/references/extraction-spec.md b/graphify/skills/claude/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/claude/references/extraction-spec.md +++ b/graphify/skills/claude/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/claw/references/extraction-spec.md b/graphify/skills/claw/references/extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/graphify/skills/claw/references/extraction-spec.md +++ b/graphify/skills/claw/references/extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/codex/references/extraction-spec.md b/graphify/skills/codex/references/extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/graphify/skills/codex/references/extraction-spec.md +++ b/graphify/skills/codex/references/extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/copilot/references/extraction-spec.md b/graphify/skills/copilot/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/copilot/references/extraction-spec.md +++ b/graphify/skills/copilot/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/droid/references/extraction-spec.md b/graphify/skills/droid/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/droid/references/extraction-spec.md +++ b/graphify/skills/droid/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/kilo/references/extraction-spec.md b/graphify/skills/kilo/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/kilo/references/extraction-spec.md +++ b/graphify/skills/kilo/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/kiro/references/extraction-spec.md b/graphify/skills/kiro/references/extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/graphify/skills/kiro/references/extraction-spec.md +++ b/graphify/skills/kiro/references/extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/opencode/references/extraction-spec.md b/graphify/skills/opencode/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/opencode/references/extraction-spec.md +++ b/graphify/skills/opencode/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/pi/references/extraction-spec.md b/graphify/skills/pi/references/extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/graphify/skills/pi/references/extraction-spec.md +++ b/graphify/skills/pi/references/extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/graphify/skills/trae/references/extraction-spec.md b/graphify/skills/trae/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/trae/references/extraction-spec.md +++ b/graphify/skills/trae/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/vscode/references/extraction-spec.md b/graphify/skills/vscode/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/vscode/references/extraction-spec.md +++ b/graphify/skills/vscode/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/graphify/skills/windows/references/extraction-spec.md b/graphify/skills/windows/references/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/graphify/skills/windows/references/extraction-spec.md +++ b/graphify/skills/windows/references/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tests/test_unicode_id_rule.py b/tests/test_unicode_id_rule.py new file mode 100644 index 0000000000..48f87eccfb --- /dev/null +++ b/tests/test_unicode_id_rule.py @@ -0,0 +1,228 @@ +"""#4432: the node-id rule in the prompts must match ``graphify.ids`` for non-ASCII paths. + +``ids.make_id`` keeps Unicode letters and digits (``docs/tz/ЛЗ4.md`` -> +``docs_tz_лз4``), but the prompts said ids are "only [a-z0-9_]". A model cannot +follow both, so it invented a different stem per chunk and per run. The prompts now +state the real rule and hand the model each file's canonical stem; these tests pin +the wording, the stem helper, the stem in the chunk prompt, and the skill's code +step that writes the stems for the subagents. +""" +from __future__ import annotations + +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from graphify.extract import extract # noqa: E402 +from graphify.llm import _EXTRACTION_SYSTEM, _read_files, _wrap_untrusted # noqa: E402 +from tools.skillgen import gen # noqa: E402 + +# Paths from the issue, with the stems `make_id(_file_stem(...))` gives for them. +CYRILLIC = "docs/tz/ЛЗ4_обоснование_с_листом_Расчет_Кдог.md" +CYRILLIC_STEM = "docs_tz_лз4_обоснование_с_листом_расчет_кдог" +SPACED = "docs/tz/СРЗ КП18-С301 ЛЗ2 (105) - 12.10.2026.md" +SPACED_STEM = "docs_tz_срз_кп18_с301_лз2_105_12_10_2026" + + +def _file_id_stem(*args, **kwargs) -> str: + from graphify.extractors import base + return base.file_id_stem(*args, **kwargs) + + +# --- the rule text, in every prompt ------------------------------------------ + + +def _spec_artifacts() -> list[tuple[str, str]]: + return [(a.path, a.content) for a in gen.render_all(gen.load_platforms()) + if a.path.endswith("references/extraction-spec.md")] + + +def test_the_spec_scan_sees_every_host(): + """The parametrized guard below must not pass vacuously.""" + assert len(_spec_artifacts()) == 14, [p for p, _ in _spec_artifacts()] + + +def test_llm_prompt_states_the_unicode_rule(): + assert "only [a-z0-9_]" not in _EXTRACTION_SYSTEM + assert "letters and digits of any script are kept" in _EXTRACTION_SYSTEM.lower() + assert _EXTRACTION_SYSTEM.count("never transliterated") == 1 + assert "Cyrillic" not in _EXTRACTION_SYSTEM + assert 'stem="..."' in _EXTRACTION_SYSTEM + + +@pytest.mark.parametrize("path,content", _spec_artifacts(), ids=[p for p, _ in _spec_artifacts()]) +def test_spec_states_the_unicode_rule(path: str, content: str): + assert "only `[a-z0-9_]`" not in content, path + assert "Letters and digits of any script are kept as written" in content, path + assert "never transliterated" in content, path + assert "stem" in content, path + + +def test_no_generated_skill_still_says_only_ascii_ids(): + offenders = [a.path for a in gen.render_all(gen.load_platforms()) + if re.search(r"only \[?`?\[a-z0-9_\]", a.content)] + assert not offenders, offenders + + +# --- file_id_stem ------------------------------------------------------------ + + +@pytest.mark.parametrize("path,stem", [(CYRILLIC, CYRILLIC_STEM), (SPACED, SPACED_STEM), + ("docs/guide.md", "docs_guide"), ("setup.py", "setup")]) +def test_file_id_stem_of_a_relative_path(path: str, stem: str): + assert _file_id_stem(path) == stem + + +def test_file_id_stem_resolves_an_absolute_path_against_root_dot(tmp_path: Path, monkeypatch): + """detect gives absolute paths and the scan root is often `.`; both must resolve + to the same root-relative stem, not to ''.""" + (tmp_path / "docs" / "tz").mkdir(parents=True) + doc = tmp_path / CYRILLIC + doc.write_text("# x\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + assert _file_id_stem(str(doc), ".") == CYRILLIC_STEM + assert _file_id_stem(str(doc), str(tmp_path)) == CYRILLIC_STEM + assert _file_id_stem(CYRILLIC, ".") == CYRILLIC_STEM + + +def test_file_id_stem_is_empty_outside_the_root(tmp_path: Path): + """No root-relative form: never bake the checkout location into an id.""" + root = tmp_path / "proj" + root.mkdir() + other = tmp_path / "elsewhere" / "a.md" + other.parent.mkdir() + other.write_text("x", encoding="utf-8") + assert _file_id_stem(str(other), str(root)) == "" + + +def test_file_id_stem_of_quotes_and_angle_brackets_is_word_characters_only(): + """`"`, `<` and `>` in a file name must not survive into the id: the stem is + written into a prompt attribute and a tab-separated file.""" + stem = _file_id_stem('docs/a"bd.md') + assert stem == "docs_a_b_c_d" + assert stem and all(ch.isalnum() or ch == "_" for ch in stem) + + +# --- the stem in the chunk prompt -------------------------------------------- + + +def test_chunk_prompt_carries_each_files_stem(tmp_path: Path): + for rel in (CYRILLIC, SPACED): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text("# заголовок\n", encoding="utf-8") + prompt = _read_files([tmp_path / CYRILLIC], tmp_path) + assert f'stem="{CYRILLIC_STEM}"' in prompt + assert f'stem="{SPACED_STEM}"' in _read_files([tmp_path / SPACED], tmp_path) + + +def test_chunk_prompt_for_an_ascii_path_differs_only_by_the_stem(tmp_path: Path): + doc = tmp_path / "docs" / "guide.md" + doc.parent.mkdir() + doc.write_text("# Intro\n", encoding="utf-8") + prompt = _read_files([doc], tmp_path) + assert 'stem="docs_guide"' in prompt + assert re.sub(r' stem="[^"]*"', "", prompt) == _wrap_untrusted("docs/guide.md", "# Intro\n") + + +def test_chunk_prompt_stem_attribute_cannot_carry_a_quote(tmp_path: Path): + doc = tmp_path / 'a"b.md' + doc.write_text("x", encoding="utf-8") + (stem,) = re.findall(r' stem="([^"]*)"', _read_files([doc], tmp_path)) + assert stem == "a_b" + + +# --- the stem equals what the real pipeline mints ----------------------------- + + +@pytest.mark.parametrize("rel", [CYRILLIC, "docs/guide.md"]) +def test_file_id_stem_is_the_prefix_the_pipeline_mints(tmp_path: Path, rel: str): + """The document pass mints the file's own node under the canonical stem; the stem + handed to the model is that same string, and a semantic node carrying it is left + alone by the build-time remap.""" + from graphify.build import _semantic_id_remap + + doc = tmp_path / rel + doc.parent.mkdir(parents=True, exist_ok=True) + doc.write_text("# Раздел\n\nТекст.\n", encoding="utf-8") + + stem = _file_id_stem(str(doc), str(tmp_path)) + nodes = extract([doc], cache_root=tmp_path, root=tmp_path)["nodes"] + (page,) = [n for n in nodes if n.get("node_kind") == "page"] + assert page["id"] == stem + assert all(n["id"] == stem or n["id"].startswith(stem + "_") for n in nodes) + + assert f'stem="{stem}"' in _read_files([doc], tmp_path) + semantic = [{"id": f"{stem}_понятие", "label": "понятие", "file_type": "concept", + "source_file": rel}] + assert _semantic_id_remap(semantic, str(tmp_path)) == {} + + +# --- the skill's code step ---------------------------------------------------- + + +def _cache_step_body(text: str) -> str: + """The python body of the Part B cache step, un-escaped from its bash quoting.""" + for block in re.findall(r'-c "\n(.*?)\n"\n', text, re.S): + if "check_semantic_cache" in block and "_uncached_stems" in block: + return block.replace('\\"', '"') + raise AssertionError("cache step not found") + + +@pytest.mark.skipif(sys.platform == "win32", reason="runs the bash-quoted python body") +def test_skill_cache_step_writes_a_stem_for_every_uncached_file(tmp_path: Path): + skill = (REPO_ROOT / "graphify" / "skill.md").read_text(encoding="utf-8") + body = _cache_step_body(skill) + assert body.count("INPUT_PATH") == 1, "the scan root is set once and reused" + + for rel in (CYRILLIC, "docs/guide.md"): + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / rel).write_text("# x\n", encoding="utf-8") + spec = tmp_path / "spec.md" + spec.write_text("prompt", encoding="utf-8") + out = tmp_path / "graphify-out" + out.mkdir() + import json + (out / ".graphify_detect.json").write_text(json.dumps( + {"files": {"document": [str(tmp_path / CYRILLIC), str(tmp_path / "docs/guide.md")]}}, + ensure_ascii=False), encoding="utf-8") + + script = body.replace("INPUT_PATH", ".").replace("SPEC_PATH", str(spec)) + done = subprocess.run([sys.executable, "-c", script], cwd=tmp_path, capture_output=True, + text=True, env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}) + assert done.returncode == 0, done.stderr + + lines = (out / ".graphify_uncached_stems.txt").read_text(encoding="utf-8").splitlines() + assert lines == [f"{tmp_path / CYRILLIC}\t{CYRILLIC_STEM}", f"{tmp_path / 'docs/guide.md'}\tdocs_guide"] + bare = (out / ".graphify_uncached.txt").read_text(encoding="utf-8").splitlines() + assert bare == [line.split("\t")[0] for line in lines] + + +# Hosts whose own monolith body has its own Part B; they never carried the rule and +# are not covered here. +_MONOLITHS = {"graphify/skill-aider.md", "graphify/skill-devin.md"} + + +def _skill_artifacts() -> list[tuple[str, str]]: + return [(a.path, a.content) for a in gen.render_all(gen.load_platforms()) + if re.fullmatch(r"graphify/skill[-a-z]*\.md", a.path) + and "check_semantic_cache" in a.content and a.path not in _MONOLITHS] + + +@pytest.mark.parametrize("path,content", _skill_artifacts(), ids=[p for p, _ in _skill_artifacts()]) +def test_skill_hands_the_stems_to_the_subagents(path: str, content: str): + assert "file_id_stem(f, root)" in content, path + assert "`pathstem`" in content, path + assert "fall back to `.graphify_uncached.txt`" in content, path + assert ".graphify_uncached_stems.txt" in content.split("Clean up temp files:")[-1].splitlines()[0], path + + +def test_the_skill_scan_sees_the_hosts(): + assert len(_skill_artifacts()) == 14, [p for p, _ in _skill_artifacts()] diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index f4fee6921d..f9b6940520 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index f4fee6921d..f9b6940520 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index d09ad219b2..b168c1d221 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index 7a3c0f3989..7246a17410 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -415,7 +420,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index d09ad219b2..b168c1d221 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index 819a47f88e..04cc93d0ec 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -411,7 +416,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index 6fe81c4a6d..72dcd21516 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index d09ad219b2..b168c1d221 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 332c01c810..6f47ad3cd0 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -406,7 +411,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index 46b7819feb..ab5a1fc957 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -408,7 +413,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index 69fcf1538b..f690dd0caf 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -412,7 +417,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 89ef00e794..ea02213b1c 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -410,7 +415,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 0219a32435..95120739e7 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -270,7 +270,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -279,6 +280,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding="utf-8") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding="utf-8") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -291,7 +296,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -442,7 +447,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached["nodes"])} from cache, {len(new.get("nodes",[]))} new)') '@ | & (Get-Content graphify-out\.graphify_python) - ``` -Clean up temp files: `Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_cached.json, graphify-out\.graphify_uncached.txt, graphify-out\.graphify_semantic_new.json` +Clean up temp files: `Remove-Item -Force -ErrorAction SilentlyContinue graphify-out\.graphify_cached.json, graphify-out\.graphify_uncached.txt, graphify-out\.graphify_uncached_stems.txt, graphify-out\.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index d09ad219b2..b168c1d221 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -242,7 +242,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -251,6 +252,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -263,7 +268,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -414,7 +419,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md index 8a2aa6aa59..1ef1909810 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__extraction-spec.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 7c997762c2..100cb7b60c 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -195,7 +195,8 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') +root = 'INPUT_PATH' +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root=root, prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -204,6 +205,10 @@ if cached_nodes or cached_edges or cached_hyperedges: else: Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True) Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\") +# One pathstem line per uncached file, same order: the subagents copy the stem +# instead of guessing it (#4432). Computed here in code, never pasted. +from graphify.extractors.base import file_id_stem +Path('graphify-out/.graphify_uncached_stems.txt').write_text('\n'.join(f'{f}{chr(9)}{file_id_stem(f, root)}' for f in uncached), encoding=\"utf-8\") # Nothing has been dispatched yet, so any chunk file on disk is a leftover from an # interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds. for stale in Path('graphify-out').glob('.graphify_chunk_*.json'): @@ -216,7 +221,7 @@ Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt **Step B1 - Split into chunks** -Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. +Load the lines of `graphify-out/.graphify_uncached_stems.txt` (the same files in the same order as `.graphify_uncached.txt`, each as `pathstem`; if that file is missing, fall back to `.graphify_uncached.txt`, one bare path per line). A chunk's FILE_LIST is its lines, copied unchanged. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted. Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225). @@ -343,7 +348,7 @@ Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, inden print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)') " ``` -Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json` +Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_uncached_stems.txt graphify-out/.graphify_semantic_new.json` #### Part C - Merge AST + semantic into final extraction diff --git a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md index 8a2aa6aa59..1ef1909810 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec-compact.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec-compact.md @@ -30,10 +30,10 @@ Rules: - If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author, contributor onto every node from that file. - confidence_score is REQUIRED on every edge — never omit it, never use 0.5 as a default. EXTRACTED = 1.0 always. INFERRED: pick exactly ONE of 0.95 (direct structural evidence), 0.85 (strong inference), 0.75 (reasonable inference), 0.65 (weak inference), 0.55 (speculative but plausible) — never 0.5; if none fit, mark the edge AMBIGUOUS. AMBIGUOUS = 0.1-0.3. -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_` (each lowercased with non-alphanumeric chars replaced by `_`) and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character becomes one `_`. Format `{stem}_{entity}` where stem is the full repo-relative path with the extension dropped, every segment joined with `_`, and entity is the symbol name similarly normalized (`docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`). A FILE_LIST line may read `pathstem`: use `stem` verbatim as the id prefix for that file, never respelled or transliterated. Use every directory level, not just the immediate parent. `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`. Top-level files use just the filename stem. This must match the AST extractor's ID. Never append chunk or sequence suffixes — IDs must be deterministic from the label alone. Output exactly this JSON (no other text): {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE: set source_file to the FILE_LIST path for that file VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. +source_file RULE: set source_file to the FILE_LIST path for that file (the part before the TAB, if any) VERBATIM (absolute, no shortening to basename, no re-relativizing, no separator change). Keeps full build and --update on one base so build_merge's replace matches instead of duplicating. ``` diff --git a/tools/skillgen/fragments/references/shared/extraction-spec.md b/tools/skillgen/fragments/references/shared/extraction-spec.md index 17c25da894..a8b0ff6bbf 100644 --- a/tools/skillgen/fragments/references/shared/extraction-spec.md +++ b/tools/skillgen/fragments/references/shared/extraction-spec.md @@ -74,12 +74,12 @@ confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a d the edge AMBIGUOUS rather than picking 0.4 or below. - AMBIGUOUS edges: 0.1-0.3 -Node ID format: lowercase, only `[a-z0-9_]`, no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. +Node ID format: lowercase. Letters and digits of any script are kept as written (never transliterated to ASCII); every run of any other character (dots, slashes, spaces, brackets, punctuation) becomes one `_`. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased, other characters replaced as above), and entity is the symbol name similarly normalized. A FILE_LIST line may read `pathstem`: then `stem` is the exact id prefix for every node from that file. Copy it character for character; never respell, shorten or transliterate it. A line with no tab: derive the stem as described here. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`; `docs/отчёт.md` + `Итоги` → `docs_отчёт_итоги`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it. Generate the extraction JSON matching this schema exactly: {"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":""}],"input_tokens":0,"output_tokens":0,"coverage":{"":0}} -source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. +source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST (the part before the TAB, if the line has one) — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate. Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost): CHUNK_PATH