Fix six open issues: stale imports, MCP validation, import-graph gaps, dup importers, I/O errors, scope docs

- pipeline: re-resolve imports for every file on update (not just changed),
  so add/remove/rename of a target fixes unchanged importers; propagate hash
  and read errors instead of treating unreadable files as empty
- discovery: report unreadable walk entries to stderr instead of dropping them
- imports: strip `pub` visibility so `pub use` re-exports resolve; share
  extension/module-entry probe lists (adds cc,cxx,hpp,hh,hxx,mts,cts,jsx,mjs,cjs)
- store: DISTINCT + ORDER in importers_of to dedup repeated edges
- mcp: validate required tool args (explicit isError), emit JSON-RPC -32700
  parse errors and -32600 invalid-request instead of silently skipping
- README: document the 13-tool surface vs toktoken's 27 and known gaps
- tests: regression coverage for update re-resolution, pub-use re-exports,
  importer dedup, and MCP error reporting

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

src/discovery/mod.rs

@@ -32,10 +32,15 @@ pub fn discover(project: &Path) -> Result<Vec<SourceFile>> {
     });
 
     let mut out = Vec::new();
+    let mut skipped = Vec::new();
     for entry in builder.build() {
         let entry = match entry {
             Ok(e) => e,
-            Err(_) => continue,
+            Err(e) => {
+                // Report unreadable entries instead of dropping them silently.
+                skipped.push(e.to_string());
+                continue;
+            }
         };
         let path = entry.path();
         if entry.file_type().is_some_and(|t| t.is_dir()) {
@@ -54,6 +59,9 @@ pub fn discover(project: &Path) -> Result<Vec<SourceFile>> {
             language,
         });
     }
+    for s in &skipped {
+        eprintln!("rustytokens: skipped unreadable entry: {s}");
+    }
     out.sort_by(|a, b| a.rel.cmp(&b.rel));
     Ok(out)
 }

src/index/imports.rs

@@ -75,10 +75,22 @@ fn extract_python(content: &str) -> Vec<Import> {
     out
 }
 
+/// Strip a leading visibility modifier (`pub`, `pub(crate)`, `pub(super)`, …)
+/// so `pub use`/`pub mod` parse the same as their private counterparts.
+fn strip_visibility(l: &str) -> &str {
+    if let Some(r) = l.strip_prefix("pub ") {
+        r
+    } else if let Some(r) = l.strip_prefix("pub(") {
+        r.find(") ").map(|i| &r[i + 2..]).unwrap_or(l)
+    } else {
+        l
+    }
+}
+
 fn extract_rust(content: &str) -> Vec<Import> {
     let mut out = Vec::new();
     for line in content.lines() {
-        let l = line.trim();
+        let l = strip_visibility(line.trim());
         if let Some(rest) = l.strip_prefix("use ") {
             let rest = rest
                 .trim_end_matches(';')
@@ -293,6 +305,17 @@ fn clean(target: &str) -> Option<String> {
     Some(t.to_string())
 }
 
+/// Code file extensions probed when resolving extensionless import targets.
+/// Mirrors `config::language_for` for code languages (html/css assets are
+/// always imported with an explicit extension, so they need no probing).
+const RESOLVE_EXTS: &[&str] = &[
+    "rs", "py", "ts", "tsx", "mts", "cts", "js", "jsx", "mjs", "cjs", "c", "h", "cc", "cpp", "cxx",
+    "hpp", "hh", "hxx", "java",
+];
+
+/// Module-directory entry files tried for `dir/mod` style resolution.
+const MODULE_ENTRIES: &[&str] = &["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"];
+
 impl Resolver {
     pub fn build(files: Vec<(String, i64)>, crate_name: Option<&str>) -> Self {
         let mut by_path = HashMap::new();
@@ -380,12 +403,12 @@ impl Resolver {
         if let Some(id) = self.by_path.get(&format!("{base}{t}")) {
             return Some(*id);
         }
-        for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
+        for ext in RESOLVE_EXTS {
             if let Some(id) = self.by_path.get(&format!("{base}{t}.{ext}")) {
                 return Some(*id);
             }
         }
-        for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
+        for entry in MODULE_ENTRIES {
             if let Some(id) = self.by_path.get(&format!("{base}{t}/{entry}")) {
                 return Some(*id);
             }
@@ -394,12 +417,12 @@ impl Resolver {
     }
 
     fn try_module(&self, base: &str) -> Option<i64> {
-        for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
+        for ext in RESOLVE_EXTS {
             if let Some(id) = self.by_path.get(&format!("{base}.{ext}")) {
                 return Some(*id);
             }
         }
-        for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
+        for entry in MODULE_ENTRIES {
             if let Some(id) = self.by_path.get(&format!("{base}/{entry}")) {
                 return Some(*id);
             }

src/index/pipeline.rs

@@ -56,15 +56,17 @@ pub fn run(project: &Path, db_path: &Path, recreate: bool) -> Result<IndexStats>
         .map(|f| (f.path.clone(), (f.id, f.content_hash.clone())))
         .collect();
 
-    // Hash all files in parallel.
+    // Hash all files in parallel. Propagate read errors instead of silently
+    // treating an unreadable file as an empty (all-zero) hash.
     let hashed: Vec<(String, String, i64, &'static str)> = files
         .par_iter()
         .map(|f| {
-            let h = hashing::hash_file(&f.path).unwrap_or_else(|_| "0000000000000000".to_string());
+            let h = hashing::hash_file(&f.path)
+                .with_context(|| format!("cannot hash {}", f.path.display()))?;
             let m = hashing::mtime_secs(&f.path);
-            (f.rel.clone(), h, m, f.language)
+            Ok((f.rel.clone(), h, m, f.language))
         })
-        .collect();
+        .collect::<Result<Vec<_>>>()?;
 
     let by_rel: HashMap<&str, &SourceFile> = files.iter().map(|f| (f.rel.as_str(), f)).collect();
     let abs_to_rel: HashMap<String, String> = files
@@ -112,11 +114,27 @@ pub fn run(project: &Path, db_path: &Path, recreate: bool) -> Result<IndexStats>
         read_crate_name(project).as_deref(),
     );
 
+    // Symbols + content FTS: only files whose content changed. Skipping ctags
+    // for unchanged files is the whole point of the hash optimization.
     for f in &changed {
         let fid = file_ids[&f.rel];
-        let content = std::fs::read_to_string(&f.path).unwrap_or_default();
+        let content = std::fs::read_to_string(&f.path)
+            .with_context(|| format!("cannot read {}", f.path.display()))?;
         let tags = tags_by_rel.get(&f.rel).cloned().unwrap_or_default();
         store::replace_symbols(&conn, fid, &tags_to_symbols(&tags, &content))?;
+        conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [fid])?;
+        conn.execute(
+            "INSERT INTO content_fts(rowid, path, content) VALUES (?1, ?2, ?3)",
+            params![fid, f.rel, content],
+        )?;
+    }
+
+    // Imports: re-resolve for EVERY file. Adding/removing/renaming a file can
+    // change whether an unchanged importer resolves, so resolution must cover
+    // the whole file set, not just the changed set. (Cheap: read + regex, no
+    // ctags subprocess.)
+    for f in &files {
+        let fid = file_ids[&f.rel];
         store::clear_imports_for_file(&conn, fid)?;
         let dir = f
             .rel
@@ -131,11 +149,6 @@ pub fn run(project: &Path, db_path: &Path, recreate: bool) -> Result<IndexStats>
                 store::insert_import(&conn, fid, to, &imp.import_type)?;
             }
         }
-        conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [fid])?;
-        conn.execute(
-            "INSERT INTO content_fts(rowid, path, content) VALUES (?1, ?2, ?3)",
-            params![fid, f.rel, content],
-        )?;
     }
 
     conn.execute_batch("COMMIT;")?;

src/index/store.rs

@@ -200,7 +200,9 @@ pub fn symbols_for_file(conn: &Connection, file_id: i64) -> Result<Vec<SymbolRow
 }
 
 pub fn importers_of(conn: &Connection, file_id: i64) -> Result<Vec<i64>> {
-    let mut stmt = conn.prepare("SELECT from_file_id FROM imports WHERE to_file_id = ?1")?;
+    let mut stmt = conn.prepare(
+        "SELECT DISTINCT from_file_id FROM imports WHERE to_file_id = ?1 ORDER BY from_file_id",
+    )?;
     let rows = stmt.query_map([file_id], |r| r.get(0))?;
     Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
 }

src/mcp/mod.rs

@@ -22,7 +22,22 @@ pub fn serve(project: &Path) -> Result<()> {
         }
         let msg: Value = match serde_json::from_str(&line) {
             Ok(m) => m,
-            Err(_) => continue,
+            Err(_) => {
+                // Unparseable input is a JSON-RPC parse error, not something to
+                // drop silently.
+                let mut out = stdout.lock();
+                serde_json::to_writer(
+                    &mut out,
+                    &json!({
+                        "jsonrpc": "2.0",
+                        "id": null,
+                        "error": { "code": -32700, "message": "parse error" }
+                    }),
+                )?;
+                out.write_all(b"\n")?;
+                out.flush()?;
+                continue;
+            }
         };
         if let Some(resp) = handle(&msg, project) {
             let mut out = stdout.lock();
@@ -35,12 +50,20 @@ pub fn serve(project: &Path) -> Result<()> {
 }
 
 fn handle(msg: &Value, project: &Path) -> Option<Value> {
-    let method = msg.get("method")?.as_str()?;
     let id = msg.get("id").cloned();
 
     // Notifications have no id -> no response.
     let id = id.as_ref()?;
 
+    // Requests must carry a string method name.
+    let Some(method) = msg.get("method").and_then(|m| m.as_str()) else {
+        return Some(json!({
+            "jsonrpc": "2.0",
+            "id": id,
+            "error": { "code": -32600, "message": "invalid request" }
+        }));
+    };
+
     match method {
         "initialize" => Some(json!({
             "jsonrpc": "2.0",
@@ -93,38 +116,38 @@ fn call_tool(name: &str, args: &Value, project: &Path) -> Value {
 
 fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
     use crate::ops;
-    let s = |k: &str| args.get(k).and_then(|v| v.as_str()).unwrap_or("");
+
+    fn req_str<'a>(args: &'a Value, key: &str, tool: &str) -> Result<&'a str> {
+        args.get(key)
+            .and_then(|v| v.as_str())
+            .filter(|s| !s.is_empty())
+            .ok_or_else(|| anyhow::anyhow!("{tool}: missing or invalid argument '{key}'"))
+    }
+    fn req_i64(args: &Value, key: &str, tool: &str) -> Result<i64> {
+        args.get(key)
+            .and_then(|v| v.as_i64())
+            .ok_or_else(|| anyhow::anyhow!("{tool}: missing or invalid argument '{key}'"))
+    }
+    let opt_str = |k: &str| args.get(k).and_then(|v| v.as_str());
+    let opt_depth = |k: &str| args.get(k).and_then(|v| v.as_u64()).map(|d| d as usize);
+
     match name {
-        "index_create" => ops::index_create(Path::new(s("path"))),
-        "index_update" => ops::index_update(Path::new(s("path"))),
-        "search_symbols" => ops::search_symbols(project, s("query")),
-        "search_text" => ops::search_text(project, s("query")),
-        "inspect_symbol" => ops::inspect_symbol(
-            project,
-            args.get("id").and_then(|v| v.as_i64()).unwrap_or(0),
-        ),
-        "inspect_outline" => ops::inspect_outline(project, s("file")),
-        "inspect_file" => ops::inspect_file(
-            project,
-            s("file"),
-            args.get("lines").and_then(|v| v.as_str()),
-        ),
-        "inspect_tree" => ops::inspect_tree(
-            project,
-            args.get("depth")
-                .and_then(|v| v.as_u64())
-                .map(|d| d as usize),
-        ),
-        "find_importers" => ops::find_importers(project, s("file")),
+        "index_create" => ops::index_create(Path::new(req_str(args, "path", name)?)),
+        "index_update" => ops::index_update(Path::new(req_str(args, "path", name)?)),
+        "search_symbols" => ops::search_symbols(project, req_str(args, "query", name)?),
+        "search_text" => ops::search_text(project, req_str(args, "query", name)?),
+        "inspect_symbol" => ops::inspect_symbol(project, req_i64(args, "id", name)?),
+        "inspect_outline" => ops::inspect_outline(project, req_str(args, "file", name)?),
+        "inspect_file" => {
+            ops::inspect_file(project, req_str(args, "file", name)?, opt_str("lines"))
+        }
+        "inspect_tree" => ops::inspect_tree(project, opt_depth("depth")),
+        "find_importers" => ops::find_importers(project, req_str(args, "file", name)?),
         "find_dead" => ops::find_dead(project),
         "inspect_cycles" => ops::inspect_cycles(project),
-        "inspect_blast_radius" => ops::inspect_blast_radius(
-            project,
-            s("file"),
-            args.get("depth")
-                .and_then(|v| v.as_u64())
-                .map(|d| d as usize),
-        ),
+        "inspect_blast_radius" => {
+            ops::inspect_blast_radius(project, req_str(args, "file", name)?, opt_depth("depth"))
+        }
         "stats" => ops::stats(project),
         _ => anyhow::bail!("unknown tool: {name}"),
     }