Add --exclude, detect/inspect_bundle tools, Spring/test entry-point detection

- Global `--exclude <glob>` (repeatable) on search symbols/text and
  find dead/dead-symbols, with built-in defaults (vendor/, node_modules/,
  *.min.*, dist/, build/). Minified/vendored assets stay indexed for import
  edges but are filtered from query results (bootstrap.min.css noise: 90% of
  dead symbols -> gone).
- `detect` CLI / `codebase_detect` MCP tool: probe a directory's languages and
  stack from file extensions and manifest files before indexing.
- `inspect bundle` CLI / `inspect_bundle` MCP tool: symbol source + direct
  callers + direct importers in one call (3 round-trips -> 1).
- find dead-symbols: skip CSS files and structural kinds (package, heading*)
  as non-callable; extend framework-wired detection with JUnit/Spring
  annotations (@Test, @Service, @Component, ...) and pytest/JUnit test-class
  naming (Test*, test_*, *Test, *Tests), plus `test_` methods reported as
  `member` kind.
- README: correct CSS statement, document new tools/limits.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
endergebnisandCopilot committed 2026-09-08 22:48:45 +02:00
1 parent b5389341a1
commit 4b55a41ece
9 files changed
+452 -41

No files matched your search

Generated
+1
View File
@@ -399,6 +399,7 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
"globset",
"ignore", "ignore",
"rayon", "rayon",
"regex", "regex",
+1
View File
@@ -8,6 +8,7 @@ license = "MIT"
clap = { version = "4", features = ["derive"] } clap = { version = "4", features = ["derive"] }
rusqlite = { version = "0.31", features = ["bundled"] } rusqlite = { version = "0.31", features = ["bundled"] }
ignore = "0.4" ignore = "0.4"
globset = "0.4"
rayon = "1" rayon = "1"
xxhash-rust = { version = "0.8", features = ["xxh3"] } xxhash-rust = { version = "0.8", features = ["xxh3"] }
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
+25 -14
View File
@@ -8,8 +8,10 @@ context tokens.
Language coverage (via ctags): Rust, Java, Python, C/C++, TypeScript, Language coverage (via ctags): Rust, Java, Python, C/C++, TypeScript,
JavaScript, HTML, C#. HTML import edges are extracted directly (not via ctags), JavaScript, HTML, C#. HTML import edges are extracted directly (not via ctags),
and `<link>`/`<script>`/`<img>` references to local `.css`/`.js`/image files are and `<link>`/`<script>`/`<img>` references to local `.css`/`.js`/image files are
tracked as imports. CSS files are discovered and import-tracked, but ctags has tracked as imports. universal-ctags (5.9.0+) *does* index CSS selectors as
no CSS parser — `search symbols` returns nothing for `.css` files. symbols, so `search symbols` can return them — `find dead-symbols` skips CSS
files and structural kinds (selectors/`package`/`heading*` are not callable
symbols) and the default `--exclude` set filters minified/generated assets.
`rustytokens` is an independent, clean-room implementation. It does not share `rustytokens` is an independent, clean-room implementation. It does not share
code with any AGPL-licensed project. code with any AGPL-licensed project.
@@ -42,22 +44,27 @@ cargo build --release
| `index create <path>` | Index a project from scratch | | `index create <path>` | Index a project from scratch |
| `index update <path>` | Re-index only changed files (content-hash diff) | | `index update <path>` | Re-index only changed files (content-hash diff) |
| `index file <file>` | Re-index a single file | | `index file <file>` | Re-index a single file |
| `search symbols <query> [--include-source] [--limit n]` | Symbol search (exact > prefix > full-text > substring) | | `detect [path]` | Detect languages/stack at a path before indexing (no index needed) |
| `search text <query> [--limit n]` | Full-text search with line numbers, read live from disk | | `search symbols <query> [--include-source] [--limit n] [--exclude glob]` | Symbol search (exact > prefix > full-text > substring) |
| `search text <query> [--limit n] [--exclude glob]` | Full-text search with line numbers, read live from disk |
| `inspect symbol <id\|name>` | Source of a single symbol (id or exact name) | | `inspect symbol <id\|name>` | Source of a single symbol (id or exact name) |
| `inspect bundle <id\|name>` | Symbol source + direct callers + direct importers in one call |
| `inspect outline <file>` | Symbol outline of a file | | `inspect outline <file>` | Symbol outline of a file |
| `inspect file <file> [--lines a-b]` | Read a file or line range | | `inspect file <file> [--lines a-b]` | Read a file or line range |
| `inspect tree [--depth n]` | Indexed file tree | | `inspect tree [--depth n]` | Indexed file tree |
| `inspect blast-radius <file> [--depth n]` | Files transitively affected by a change | | `inspect blast-radius <file> [--depth n]` | Files transitively affected by a change |
| `find importers <file>` | Who imports this file | | `find importers <file>` | Who imports this file |
| `find dead` | Files nobody imports (entry points excluded) | | `find dead [--exclude glob]` | Files nobody imports (entry points excluded) |
| `find dead-symbols` | Symbols never referenced (framework-wired entry points excluded) | | `find dead-symbols [--exclude glob]` | Symbols never referenced (framework-wired entry points excluded) |
| `find callers <id\|name>` | Symbols that call the given symbol | | `find callers <id\|name>` | Symbols that call the given symbol |
| `find references <identifier>` | Code lines referencing an identifier (definitions/strings/comments excluded) | | `find references <identifier>` | Code lines referencing an identifier (definitions/strings/comments excluded) |
| `inspect cycles` | Circular imports | | `inspect cycles` | Circular imports |
| `serve` | MCP server over stdio | | `serve` | MCP server over stdio |
| `stats` | Index statistics | | `stats` | Index statistics |
`--exclude` is a repeatable glob; it adds to built-in defaults (`vendor/`,
`node_modules/`, `*.min.*`, `dist/`, `build/`).
Global options: `--project <dir>` (default `.`), `--format json|table` Global options: `--project <dir>` (default `.`), `--format json|table`
(default `json`), `--compact`. (default `json`), `--compact`.
@@ -150,11 +157,11 @@ Notes:
## Scope and known gaps ## Scope and known gaps
`rustytokens` covers the same *core* retrieval surface for Java/Python/Rust/ `rustytokens` covers the same *core* retrieval surface for Java/Python/Rust/
C/C++/HTML as `toktoken`, but intentionally implements a smaller tool set: 17 C/C++/HTML as `toktoken`, but intentionally implements a smaller tool set: 19
tools vs `toktoken`'s 27. The following `toktoken` tools have **no equivalent tools vs `toktoken`'s 27. The following `toktoken` tools have **no equivalent
here yet** (deliberately, not an oversight): here yet** (deliberately, not an oversight):
- `inspect_bundle`, `inspect_dependencies`, `inspect_hierarchy` (graph views) - `inspect_dependencies`, `inspect_hierarchy` (graph views)
- `search_similar`, `search_cooccurrence` (similarity search) - `search_similar`, `search_cooccurrence` (similarity search)
- GitHub indexing and centrality ranking - GitHub indexing and centrality ranking
@@ -163,17 +170,21 @@ for the "roughly cover Java/Python/Rust/C/C++/HTML" goal.
Known limits that are by design, not bugs: Known limits that are by design, not bugs:
- `.css` files are discovered and import-tracked, but `ctags` has no CSS - universal-ctags (5.9.0+) indexes CSS selectors as symbols, so `search
parser — `search symbols` returns nothing for `.css`. symbols` may return them. `find dead-symbols` skips CSS files and structural
kinds (`package`, `heading*`) — selectors/declarations are not callable
symbols — and the default `--exclude` set drops minified assets.
- `find dead` over-counts for C/C++ header-only entry points (documented - `find dead` over-counts for C/C++ header-only entry points (documented
conservatism); treat results as candidates, not verdicts. conservatism); treat results as candidates, not verdicts.
- `find callers` / `find references` are word-boundary heuristics: they match a - `find callers` / `find references` are word-boundary heuristics: they match a
symbol's name in text, so same-name symbols in unrelated files can produce symbol's name in text, so same-name symbols in unrelated files can produce
false positives (`find callers` narrows this via the import graph). false positives (`find callers` narrows this via the import graph).
- `find dead-symbols` treats pytest `test_*` functions, `#[test]`-wired Rust - `find dead-symbols` treats pytest `test_*` functions, `Test*`/`test_*` test
tests and framework-decorated symbols (FastAPI/Flask routes, Celery tasks, classes, `#[test]`-wired Rust tests, JUnit/Spring annotations (`@Test`,
click/typer commands, …) as live entry points. A pure name-scan cannot see `@Service`, `@Component`, …) and other framework-decorated symbols
these call sites, so this is a documented heuristic. (FastAPI/Flask routes, Celery tasks, click/typer commands, …) as live entry
points. A pure name-scan cannot see these call sites, so this is a documented
heuristic.
- `find references` filters the definition line and occurrences inside strings - `find references` filters the definition line and occurrences inside strings
and comments; multi-line docstrings/block comments are approximated and comments; multi-line docstrings/block comments are approximated
line-by-line, so a symbol name inside a long string block may still appear. line-by-line, so a symbol name inside a long string block may still appear.
+13
View File
@@ -23,6 +23,11 @@ pub struct Cli {
#[arg(long, global = true)] #[arg(long, global = true)]
pub compact: bool, pub compact: bool,
/// Exclude files matching a glob from query results (repeatable; adds to
/// built-in defaults: vendor/, node_modules/, *.min.*, dist/, build/)
#[arg(long, global = true, action = clap::ArgAction::Append)]
pub exclude: Vec<String>,
#[command(subcommand)] #[command(subcommand)]
pub command: Command, pub command: Command,
} }
@@ -49,6 +54,12 @@ pub enum Command {
#[command(subcommand)] #[command(subcommand)]
sub: FindCommand, sub: FindCommand,
}, },
/// Detect languages/stack at a path before indexing
Detect {
/// Path to probe (defaults to current directory)
#[arg(default_value = ".")]
path: PathBuf,
},
/// Start the MCP server over stdio /// Start the MCP server over stdio
Serve, Serve,
/// Show index statistics /// Show index statistics
@@ -90,6 +101,8 @@ pub enum SearchCommand {
pub enum InspectCommand { pub enum InspectCommand {
/// Source code of a symbol by id or exact name /// Source code of a symbol by id or exact name
Symbol { target: String }, Symbol { target: String },
/// Symbol source + direct callers + direct importers in one call
Bundle { target: String },
/// Symbol outline of a file /// Symbol outline of a file
Outline { file: String }, Outline { file: String },
/// Read a file or a line range (e.g. --lines 10-40) /// Read a file or a line range (e.g. --lines 10-40)
+101
View File
@@ -2,6 +2,69 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
/// Default `--exclude` globs applied to query results (`find dead`,
/// `find dead-symbols`, `search symbols`, `search text`). Minified/generated
/// assets and vendored trees are indexed (so import edges resolve) but are
/// almost pure noise for symbol/call-site queries, so they are filtered out by
/// default.
pub const DEFAULT_EXCLUDES: &[&str] = &["vendor/", "node_modules/", "*.min.*", "dist/", "build/"];
/// Compiled glob matcher for `--exclude`. A path matches if any pattern (the
/// defaults plus user-supplied globs) matches the slash-normalized relative
/// path. A trailing `/` marks a directory: it is expanded to `**/<dir>/**` so
/// it matches at any depth. A bare filename glob (`*.min.*`) is expanded to
/// `**/*.min.*` for the same reason.
#[derive(Clone)]
pub struct ExcludeSet {
set: globset::GlobSet,
}
impl ExcludeSet {
pub fn new(extra: &[String]) -> ExcludeSet {
let mut builder = globset::GlobSetBuilder::new();
for pat in DEFAULT_EXCLUDES
.iter()
.copied()
.chain(extra.iter().map(|s| s.as_str()))
{
let normalized = normalize_glob(pat);
if normalized.is_empty() {
continue;
}
if let Ok(g) = globset::Glob::new(&normalized) {
builder.add(g);
}
}
let set = builder.build().unwrap_or_default();
ExcludeSet { set }
}
/// True if `rel` (slash-normalized relative path) is excluded.
pub fn matches(&self, rel: &str) -> bool {
self.set.is_match(rel)
}
}
/// Expand a user glob into a form that matches at any depth.
fn normalize_glob(pat: &str) -> String {
let p = pat.trim();
let dir = p.strip_suffix('/');
if let Some(d) = dir {
if d.is_empty() {
return String::new();
}
return format!("**/{d}/**");
}
if !p.contains('/') && !p.contains('*') && !p.contains('?') {
// A bare name with no slash or glob meta is a directory name.
return format!("**/{p}/**");
}
if !p.contains('/') {
return format!("**/{p}");
}
p.to_string()
}
/// Directory names skipped during discovery (own list). /// Directory names skipped during discovery (own list).
pub const SKIP_DIRS: &[&str] = &[ pub const SKIP_DIRS: &[&str] = &[
".git", ".git",
@@ -112,3 +175,41 @@ pub fn index_db_path(project: &Path) -> PathBuf {
.unwrap_or_else(|| "project".to_string()); .unwrap_or_else(|| "project".to_string());
cache_dir().join(format!("{name}-{key}.db")) cache_dir().join(format!("{name}-{key}.db"))
} }
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn normalize_glob_expands_dirs_and_bare_names() {
assert_eq!(normalize_glob("vendor/"), "**/vendor/**");
assert_eq!(normalize_glob("build"), "**/build/**");
assert_eq!(normalize_glob("*.min.*"), "**/*.min.*");
assert_eq!(normalize_glob("src/generated/"), "**/src/generated/**");
// A slash-containing glob is kept verbatim.
assert_eq!(normalize_glob("a/b/*.c"), "a/b/*.c");
// Trailing slash alone is dropped.
assert_eq!(normalize_glob("/"), "");
}
#[test]
fn exclude_set_matches_defaults_at_any_depth() {
let set = ExcludeSet::new(&[]);
assert!(set.matches("node_modules/left-pad/index.js"));
assert!(set.matches("web/bootstrap.min.css"));
assert!(set.matches("dist/bundle.js"));
assert!(set.matches("sub/build/out.txt"));
assert!(set.matches("vendor/foo.rs"));
assert!(!set.matches("src/main.rs"));
assert!(!set.matches("web/app.css"));
}
#[test]
fn exclude_set_merges_extra_globs() {
let extra = vec!["generated/".to_string(), "*.pb.go".to_string()];
let set = ExcludeSet::new(&extra);
assert!(set.matches("generated/foo.rs"));
assert!(set.matches("api/gen.pb.go"));
assert!(!set.matches("src/foo.rs"));
}
}
+14 -6
View File
@@ -29,6 +29,7 @@ fn main() {
fn run() -> Result<()> { fn run() -> Result<()> {
let cli = Cli::parse(); let cli = Cli::parse();
let excludes = config::ExcludeSet::new(&cli.exclude);
match &cli.command { match &cli.command {
Command::Index { sub } => match sub { Command::Index { sub } => match sub {
IndexCommand::Create { path } => emit(&ops::index_create(path)?, &cli)?, IndexCommand::Create { path } => emit(&ops::index_create(path)?, &cli)?,
@@ -41,17 +42,21 @@ fn run() -> Result<()> {
include_source, include_source,
limit, limit,
} => emit( } => emit(
&ops::search_symbols(&cli.project, query, *include_source, *limit)?, &ops::search_symbols(&cli.project, query, *include_source, *limit, &excludes)?,
&cli,
)?,
SearchCommand::Text { query, limit } => emit(
&ops::search_text(&cli.project, query, *limit, &excludes)?,
&cli, &cli,
)?, )?,
SearchCommand::Text { query, limit } => {
emit(&ops::search_text(&cli.project, query, *limit)?, &cli)?
}
}, },
Command::Inspect { sub } => match sub { Command::Inspect { sub } => match sub {
InspectCommand::Symbol { target } => { InspectCommand::Symbol { target } => {
emit(&ops::inspect_symbol(&cli.project, target)?, &cli)? emit(&ops::inspect_symbol(&cli.project, target)?, &cli)?
} }
InspectCommand::Bundle { target } => {
emit(&ops::inspect_bundle(&cli.project, target)?, &cli)?
}
InspectCommand::Outline { file } => { InspectCommand::Outline { file } => {
emit(&ops::inspect_outline(&cli.project, file)?, &cli)? emit(&ops::inspect_outline(&cli.project, file)?, &cli)?
} }
@@ -72,8 +77,10 @@ fn run() -> Result<()> {
FindCommand::Importers { file } => { FindCommand::Importers { file } => {
emit(&ops::find_importers(&cli.project, file)?, &cli)? emit(&ops::find_importers(&cli.project, file)?, &cli)?
} }
FindCommand::Dead => emit(&ops::find_dead(&cli.project)?, &cli)?, FindCommand::Dead => emit(&ops::find_dead(&cli.project, &excludes)?, &cli)?,
FindCommand::DeadSymbols => emit(&ops::find_dead_symbols(&cli.project)?, &cli)?, FindCommand::DeadSymbols => {
emit(&ops::find_dead_symbols(&cli.project, &excludes)?, &cli)?
}
FindCommand::Callers { target } => { FindCommand::Callers { target } => {
emit(&ops::find_callers(&cli.project, target)?, &cli)? emit(&ops::find_callers(&cli.project, target)?, &cli)?
} }
@@ -81,6 +88,7 @@ fn run() -> Result<()> {
emit(&ops::find_references(&cli.project, identifier)?, &cli)? emit(&ops::find_references(&cli.project, identifier)?, &cli)?
} }
}, },
Command::Detect { path } => emit(&ops::detect(path)?, &cli)?,
Command::Serve => mcp::serve(&cli.project)?, Command::Serve => mcp::serve(&cli.project)?,
Command::Stats => emit(&ops::stats(&cli.project)?, &cli)?, Command::Stats => emit(&ops::stats(&cli.project)?, &cli)?,
} }
+80 -7
View File
@@ -125,6 +125,7 @@ fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
} }
let opt_str = |k: &str| args.get(k).and_then(|v| v.as_str()); let opt_str = |k: &str| args.get(k).and_then(|v| v.as_str());
let opt_depth = |k: &str| args.get(k).and_then(|v| v.as_u64()).map(|d| d as usize); let opt_depth = |k: &str| args.get(k).and_then(|v| v.as_u64()).map(|d| d as usize);
let excludes = crate::config::ExcludeSet::new(&exclude_list(args));
match name { match name {
"index_create" => ops::index_create(Path::new(req_str(args, "path", name)?)), "index_create" => ops::index_create(Path::new(req_str(args, "path", name)?)),
@@ -140,6 +141,7 @@ fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.unwrap_or(200), .unwrap_or(200),
&excludes,
), ),
"search_text" => ops::search_text( "search_text" => ops::search_text(
project, project,
@@ -148,6 +150,7 @@ fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
.and_then(|v| v.as_u64()) .and_then(|v| v.as_u64())
.map(|n| n as usize) .map(|n| n as usize)
.unwrap_or(100), .unwrap_or(100),
&excludes,
), ),
"inspect_symbol" => { "inspect_symbol" => {
if let Some(id) = args.get("id").and_then(|v| v.as_i64()) { if let Some(id) = args.get("id").and_then(|v| v.as_i64()) {
@@ -156,14 +159,21 @@ fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
ops::inspect_symbol(project, req_str(args, "name", name)?) ops::inspect_symbol(project, req_str(args, "name", name)?)
} }
} }
"inspect_bundle" => {
if let Some(id) = args.get("id").and_then(|v| v.as_i64()) {
ops::inspect_bundle(project, &id.to_string())
} else {
ops::inspect_bundle(project, req_str(args, "name", name)?)
}
}
"inspect_outline" => ops::inspect_outline(project, req_str(args, "file", name)?), "inspect_outline" => ops::inspect_outline(project, req_str(args, "file", name)?),
"inspect_file" => { "inspect_file" => {
ops::inspect_file(project, req_str(args, "file", name)?, opt_str("lines")) ops::inspect_file(project, req_str(args, "file", name)?, opt_str("lines"))
} }
"inspect_tree" => ops::inspect_tree(project, opt_depth("depth")), "inspect_tree" => ops::inspect_tree(project, opt_depth("depth")),
"find_importers" => ops::find_importers(project, req_str(args, "file", name)?), "find_importers" => ops::find_importers(project, req_str(args, "file", name)?),
"find_dead" => ops::find_dead(project), "find_dead" => ops::find_dead(project, &excludes),
"find_dead_symbols" => ops::find_dead_symbols(project), "find_dead_symbols" => ops::find_dead_symbols(project, &excludes),
"find_callers" => { "find_callers" => {
if let Some(id) = args.get("id").and_then(|v| v.as_i64()) { if let Some(id) = args.get("id").and_then(|v| v.as_i64()) {
ops::find_callers(project, &id.to_string()) ops::find_callers(project, &id.to_string())
@@ -176,13 +186,37 @@ fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
"inspect_blast_radius" => { "inspect_blast_radius" => {
ops::inspect_blast_radius(project, req_str(args, "file", name)?, opt_depth("depth")) ops::inspect_blast_radius(project, req_str(args, "file", name)?, opt_depth("depth"))
} }
"codebase_detect" => {
let path = opt_str("path").map(Path::new).unwrap_or(project);
ops::detect(path)
}
"stats" => ops::stats(project), "stats" => ops::stats(project),
_ => anyhow::bail!("unknown tool: {name}"), _ => anyhow::bail!("unknown tool: {name}"),
} }
} }
/// Parse the optional `exclude` array argument into a list of globs.
fn exclude_list(args: &Value) -> Vec<String> {
args.get("exclude")
.and_then(|v| v.as_array())
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(String::from))
.collect()
})
.unwrap_or_default()
}
fn tools() -> Value { fn tools() -> Value {
json!([ json!([
tool(
"codebase_detect",
"Detect the languages and stack of a directory before indexing: scans file extensions and manifest files (Cargo.toml, package.json, pom.xml, etc.) without needing ctags or an index. Use first on an unknown project to decide whether and how to index it.",
json!({
"type": "object",
"properties": { "path": { "type": "string", "description": "Directory to probe (defaults to the served project)" } }
})
),
tool( tool(
"index_create", "index_create",
"Index a project from scratch: scan source files, extract symbols via ctags and imports, and write a SQLite index. Run before any other tool on a new project, or to rebuild a stale index.", "Index a project from scratch: scan source files, extract symbols via ctags and imports, and write a SQLite index. Run before any other tool on a new project, or to rebuild a stale index.",
@@ -221,7 +255,12 @@ fn tools() -> Value {
"type": "boolean", "type": "boolean",
"description": "Include each symbol's source inline (default false)" "description": "Include each symbol's source inline (default false)"
}, },
"limit": { "type": "integer", "description": "Maximum results (default 200)" } "limit": { "type": "integer", "description": "Maximum results (default 200)" },
"exclude": {
"type": "array",
"items": { "type": "string" },
"description": "Extra globs to exclude from results (adds to built-in defaults: vendor/, node_modules/, *.min.*, dist/, build/)"
}
}, },
"required": ["query"] "required": ["query"]
}) })
@@ -233,7 +272,12 @@ fn tools() -> Value {
"type": "object", "type": "object",
"properties": { "properties": {
"query": { "type": "string", "description": "Text or phrase to search for" }, "query": { "type": "string", "description": "Text or phrase to search for" },
"limit": { "type": "integer", "description": "Maximum matching files (default 100)" } "limit": { "type": "integer", "description": "Maximum matching files (default 100)" },
"exclude": {
"type": "array",
"items": { "type": "string" },
"description": "Extra globs to exclude from results (adds to built-in defaults: vendor/, node_modules/, *.min.*, dist/, build/)"
}
}, },
"required": ["query"] "required": ["query"]
}) })
@@ -249,6 +293,17 @@ fn tools() -> Value {
} }
}) })
), ),
tool(
"inspect_bundle",
"Get a symbol's full source, direct callers, and direct importers in a single call. Replaces three round-trips (inspect_symbol + find_callers + find_importers). Resolve by id or exact name.",
json!({
"type": "object",
"properties": {
"id": { "type": "integer", "description": "Symbol id from search_symbols (unambiguous)" },
"name": { "type": "string", "description": "Exact symbol name (alternative to id; may match multiple)" }
}
})
),
tool( tool(
"inspect_outline", "inspect_outline",
"Get the symbol outline of a file: names, kinds, signatures, and line ranges for every top-level symbol. Use to orient yourself in a file without reading it whole.", "Get the symbol outline of a file: names, kinds, signatures, and line ranges for every top-level symbol. Use to orient yourself in a file without reading it whole.",
@@ -290,12 +345,30 @@ fn tools() -> Value {
tool( tool(
"find_dead", "find_dead",
"List files with no importers (entry-point files like main/lib are excluded). Use to spot orphaned or unreferenced modules.", "List files with no importers (entry-point files like main/lib are excluded). Use to spot orphaned or unreferenced modules.",
json!({ "type": "object" }) json!({
"type": "object",
"properties": {
"exclude": {
"type": "array",
"items": { "type": "string" },
"description": "Extra globs to exclude from results (adds to built-in defaults)"
}
}
})
), ),
tool( tool(
"find_dead_symbols", "find_dead_symbols",
"List symbols never referenced by name outside their own definition. Heuristic: framework-wired symbols (routes, test functions) may appear here because they are invoked by convention, not by name. Use to find genuinely unused functions and methods.", "List symbols never referenced by name outside their own definition. Heuristic: framework-wired symbols (routes, test functions, Spring/JUnit beans) are excluded; CSS selectors and vendored/minified files are skipped. Use to find genuinely unused functions and methods.",
json!({ "type": "object" }) json!({
"type": "object",
"properties": {
"exclude": {
"type": "array",
"items": { "type": "string" },
"description": "Extra globs to exclude from results (adds to built-in defaults)"
}
}
})
), ),
tool( tool(
"find_callers", "find_callers",
+181 -9
View File
@@ -10,6 +10,7 @@ use serde::Serialize;
use serde_json::Value; use serde_json::Value;
use crate::config; use crate::config;
use crate::discovery;
use crate::graph::Graph; use crate::graph::Graph;
use crate::index::{pipeline, store}; use crate::index::{pipeline, store};
use crate::search; use crate::search;
@@ -34,6 +35,72 @@ pub fn index_file(project: &Path, file: &str) -> Result<Value> {
Ok(serde_json::to_value(stats)?) Ok(serde_json::to_value(stats)?)
} }
// ---- detect ----
/// Manifest files that identify a project's stack, independent of whether the
/// language is indexed. Used by `detect` before `index create`.
const MANIFESTS: &[(&str, &str)] = &[
("Cargo.toml", "rust"),
("package.json", "javascript"),
("pom.xml", "java"),
("build.gradle", "java"),
("build.gradle.kts", "java"),
("settings.gradle", "java"),
("requirements.txt", "python"),
("pyproject.toml", "python"),
("setup.py", "python"),
("Pipfile", "python"),
("go.mod", "go"),
("Gemfile", "ruby"),
("composer.json", "php"),
("CMakeLists.txt", "c"),
];
/// Lightweight pre-index onboarding probe: report the languages/stack present
/// at `path` from file extensions and manifest files only (no ctags, no
/// index). Lets an agent decide whether/how to index before paying for it.
pub fn detect(path: &Path) -> Result<Value> {
let files = discovery::discover(path)?;
let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
for f in &files {
*counts.entry(f.language).or_default() += 1;
}
let languages: Vec<Value> = counts
.iter()
.map(|(l, c)| serde_json::json!({"language": l, "files": c}))
.collect();
let manifests: Vec<Value> = detect_manifests(path)
.into_iter()
.map(|(file, stack)| serde_json::json!({"file": file, "stack": stack}))
.collect();
let root = path
.canonicalize()
.unwrap_or_else(|_| path.to_path_buf())
.to_string_lossy()
.into_owned();
Ok(serde_json::json!({
"root": root,
"languages": languages,
"manifests": manifests,
"total_files": files.len(),
"indexable": !files.is_empty(),
}))
}
fn detect_manifests(path: &Path) -> Vec<(String, &'static str)> {
let mut found = Vec::new();
let Ok(entries) = std::fs::read_dir(path) else {
return found;
};
for entry in entries.flatten() {
let name = entry.file_name().to_string_lossy().into_owned();
if let Some((_, stack)) = MANIFESTS.iter().find(|(n, _)| *n == name) {
found.push((name, stack));
}
}
found
}
// ---- search ---- // ---- search ----
pub fn search_symbols( pub fn search_symbols(
@@ -41,9 +108,10 @@ pub fn search_symbols(
query: &str, query: &str,
include_source: bool, include_source: bool,
limit: usize, limit: usize,
excludes: &config::ExcludeSet,
) -> Result<Value> { ) -> Result<Value> {
let conn = open_read(project)?; let conn = open_read(project)?;
let hits = search::search_symbols(&conn, query, limit)?; let hits = search::search_symbols(&conn, query, limit, Some(excludes))?;
if !include_source { if !include_source {
return Ok(serde_json::to_value(hits)?); return Ok(serde_json::to_value(hits)?);
} }
@@ -74,7 +142,12 @@ pub fn search_symbols(
/// immediately, consistent with `inspect file`/`inspect symbol`. `limit` caps /// immediately, consistent with `inspect file`/`inspect symbol`. `limit` caps
/// the number of files that contribute matches; within a file, identical /// the number of files that contribute matches; within a file, identical
/// snippets are collapsed to the first occurrence to reduce noise. /// snippets are collapsed to the first occurrence to reduce noise.
pub fn search_text(project: &Path, query: &str, limit: usize) -> Result<Value> { pub fn search_text(
project: &Path,
query: &str,
limit: usize,
excludes: &config::ExcludeSet,
) -> Result<Value> {
let conn = open_read(project)?; let conn = open_read(project)?;
let q = query.trim(); let q = query.trim();
if q.is_empty() { if q.is_empty() {
@@ -87,6 +160,9 @@ pub fn search_text(project: &Path, query: &str, limit: usize) -> Result<Value> {
if files_hit >= limit { if files_hit >= limit {
break; break;
} }
if excludes.matches(&f.path) {
continue;
}
let path = project.join(&f.path); let path = project.join(&f.path);
let Ok(content) = std::fs::read_to_string(&path) else { let Ok(content) = std::fs::read_to_string(&path) else {
continue; continue;
@@ -174,6 +250,50 @@ pub fn inspect_outline(project: &Path, file: &str) -> Result<Value> {
Ok(serde_json::to_value(symbols)?) Ok(serde_json::to_value(symbols)?)
} }
/// Symbol source + direct callers + direct importers in a single call. `target`
/// is a numeric id or an exact (case-insensitive) name; a name matching several
/// symbols returns one bundle per match (same shape as `inspect symbol`).
pub fn inspect_bundle(project: &Path, target: &str) -> Result<Value> {
let conn = open_read(project)?;
let syms = resolve_symbols(&conn, target)?;
let bundles: Vec<Value> = syms
.iter()
.map(|sym| bundle_object(project, &conn, sym))
.collect::<Result<_>>()?;
if target.trim().parse::<i64>().is_ok() {
Ok(bundles
.into_iter()
.next()
.expect("resolve_symbols is non-empty"))
} else {
Ok(serde_json::json!(bundles))
}
}
fn bundle_object(project: &Path, conn: &Connection, sym: &store::SymbolRow) -> Result<Value> {
let base = symbol_object(project, conn, sym)?;
let callers = callers_of(project, conn, std::slice::from_ref(sym))?;
let callers_count = callers.len();
let importers = store::importers_of(conn, sym.file_id)?
.into_iter()
.map(|id| store::file_by_id(conn, id))
.collect::<std::result::Result<Vec<_>, _>>()?
.into_iter()
.flatten()
.map(|f| f.path)
.collect::<Vec<_>>();
let importers_count = importers.len();
let mut obj = match base {
Value::Object(m) => m,
_ => serde_json::Map::new(),
};
obj.insert("callers".into(), serde_json::json!(callers));
obj.insert("callers_count".into(), serde_json::json!(callers_count));
obj.insert("importers".into(), serde_json::json!(importers));
obj.insert("importers_count".into(), serde_json::json!(importers_count));
Ok(Value::Object(obj))
}
pub fn inspect_file(project: &Path, file: &str, lines: Option<&str>) -> Result<Value> { pub fn inspect_file(project: &Path, file: &str, lines: Option<&str>) -> Result<Value> {
let conn = open_read(project)?; let conn = open_read(project)?;
let fid = file_id(&conn, file)?; let fid = file_id(&conn, file)?;
@@ -222,7 +342,7 @@ pub fn find_importers(project: &Path, file: &str) -> Result<Value> {
Ok(serde_json::to_value(paths)?) Ok(serde_json::to_value(paths)?)
} }
pub fn find_dead(project: &Path) -> Result<Value> { pub fn find_dead(project: &Path, excludes: &config::ExcludeSet) -> Result<Value> {
let conn = open_read(project)?; let conn = open_read(project)?;
let files = store::all_files(&conn)?; let files = store::all_files(&conn)?;
let ids: Vec<i64> = files.iter().map(|f| f.id).collect(); let ids: Vec<i64> = files.iter().map(|f| f.id).collect();
@@ -233,7 +353,7 @@ pub fn find_dead(project: &Path) -> Result<Value> {
.into_iter() .into_iter()
.filter_map(|id| { .filter_map(|id| {
let f = files.iter().find(|f| f.id == id)?; let f = files.iter().find(|f| f.id == id)?;
if is_entry_point(&f.path, &f.language) { if excludes.matches(&f.path) || is_entry_point(&f.path, &f.language) {
return None; return None;
} }
Some(serde_json::json!({"id": f.id, "path": f.path, "language": f.language})) Some(serde_json::json!({"id": f.id, "path": f.path, "language": f.language}))
@@ -250,14 +370,26 @@ pub fn find_dead(project: &Path) -> Result<Value> {
/// reference exists, so this errs toward under-reporting rather than false /// reference exists, so this errs toward under-reporting rather than false
/// positives. Includes `kind` so callers can filter out structural kinds /// positives. Includes `kind` so callers can filter out structural kinds
/// (fields, variants) whose names legitimately appear only via qualified paths. /// (fields, variants) whose names legitimately appear only via qualified paths.
pub fn find_dead_symbols(project: &Path) -> Result<Value> { pub fn find_dead_symbols(project: &Path, excludes: &config::ExcludeSet) -> Result<Value> {
let conn = open_read(project)?; let conn = open_read(project)?;
let symbols = store::all_symbols(&conn)?; let symbols = store::all_symbols(&conn)?;
let files = store::all_files(&conn)?; let files = store::all_files(&conn)?;
let path_of: HashMap<i64, &str> = files.iter().map(|f| (f.id, f.path.as_str())).collect(); let path_of: HashMap<i64, &str> = files.iter().map(|f| (f.id, f.path.as_str())).collect();
let lang_of: HashMap<i64, &str> = files.iter().map(|f| (f.id, f.language.as_str())).collect();
let mut by_name: BTreeMap<String, Vec<&store::SymbolRow>> = BTreeMap::new(); let mut by_name: BTreeMap<String, Vec<&store::SymbolRow>> = BTreeMap::new();
for s in &symbols { for s in &symbols {
let own = path_of.get(&s.file_id).copied().unwrap_or("");
// Vendored/generated assets are indexed for import edges but are noise
// for symbol queries; CSS selectors and structural kinds (Java
// `package`, HTML/Markdown `heading*`) are not callable symbols, so
// they can never be meaningfully "dead".
if excludes.matches(own)
|| lang_of.get(&s.file_id).copied() == Some("css")
|| is_structural_kind(&s.kind)
{
continue;
}
by_name.entry(s.name.to_lowercase()).or_default().push(s); by_name.entry(s.name.to_lowercase()).or_default().push(s);
} }
// Lowercased symbol names, used to bound the per-file word index to tokens // Lowercased symbol names, used to bound the per-file word index to tokens
@@ -330,6 +462,12 @@ fn index_words(project: &Path, rel: &str, names: &HashSet<&str>) -> HashMap<Stri
m m
} }
/// Structural/documentation kinds that are never "called" and therefore can't
/// be meaningfully reported as dead symbols.
fn is_structural_kind(kind: &str) -> bool {
kind == "package" || kind.starts_with("heading")
}
/// Decorator keywords that wire a symbol to an external framework. Such symbols /// Decorator keywords that wire a symbol to an external framework. Such symbols
/// are referenced by convention/framework, not by a name call-site, so a pure /// are referenced by convention/framework, not by a name call-site, so a pure
/// name-scan wrongly flags them dead. /// name-scan wrongly flags them dead.
@@ -349,13 +487,36 @@ const WIRED_DECORATORS: &[&str] = &[
"receiver", "receiver",
"signal", "signal",
"listener", "listener",
// Java/JUnit & Spring: `@Test`, `@Before(Each)`, plus DI/lifecycle
// annotations that wire a bean/component without a name call-site.
"test",
"before",
"service",
"component",
"bean",
"eventlistener",
"restcontroller",
"scheduled",
"configuration",
]; ];
/// A symbol is externally wired if it is a pytest `test_*` function or its /// A symbol is externally wired if it is a test by naming convention or its
/// definition is preceded by a framework decorator (Flask/FastAPI route, /// definition is preceded by a framework decorator (Flask/FastAPI route,
/// pytest fixture, Celery task, click/typer command, discord.py command, …). /// pytest fixture, Celery task, click/typer command, discord.py command,
/// JUnit/Spring annotation, …).
fn is_wired_entry_point(project: &Path, own: &str, s: &store::SymbolRow) -> bool { fn is_wired_entry_point(project: &Path, own: &str, s: &store::SymbolRow) -> bool {
if matches!(s.kind.as_str(), "function" | "method") && s.name.starts_with("test_") { if matches!(s.kind.as_str(), "function" | "method" | "member") && s.name.starts_with("test_") {
return true;
}
// Test classes by convention: pytest `TestHealth`/`test_health`, JUnit
// `UserServiceTest` (and friends). The runner discovers them, not a name
// call-site, so a pure name-scan wrongly flags them dead.
if s.kind == "class"
&& (s.name.starts_with("Test")
|| s.name.starts_with("test_")
|| s.name.ends_with("Test")
|| s.name.ends_with("Tests"))
{
return true; return true;
} }
let Ok(lines) = read_all_lines(project, own) else { let Ok(lines) = read_all_lines(project, own) else {
@@ -850,7 +1011,18 @@ fn raw_to_nodes(node: &RawNode, base: &str, depth: Option<usize>) -> Vec<TreeNod
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::{code_only_line, is_entry_point}; use super::{code_only_line, is_entry_point, is_structural_kind};
#[test]
fn structural_kinds_are_never_dead() {
assert!(is_structural_kind("package"));
assert!(is_structural_kind("heading1"));
assert!(is_structural_kind("heading3"));
assert!(!is_structural_kind("function"));
assert!(!is_structural_kind("method"));
assert!(!is_structural_kind("class"));
assert!(!is_structural_kind("member"));
}
#[test] #[test]
fn entry_point_is_language_scoped() { fn entry_point_is_language_scoped() {
+36 -5
View File
@@ -6,6 +6,8 @@ use anyhow::Result;
use rusqlite::{Connection, params}; use rusqlite::{Connection, params};
use serde::Serialize; use serde::Serialize;
use crate::config::ExcludeSet;
#[derive(Debug, Serialize)] #[derive(Debug, Serialize)]
pub struct SymbolHit { pub struct SymbolHit {
pub id: i64, pub id: i64,
@@ -30,13 +32,26 @@ pub struct TextHit {
const SYMBOL_COLS: &str = const SYMBOL_COLS: &str =
"s.id, s.name, s.qualified_name, s.kind, s.signature, f.path, s.line_start, s.line_end"; "s.id, s.name, s.qualified_name, s.kind, s.signature, f.path, s.line_start, s.line_end";
pub fn search_symbols(conn: &Connection, query: &str, limit: usize) -> Result<Vec<SymbolHit>> { pub fn search_symbols(
conn: &Connection,
query: &str,
limit: usize,
excludes: Option<&ExcludeSet>,
) -> Result<Vec<SymbolHit>> {
let q = query.trim(); let q = query.trim();
let mut seen: HashSet<i64> = HashSet::new(); let mut seen: HashSet<i64> = HashSet::new();
let mut out: Vec<SymbolHit> = Vec::new(); let mut out: Vec<SymbolHit> = Vec::new();
if q.is_empty() { if q.is_empty() {
return Ok(out); return Ok(out);
} }
// Fetch a superset when excluding so filtering doesn't starve the output;
// the final cap is still `limit`.
let fetch_limit = if excludes.is_some() {
limit.saturating_mul(8).max(200)
} else {
limit
} as i64;
let excluded = |h: &SymbolHit| excludes.is_some_and(|e| e.matches(&h.file));
// 1) exact name (case-insensitive) // 1) exact name (case-insensitive)
{ {
@@ -44,7 +59,11 @@ pub fn search_symbols(conn: &Connection, query: &str, limit: usize) -> Result<Ve
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \ "SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
WHERE s.name = ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2" WHERE s.name = ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2"
))?; ))?;
let rows = stmt.query_map(params![q, limit as i64], |r| row_to_hit(r, "exact"))?; let rows = stmt.query_map(params![q, fetch_limit], |r| row_to_hit(r, "exact"))?;
let rows = rows.filter_map(|r| match r {
Ok(h) if excluded(&h) => None,
other => Some(other),
});
push_hits(&mut out, &mut seen, rows, limit)?; push_hits(&mut out, &mut seen, rows, limit)?;
} }
@@ -55,7 +74,11 @@ pub fn search_symbols(conn: &Connection, query: &str, limit: usize) -> Result<Ve
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \ "SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2" WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2"
))?; ))?;
let rows = stmt.query_map(params![pat, limit as i64], |r| row_to_hit(r, "prefix"))?; let rows = stmt.query_map(params![pat, fetch_limit], |r| row_to_hit(r, "prefix"))?;
let rows = rows.filter_map(|r| match r {
Ok(h) if excluded(&h) => None,
other => Some(other),
});
push_hits(&mut out, &mut seen, rows, limit)?; push_hits(&mut out, &mut seen, rows, limit)?;
} }
@@ -68,7 +91,11 @@ pub fn search_symbols(conn: &Connection, query: &str, limit: usize) -> Result<Ve
JOIN files f ON f.id = s.file_id \ JOIN files f ON f.id = s.file_id \
WHERE symbols_fts MATCH ?1 ORDER BY rank LIMIT ?2" WHERE symbols_fts MATCH ?1 ORDER BY rank LIMIT ?2"
))?; ))?;
let rows = stmt.query_map(params![phrase, limit as i64], |r| row_to_hit(r, "fulltext"))?; let rows = stmt.query_map(params![phrase, fetch_limit], |r| row_to_hit(r, "fulltext"))?;
let rows = rows.filter_map(|r| match r {
Ok(h) if excluded(&h) => None,
other => Some(other),
});
push_hits(&mut out, &mut seen, rows, limit)?; push_hits(&mut out, &mut seen, rows, limit)?;
} }
@@ -79,7 +106,11 @@ pub fn search_symbols(conn: &Connection, query: &str, limit: usize) -> Result<Ve
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \ "SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2" WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT ?2"
))?; ))?;
let rows = stmt.query_map(params![pat, limit as i64], |r| row_to_hit(r, "substring"))?; let rows = stmt.query_map(params![pat, fetch_limit], |r| row_to_hit(r, "substring"))?;
let rows = rows.filter_map(|r| match r {
Ok(h) if excluded(&h) => None,
other => Some(other),
});
push_hits(&mut out, &mut seen, rows, limit)?; push_hits(&mut out, &mut seen, rows, limit)?;
} }