Initial commit: rustytokens — native codebase indexer + MCP server

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
endergebnisandCopilot committed 2026-09-04 02:43:00 +02:00
commit 354132e887
43 files changed
+3783

No files matched your search

+107
View File
@@ -0,0 +1,107 @@
//! clap definitions for all subcommands.
use std::path::PathBuf;
use clap::{Parser, Subcommand};
#[derive(Parser)]
#[command(
name = "rustytokens",
version,
about = "Native codebase indexer + MCP server"
)]
pub struct Cli {
/// Project root for index-backed commands (defaults to current directory)
#[arg(short, long, global = true, default_value = ".")]
pub project: PathBuf,
/// Output format: json or table
#[arg(long, global = true, default_value = "json")]
pub format: String,
/// Compact JSON output
#[arg(long, global = true)]
pub compact: bool,
#[command(subcommand)]
pub command: Command,
}
#[derive(Subcommand)]
pub enum Command {
/// Build or refresh the index
Index {
#[command(subcommand)]
sub: IndexCommand,
},
/// Search symbols or file contents
Search {
#[command(subcommand)]
sub: SearchCommand,
},
/// Inspect symbols, files, outlines, the tree, or cycles
Inspect {
#[command(subcommand)]
sub: InspectCommand,
},
/// Find importers or dead code
Find {
#[command(subcommand)]
sub: FindCommand,
},
/// Start the MCP server over stdio
Serve,
/// Show index statistics
Stats,
}
#[derive(Subcommand)]
pub enum IndexCommand {
/// Index a project from scratch
Create { path: PathBuf },
/// Re-index only changed files
Update { path: PathBuf },
}
#[derive(Subcommand)]
pub enum SearchCommand {
/// Search symbols by name
Symbols { query: String },
/// Full-text search over file contents
Text { query: String },
}
#[derive(Subcommand)]
pub enum InspectCommand {
/// Source code of a single symbol by id
Symbol { id: i64 },
/// Symbol outline of a file
Outline { file: String },
/// Read a file or a line range (e.g. --lines 10-40)
File {
file: String,
#[arg(long)]
lines: Option<String>,
},
/// Indexed file tree
Tree {
#[arg(long)]
depth: Option<usize>,
},
/// Files transitively affected by a change
BlastRadius {
file: String,
#[arg(long)]
depth: Option<usize>,
},
/// Find circular imports
Cycles,
}
#[derive(Subcommand)]
pub enum FindCommand {
/// Who imports this file
Importers { file: String },
/// Files with no importers
Dead,
}
+81
View File
@@ -0,0 +1,81 @@
//! Project configuration: paths, cache dir, ignore patterns, language mapping.
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::path::{Path, PathBuf};
/// Directory names skipped during discovery (own list).
pub const SKIP_DIRS: &[&str] = &[
".git",
".hg",
".svn",
"node_modules",
"target",
"vendor",
"venv",
".venv",
"__pycache__",
"dist",
"build",
".idea",
".vscode",
".mypy_cache",
".pytest_cache",
".next",
".nuxt",
"bower_components",
"coverage",
"out",
];
/// Map a file extension to a language label, or `None` if unsupported.
pub fn language_for(path: &Path) -> Option<&'static str> {
let ext = path.extension()?.to_str()?;
Some(match ext {
"rs" => "rust",
"java" => "java",
"py" => "python",
"c" | "h" => "c",
"cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => "cpp",
"ts" | "tsx" | "mts" | "cts" => "typescript",
"js" | "jsx" | "mjs" | "cjs" => "javascript",
"html" | "htm" => "html",
"css" => "css",
_ => return None,
})
}
/// Base cache directory for index databases.
pub fn cache_dir() -> PathBuf {
let base = std::env::var("XDG_CACHE_HOME")
.ok()
.filter(|s| !s.is_empty())
.map(PathBuf::from)
.or_else(|| {
std::env::var("HOME")
.ok()
.map(|h| PathBuf::from(h).join(".cache"))
})
.unwrap_or_else(|| PathBuf::from("."));
base.join("rustytokens")
}
/// Deterministic short key for a project path (used to name the cache DB).
fn project_key(path: &str) -> String {
let mut h = DefaultHasher::new();
path.hash(&mut h);
format!("{:016x}", h.finish())
}
/// Path of the SQLite index DB for a project root.
pub fn index_db_path(project: &Path) -> PathBuf {
let canon = project
.canonicalize()
.unwrap_or_else(|_| project.to_path_buf());
let key = project_key(&canon.to_string_lossy());
let name = project
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "project".to_string());
cache_dir().join(format!("{name}-{key}.db"))
}
+65
View File
@@ -0,0 +1,65 @@
//! Thin subprocess wrapper around universal-ctags (no linking, own structs).
use std::io::Write;
use std::path::Path;
use std::process::{Command, Stdio};
use anyhow::{Context, Result, anyhow};
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize)]
pub struct Tag {
pub name: String,
pub path: String,
#[serde(default)]
pub line: Option<u64>,
#[serde(default)]
pub end: Option<u64>,
#[serde(default)]
pub kind: Option<String>,
#[serde(default)]
pub scope: Option<String>,
#[serde(default)]
pub signature: Option<String>,
}
/// Run ctags over `files`, returning parsed JSON tags.
pub fn run(files: &[&Path]) -> Result<Vec<Tag>> {
if files.is_empty() {
return Ok(Vec::new());
}
let mut child = Command::new("ctags")
.args(["--output-format=json", "--fields=+nKeSt", "-L", "-"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.context("failed to start ctags (is universal-ctags installed?)")?;
{
let stdin = child
.stdin
.as_mut()
.ok_or_else(|| anyhow!("ctags stdin unavailable"))?;
for f in files {
writeln!(stdin, "{}", f.to_string_lossy())?;
}
}
let output = child.wait_with_output().context("ctags failed")?;
if !output.status.success() {
let err = String::from_utf8_lossy(&output.stderr);
return Err(anyhow!("ctags exited with {}: {}", output.status, err));
}
let mut tags = Vec::new();
for line in output.stdout.split(|b| *b == b'\n') {
if line.is_empty() {
continue;
}
if let Ok(t) = serde_json::from_slice::<Tag>(line) {
tags.push(t);
}
}
Ok(tags)
}
+59
View File
@@ -0,0 +1,59 @@
//! Filesystem walk with smart filtering, producing the list of source files to index.
use std::path::{Path, PathBuf};
use anyhow::Result;
use crate::config::{SKIP_DIRS, language_for};
#[derive(Debug, Clone)]
pub struct SourceFile {
/// Absolute path on disk.
pub path: PathBuf,
/// Path relative to the project root (forward slashes).
pub rel: String,
pub language: &'static str,
}
pub fn discover(project: &Path) -> Result<Vec<SourceFile>> {
let project_abs = project
.canonicalize()
.unwrap_or_else(|_| project.to_path_buf());
let mut builder = ignore::WalkBuilder::new(&project_abs);
builder.standard_filters(true);
builder.require_git(false);
builder.filter_entry(|entry| {
if entry.file_type().is_some_and(|t| t.is_dir()) {
let name = entry.file_name().to_string_lossy();
!SKIP_DIRS.iter().any(|d| *d == name)
} else {
true
}
});
let mut out = Vec::new();
for entry in builder.build() {
let entry = match entry {
Ok(e) => e,
Err(_) => continue,
};
let path = entry.path();
if entry.file_type().is_some_and(|t| t.is_dir()) {
continue;
}
let Some(language) = language_for(path) else {
continue;
};
let rel = path
.strip_prefix(&project_abs)
.map(|p| p.to_string_lossy().replace('\\', "/"))
.unwrap_or_else(|_| path.to_string_lossy().replace('\\', "/"));
out.push(SourceFile {
path: path.to_path_buf(),
rel,
language,
});
}
out.sort_by(|a, b| a.rel.cmp(&b.rel));
Ok(out)
}
+177
View File
@@ -0,0 +1,177 @@
//! Import graph: cycles (Tarjan SCC), dead files, blast radius (BFS).
use std::collections::{HashMap, VecDeque};
use crate::index::store::ImportRow;
pub struct Graph {
adj: HashMap<i64, Vec<i64>>, // from_file_id -> to_file_id
rev: HashMap<i64, Vec<i64>>, // to_file_id -> from_file_id
}
impl Graph {
pub fn from_imports(imports: &[ImportRow]) -> Self {
let mut adj: HashMap<i64, Vec<i64>> = HashMap::new();
let mut rev: HashMap<i64, Vec<i64>> = HashMap::new();
for imp in imports {
adj.entry(imp.from_file_id)
.or_default()
.push(imp.to_file_id);
rev.entry(imp.to_file_id)
.or_default()
.push(imp.from_file_id);
}
Graph { adj, rev }
}
/// Strongly-connected components with more than one node (or a self-loop).
pub fn cycles(&self) -> Vec<Vec<i64>> {
let mut sccs = tarjan_scc(&self.adj);
sccs.retain(|scc| {
scc.len() > 1
|| (scc.len() == 1
&& self
.adj
.get(&scc[0])
.map(|v| v.contains(&scc[0]))
.unwrap_or(false))
});
for scc in &mut sccs {
scc.sort_unstable();
}
sccs
}
/// Files with no incoming import edges (nobody imports them).
pub fn dead_nodes(&self, all_file_ids: &[i64]) -> Vec<i64> {
let mut dead: Vec<i64> = all_file_ids
.iter()
.copied()
.filter(|id| self.rev.get(id).map(|v| v.is_empty()).unwrap_or(true))
.collect();
dead.sort_unstable();
dead
}
/// BFS over the reverse graph: which files (transitively) import `start`.
pub fn blast_radius(&self, start: i64, max_depth: usize) -> Vec<(i64, usize)> {
let mut out = Vec::new();
let mut seen = HashMap::new();
seen.insert(start, 0usize);
let mut queue = VecDeque::from([(start, 0usize)]);
while let Some((node, depth)) = queue.pop_front() {
if depth >= max_depth {
continue;
}
if let Some(parents) = self.rev.get(&node) {
for &p in parents {
if let std::collections::hash_map::Entry::Vacant(e) = seen.entry(p) {
e.insert(depth + 1);
queue.push_back((p, depth + 1));
}
}
}
}
for (id, d) in seen {
if id != start {
out.push((id, d));
}
}
out.sort_by_key(|(_, d)| *d);
out
}
}
/// Iterative Tarjan SCC. Returns strongly-connected components.
fn tarjan_scc(adj: &HashMap<i64, Vec<i64>>) -> Vec<Vec<i64>> {
let mut index_counter = 0usize;
let mut indices: HashMap<i64, usize> = HashMap::new();
let mut lowlinks: HashMap<i64, usize> = HashMap::new();
let mut stack: Vec<i64> = Vec::new();
let mut on_stack: HashMap<i64, bool> = HashMap::new();
let mut sccs: Vec<Vec<i64>> = Vec::new();
let nodes: Vec<i64> = {
let mut set = std::collections::HashSet::new();
for (k, v) in adj {
set.insert(*k);
for t in v {
set.insert(*t);
}
}
let mut v: Vec<i64> = set.into_iter().collect();
v.sort_unstable();
v
};
#[allow(clippy::too_many_arguments)]
fn strongconnect(
v: i64,
adj: &HashMap<i64, Vec<i64>>,
index_counter: &mut usize,
indices: &mut HashMap<i64, usize>,
lowlinks: &mut HashMap<i64, usize>,
stack: &mut Vec<i64>,
on_stack: &mut HashMap<i64, bool>,
sccs: &mut Vec<Vec<i64>>,
) {
indices.insert(v, *index_counter);
lowlinks.insert(v, *index_counter);
*index_counter += 1;
stack.push(v);
on_stack.insert(v, true);
if let Some(neighbors) = adj.get(&v) {
for &w in neighbors {
if !indices.contains_key(&w) {
strongconnect(
w,
adj,
index_counter,
indices,
lowlinks,
stack,
on_stack,
sccs,
);
let lw = lowlinks[&w];
let lv = lowlinks[&v];
lowlinks.insert(v, lv.min(lw));
} else if on_stack.get(&w).copied().unwrap_or(false) {
let iw = indices[&w];
let lv = lowlinks[&v];
lowlinks.insert(v, lv.min(iw));
}
}
}
if lowlinks[&v] == indices[&v] {
let mut component = Vec::new();
loop {
let w = stack.pop().unwrap();
on_stack.insert(w, false);
component.push(w);
if w == v {
break;
}
}
sccs.push(component);
}
}
for &n in &nodes {
if !indices.contains_key(&n) {
strongconnect(
n,
adj,
&mut index_counter,
&mut indices,
&mut lowlinks,
&mut stack,
&mut on_stack,
&mut sccs,
);
}
}
sccs
}
+23
View File
@@ -0,0 +1,23 @@
//! Content hashing for change detection.
use std::fs::File;
use std::io::Read;
use std::path::Path;
use anyhow::Result;
pub fn hash_file(path: &Path) -> Result<String> {
let mut f = File::open(path)?;
let mut buf = Vec::new();
f.read_to_end(&mut buf)?;
Ok(format!("{:016x}", xxhash_rust::xxh3::xxh3_64(&buf)))
}
pub fn mtime_secs(path: &Path) -> i64 {
std::fs::metadata(path)
.and_then(|m| m.modified())
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
+398
View File
@@ -0,0 +1,398 @@
//! Import extraction (per-language) + best-effort resolution to file ids.
use std::collections::HashMap;
use std::sync::OnceLock;
use regex::Regex;
use crate::discovery::SourceFile;
#[derive(Debug, Clone)]
pub struct Import {
pub target: String,
pub import_type: String,
/// Directory levels a relative import ascends (Python `from ..x`).
pub levels: usize,
}
/// Extract import targets from a source file (own heuristic parsers).
pub fn extract(f: &SourceFile) -> Vec<Import> {
let content = std::fs::read_to_string(&f.path).unwrap_or_default();
match f.language {
"python" => extract_python(&content),
"rust" => extract_rust(&content),
"typescript" | "javascript" => extract_ts(&content),
"c" | "cpp" => extract_c(&content),
"java" => extract_java(&content),
"html" => extract_html(&content),
_ => Vec::new(),
}
}
fn extract_python(content: &str) -> Vec<Import> {
let mut out = Vec::new();
for line in content.lines() {
let l = line.trim();
if let Some(rest) = l.strip_prefix("import ") {
for part in rest.split(',') {
let p = part.trim();
if !p.is_empty() {
out.push(Import {
target: p.to_string(),
import_type: "import".into(),
levels: 0,
});
}
}
} else if let Some(rest) = l.strip_prefix("from ") {
// `from [..]<mod> import names` — leading dots = parent levels.
let module = rest.split_whitespace().next().unwrap_or("");
let dots = module.bytes().take_while(|b| *b == b'.').count();
let name = &module[dots..];
let levels = dots.saturating_sub(1);
if name.is_empty() {
// `from . import x` -> `x` lives `levels` dirs up.
let after = rest.split_once("import ").map(|(_, s)| s).unwrap_or("");
for n in after.split(',') {
let n = n.trim().trim_end_matches('\\').trim();
if !n.is_empty() {
out.push(Import {
target: n.to_string(),
import_type: "from".into(),
levels,
});
}
}
} else {
out.push(Import {
target: name.to_string(),
import_type: "from".into(),
levels,
});
}
}
}
out
}
fn extract_rust(content: &str) -> Vec<Import> {
let mut out = Vec::new();
for line in content.lines() {
let l = line.trim();
if let Some(rest) = l.strip_prefix("use ") {
let rest = rest
.trim_end_matches(';')
.split('{')
.next()
.unwrap_or(rest)
.trim();
let path = rest
.trim_start_matches("crate::")
.trim_start_matches("self::")
.trim_start_matches("super::")
.trim_start_matches("::");
let mut segs: Vec<&str> = path.split("::").filter(|s| !s.is_empty()).collect();
if segs.is_empty() || matches!(segs[0], "std" | "core" | "alloc") {
continue;
}
// A trailing CamelCase segment is a type/symbol, not a module path.
if let Some(last) = segs.last()
&& last.chars().next().is_some_and(char::is_uppercase)
{
segs.pop();
}
if let Some(first) = segs.first() {
out.push(Import {
target: first.to_string(),
import_type: "use".into(),
levels: 0,
});
}
if segs.len() > 1 {
out.push(Import {
target: segs.join("/"),
import_type: "use".into(),
levels: 0,
});
}
} else {
let mod_name = strip_mod_decl(l);
if let Some(name) = mod_name {
out.push(Import {
target: name,
import_type: "mod".into(),
levels: 0,
});
}
}
}
out
}
/// Extract the module name from a `mod` declaration, handling visibility
/// modifiers such as `pub`, `pub(crate)`, `pub(super)`, and attributes.
fn strip_mod_decl(l: &str) -> Option<String> {
let mut rest = l;
// Strip leading attributes like `#[cfg(...)]`.
while rest.starts_with("#[") {
let end = rest.find("]\n").or_else(|| rest.find(']'))? + 1;
rest = rest[end..].trim_start();
}
if let Some(r) = rest.strip_prefix("pub ") {
rest = r;
} else if let Some(r) = rest.strip_prefix("pub(") {
let end = r.find(") ")? + 2;
rest = r[end..].trim_start();
}
let name = rest
.strip_prefix("mod ")?
.trim()
.trim_end_matches(';')
.trim();
if name.is_empty() || name.contains(' ') {
None
} else {
Some(name.to_string())
}
}
fn ts_import_re() -> &'static Regex {
static RE: OnceLock<Regex> = OnceLock::new();
RE.get_or_init(|| {
Regex::new(r#"(?:import|export)\s+(?:[^'"]*?from\s+)?['"]([^'"]+)['"]"#).unwrap()
})
}
fn ts_require_re() -> &'static Regex {
static RE: OnceLock<Regex> = OnceLock::new();
RE.get_or_init(|| Regex::new(r#"require\(\s*['"]([^'"]+)['"]\s*\)"#).unwrap())
}
fn extract_ts(content: &str) -> Vec<Import> {
let mut out = Vec::new();
for cap in ts_import_re().captures_iter(content) {
let target = cap[1].to_string();
let t = target.trim_start_matches("./");
out.push(Import {
target: t.to_string(),
import_type: "import".into(),
levels: 0,
});
}
for cap in ts_require_re().captures_iter(content) {
let target = cap[1].to_string();
let t = target.trim_start_matches("./");
out.push(Import {
target: t.to_string(),
import_type: "require".into(),
levels: 0,
});
}
out
}
fn extract_c(content: &str) -> Vec<Import> {
let mut out = Vec::new();
for line in content.lines() {
let l = line.trim();
if let Some(rest) = l.strip_prefix("#include") {
let inner = rest
.trim()
.trim_start_matches('"')
.trim_start_matches('<')
.split(['"', '>'])
.next()
.unwrap_or("");
if !inner.is_empty() {
out.push(Import {
target: inner.to_string(),
import_type: "include".into(),
levels: 0,
});
}
}
}
out
}
fn extract_java(content: &str) -> Vec<Import> {
let mut out = Vec::new();
for line in content.lines() {
let l = line.trim();
if let Some(rest) = l.strip_prefix("import ") {
let rest = rest.trim_end_matches(';').trim_start_matches("static ");
let rest = rest.split_whitespace().next().unwrap_or(rest);
if !rest.is_empty() {
out.push(Import {
target: rest.to_string(),
import_type: "import".into(),
levels: 0,
});
}
}
}
out
}
fn html_attr_re() -> &'static Regex {
static RE: OnceLock<Regex> = OnceLock::new();
RE.get_or_init(|| Regex::new(r#"(?:src|href)\s*=\s*["']([^"']+)["']"#).unwrap())
}
/// `<script src="...">` / `<link href="...">` as relative imports. ctags' HTML
/// parser is weak, so symbol coverage here is best-effort.
fn extract_html(content: &str) -> Vec<Import> {
html_attr_re()
.captures_iter(content)
.filter_map(|cap| {
let target = cap[1].trim();
// Skip absolute URLs, data URIs, and fragments.
if target.starts_with("http://")
|| target.starts_with("https://")
|| target.starts_with("//")
|| target.starts_with("data:")
|| target.starts_with('#')
{
return None;
}
let t = target.trim_start_matches("./");
if t.is_empty() {
return None;
}
Some(Import {
target: t.to_string(),
import_type: "asset".into(),
levels: 0,
})
})
.collect()
}
/// Best-effort mapping of an import target string to a file id.
pub struct Resolver {
by_path: HashMap<String, i64>,
by_name: HashMap<String, i64>,
by_stem: HashMap<String, i64>,
}
/// Normalize an import target string (strip prefixes like `./`, `crate::`).
fn clean(target: &str) -> Option<String> {
let t = target.trim();
if t.is_empty() {
return None;
}
let t = t.trim_start_matches("./");
let t = t
.trim_start_matches("crate::")
.trim_start_matches("self::")
.trim_start_matches("super::")
.trim_start_matches("::");
if t.is_empty() {
return None;
}
Some(t.to_string())
}
impl Resolver {
pub fn build(files: Vec<(String, i64)>) -> Self {
let mut by_path = HashMap::new();
let mut by_name = HashMap::new();
let mut by_stem = HashMap::new();
for (rel, id) in files {
by_path.insert(rel.clone(), id);
let name = rel.rsplit('/').next().unwrap_or(&rel).to_string();
by_name.entry(name.clone()).or_insert(id);
let stem = name
.rsplit_once('.')
.map(|(s, _)| s.to_string())
.unwrap_or(name);
by_stem.entry(stem).or_insert(id);
}
Resolver {
by_path,
by_name,
by_stem,
}
}
/// Resolve an import relative to the importing file's directory first,
/// ascending `levels` parent dirs for relative imports.
pub fn resolve_with_context(&self, target: &str, dir: &str, levels: usize) -> Option<i64> {
let t = clean(target)?;
let mut base = dir;
for _ in 0..levels {
base = base.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
}
let base = if base.is_empty() {
String::new()
} else {
format!("{base}/")
};
self.resolve_relative(&base, &t)
.or_else(|| self.resolve_clean(&t))
}
fn resolve_clean(&self, t: &str) -> Option<i64> {
if let Some(id) = self.by_path.get(t) {
return Some(*id);
}
let slashed = t.replace('.', "/");
if slashed != t {
if let Some(id) = self.by_path.get(&slashed) {
return Some(*id);
}
if let Some(id) = self.try_module(&slashed) {
return Some(id);
}
}
if let Some(id) = self.try_module(t) {
return Some(id);
}
if let Some(id) = self.by_stem.get(t) {
return Some(*id);
}
let last = t.rsplit('/').next().unwrap_or(t);
if let Some(id) = self.by_stem.get(last) {
return Some(*id);
}
// Java dotted packages (`demo.Util`) — fall back to the final segment.
let last_dot = last.rsplit('.').next().unwrap_or(last);
if last_dot != last
&& let Some(id) = self.by_stem.get(last_dot)
{
return Some(*id);
}
self.by_name.get(last).copied()
}
fn resolve_relative(&self, base: &str, t: &str) -> Option<i64> {
if let Some(id) = self.by_path.get(&format!("{base}{t}")) {
return Some(*id);
}
for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
if let Some(id) = self.by_path.get(&format!("{base}{t}.{ext}")) {
return Some(*id);
}
}
for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
if let Some(id) = self.by_path.get(&format!("{base}{t}/{entry}")) {
return Some(*id);
}
}
None
}
fn try_module(&self, base: &str) -> Option<i64> {
for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
if let Some(id) = self.by_path.get(&format!("{base}.{ext}")) {
return Some(*id);
}
}
for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
if let Some(id) = self.by_path.get(&format!("{base}/{entry}")) {
return Some(*id);
}
}
None
}
}
+5
View File
@@ -0,0 +1,5 @@
pub mod hashing;
pub mod imports;
pub mod pipeline;
pub mod schema;
pub mod store;
+209
View File
@@ -0,0 +1,209 @@
//! Index pipeline: hash, run ctags, extract imports, write to SQLite in one transaction.
use std::collections::HashMap;
use std::path::Path;
use anyhow::{Context, Result};
use rayon::prelude::*;
use rusqlite::params;
use crate::ctags;
use crate::discovery::{self, SourceFile};
use crate::index::{hashing, imports, store};
#[derive(Debug, serde::Serialize)]
pub struct IndexStats {
pub files: i64,
pub symbols: i64,
pub imports: i64,
}
/// (Re)index a project. `recreate` drops any existing index first.
pub fn run(project: &Path, db_path: &Path, recreate: bool) -> Result<IndexStats> {
if recreate && db_path.exists() {
std::fs::remove_file(db_path).context("remove old index")?;
}
let conn = store::open(db_path)?;
let files = discovery::discover(project)?;
let existing: HashMap<String, (i64, String)> = store::all_files(&conn)?
.into_iter()
.map(|f| (f.path.clone(), (f.id, f.content_hash.clone())))
.collect();
// Hash all files in parallel.
let hashed: Vec<(String, String, i64, &'static str)> = files
.par_iter()
.map(|f| {
let h = hashing::hash_file(&f.path).unwrap_or_else(|_| "0000000000000000".to_string());
let m = hashing::mtime_secs(&f.path);
(f.rel.clone(), h, m, f.language)
})
.collect();
let by_rel: HashMap<&str, &SourceFile> = files.iter().map(|f| (f.rel.as_str(), f)).collect();
let abs_to_rel: HashMap<String, String> = files
.iter()
.map(|f| (f.path.to_string_lossy().into_owned(), f.rel.clone()))
.collect();
let changed: Vec<&SourceFile> = hashed
.iter()
.filter(|(rel, h, _, _)| existing.get(rel).map(|(_, oh)| oh != h).unwrap_or(true))
.filter_map(|(rel, ..)| by_rel.get(rel.as_str()).copied())
.collect();
let deleted_ids: Vec<i64> = existing
.iter()
.filter(|(rel, _)| !by_rel.contains_key(rel.as_str()))
.map(|(_, (id, _))| *id)
.collect();
conn.execute_batch("BEGIN;")?;
let mut file_ids: HashMap<String, i64> = HashMap::new();
for (rel, h, m, lang) in &hashed {
let id = store::upsert_file(&conn, rel, h, lang, *m)?;
file_ids.insert(rel.clone(), id);
}
for id in &deleted_ids {
store::delete_file(&conn, *id)?;
conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [id])?;
}
// Run ctags once over changed files.
let changed_paths: Vec<&Path> = changed.iter().map(|f| f.path.as_path()).collect();
let tags = ctags::run(&changed_paths)?;
let mut tags_by_rel: HashMap<String, Vec<ctags::Tag>> = HashMap::new();
for t in tags {
if let Some(rel) = abs_to_rel.get(&t.path) {
tags_by_rel.entry(rel.clone()).or_default().push(t);
}
}
let resolver =
imports::Resolver::build(file_ids.iter().map(|(k, v)| (k.clone(), *v)).collect());
for f in &changed {
let fid = file_ids[&f.rel];
let content = std::fs::read_to_string(&f.path).unwrap_or_default();
let tags = tags_by_rel.get(&f.rel).cloned().unwrap_or_default();
store::replace_symbols(&conn, fid, &tags_to_symbols(&tags, &content))?;
store::clear_imports_for_file(&conn, fid)?;
let dir = f
.rel
.rsplit_once('/')
.map(|(d, _)| d)
.unwrap_or("")
.to_string();
for imp in imports::extract(f) {
if let Some(to) = resolver.resolve_with_context(&imp.target, &dir, imp.levels)
&& to != fid
{
store::insert_import(&conn, fid, to, &imp.import_type)?;
}
}
conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [fid])?;
conn.execute(
"INSERT INTO content_fts(rowid, path, content) VALUES (?1, ?2, ?3)",
params![fid, f.rel, content],
)?;
}
conn.execute_batch("COMMIT;")?;
let (files_n, symbols_n, imports_n) = store::stats(&conn)?;
Ok(IndexStats {
files: files_n,
symbols: symbols_n,
imports: imports_n,
})
}
fn tags_to_symbols(tags: &[ctags::Tag], content: &str) -> Vec<store::NewSymbol> {
let lines: Vec<&str> = content.lines().collect();
tags.iter()
.filter_map(|t| {
let line = t.line? as i64;
let end = t.end.map(|e| e as i64).unwrap_or(line);
let kind = t.kind.clone().unwrap_or_default();
let end = extend_block_end(&lines, line, end, &kind);
let name = t.name.clone();
let qualified_name = match &t.scope {
Some(s) if !name.starts_with(&format!("{s}.")) => format!("{s}.{name}"),
_ => name.clone(),
};
Some(store::NewSymbol {
name,
qualified_name,
kind,
signature: t.signature.clone(),
line_start: line,
line_end: end,
})
})
.collect()
}
/// ctags omits a body span for some kinds (e.g. Rust `fn`), leaving `end ==
/// start` even for a multi-line block. Extend brace-based blocks to their
/// closing brace so stored spans are correct for every reader.
fn extend_block_end(lines: &[&str], start: i64, end: i64, kind: &str) -> i64 {
if !is_block_kind(kind) || end > start {
return end;
}
// Semicolon-terminated declarations (`mod foo;`, `struct Foo;`) have no body.
if let Some(line) = lines.get((start - 1) as usize)
&& line.trim_end().ends_with(';')
{
return end;
}
let mut depth = 0i64;
let mut opened = false;
let mut out = end;
// ponytail: naive char brace count; braces in string/comment literals skew
// depth. Swap for a tokenizer only if real spans come back wrong.
for (i, line) in lines.iter().enumerate() {
let n = (i as i64) + 1;
if n < start {
continue;
}
for c in line.chars() {
match c {
'{' => {
depth += 1;
opened = true;
}
'}' => depth -= 1,
_ => {}
}
}
if opened {
out = n;
}
if opened && depth <= 0 {
break;
}
}
out
}
fn is_block_kind(kind: &str) -> bool {
matches!(
kind,
"function"
| "method"
| "struct"
| "class"
| "interface"
| "enum"
| "trait"
| "impl"
| "implementation"
| "module"
| "namespace"
| "union"
)
}
+71
View File
@@ -0,0 +1,71 @@
//! SQLite schema + minimal migration (schema_version in metadata).
use anyhow::Result;
use rusqlite::Connection;
const SCHEMA_VERSION: i64 = 1;
const SCHEMA: &str = r#"
CREATE TABLE files (
id INTEGER PRIMARY KEY,
path TEXT NOT NULL UNIQUE,
content_hash TEXT NOT NULL,
language TEXT NOT NULL,
mtime INTEGER NOT NULL
);
CREATE TABLE symbols (
id INTEGER PRIMARY KEY,
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
name TEXT NOT NULL,
qualified_name TEXT NOT NULL,
kind TEXT NOT NULL,
signature TEXT,
line_start INTEGER NOT NULL,
line_end INTEGER NOT NULL
);
CREATE INDEX idx_symbols_file ON symbols(file_id);
CREATE INDEX idx_symbols_name ON symbols(name);
CREATE TABLE imports (
id INTEGER PRIMARY KEY,
from_file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
to_file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
import_type TEXT NOT NULL
);
CREATE INDEX idx_imports_from ON imports(from_file_id);
CREATE INDEX idx_imports_to ON imports(to_file_id);
CREATE VIRTUAL TABLE symbols_fts USING fts5(
name, qualified_name, kind, signature,
content='symbols', content_rowid='id'
);
CREATE TRIGGER symbols_ai AFTER INSERT ON symbols BEGIN
INSERT INTO symbols_fts(rowid, name, qualified_name, kind, signature)
VALUES (new.id, new.name, new.qualified_name, new.kind, new.signature);
END;
CREATE TRIGGER symbols_ad AFTER DELETE ON symbols BEGIN
INSERT INTO symbols_fts(symbols_fts, rowid, name, qualified_name, kind, signature)
VALUES ('delete', old.id, old.name, old.qualified_name, old.kind, old.signature);
END;
CREATE VIRTUAL TABLE content_fts USING fts5(path UNINDEXED, content);
"#;
/// Create schema if missing, or migrate to the current version.
pub fn init(conn: &mut Connection) -> Result<()> {
conn.execute_batch(
"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);",
)?;
let version: i64 = conn
.query_row(
"SELECT COALESCE(CAST(value AS INTEGER), 0) FROM metadata WHERE key='schema_version'",
[],
|r| r.get(0),
)
.unwrap_or(0);
if version < SCHEMA_VERSION {
conn.execute_batch(SCHEMA)?;
conn.execute(
"INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?1)",
[SCHEMA_VERSION.to_string()],
)?;
}
Ok(())
}
+311
View File
@@ -0,0 +1,311 @@
//! CRUD on files/symbols/imports.
use std::path::Path;
use anyhow::Result;
use rusqlite::{Connection, OptionalExtension, params};
use crate::index::schema;
#[derive(Debug, Clone)]
pub struct FileRow {
pub id: i64,
pub path: String,
pub content_hash: String,
pub language: String,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct SymbolRow {
pub id: i64,
pub file_id: i64,
pub name: String,
pub qualified_name: String,
pub kind: String,
pub signature: Option<String>,
pub line_start: i64,
pub line_end: i64,
}
#[derive(Debug, Clone)]
pub struct NewSymbol {
pub name: String,
pub qualified_name: String,
pub kind: String,
pub signature: Option<String>,
pub line_start: i64,
pub line_end: i64,
}
#[derive(Debug, Clone)]
pub struct ImportRow {
pub from_file_id: i64,
pub to_file_id: i64,
}
/// Open (creating if needed) the index DB and apply schema.
pub fn open(path: &Path) -> Result<Connection> {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
let mut conn = Connection::open(path)?;
conn.execute_batch(
"PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL;",
)?;
schema::init(&mut conn)?;
Ok(conn)
}
pub fn all_files(conn: &Connection) -> Result<Vec<FileRow>> {
let mut stmt = conn.prepare("SELECT id, path, content_hash, language FROM files")?;
let rows = stmt.query_map([], |r| {
Ok(FileRow {
id: r.get(0)?,
path: r.get(1)?,
content_hash: r.get(2)?,
language: r.get(3)?,
})
})?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
pub fn upsert_file(
conn: &Connection,
path: &str,
hash: &str,
language: &str,
mtime: i64,
) -> Result<i64> {
conn.execute(
"INSERT INTO files (path, content_hash, language, mtime) VALUES (?1, ?2, ?3, ?4)
ON CONFLICT(path) DO UPDATE SET content_hash=?2, language=?3, mtime=?4",
params![path, hash, language, mtime],
)?;
// last_insert_rowid() is unreliable after an upsert that updates; query it.
Ok(conn.query_row("SELECT id FROM files WHERE path = ?1", [path], |r| r.get(0))?)
}
pub fn delete_file(conn: &Connection, id: i64) -> Result<()> {
conn.execute("DELETE FROM files WHERE id = ?1", [id])?;
Ok(())
}
pub fn file_by_id(conn: &Connection, id: i64) -> Result<Option<FileRow>> {
conn.query_row(
"SELECT id, path, content_hash, language FROM files WHERE id = ?1",
[id],
|r| {
Ok(FileRow {
id: r.get(0)?,
path: r.get(1)?,
content_hash: r.get(2)?,
language: r.get(3)?,
})
},
)
.optional()
.map_err(Into::into)
}
pub fn file_id_by_path(conn: &Connection, path: &str) -> Result<Option<i64>> {
Ok(conn
.query_row("SELECT id FROM files WHERE path = ?1", [path], |r| r.get(0))
.optional()?)
}
pub fn replace_symbols(conn: &Connection, file_id: i64, symbols: &[NewSymbol]) -> Result<()> {
conn.execute("DELETE FROM symbols WHERE file_id = ?1", [file_id])?;
let mut stmt = conn.prepare(
"INSERT INTO symbols (file_id, name, qualified_name, kind, signature, line_start, line_end)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
)?;
for s in symbols {
stmt.execute(params![
file_id,
s.name,
s.qualified_name,
s.kind,
s.signature,
s.line_start,
s.line_end
])?;
}
Ok(())
}
pub fn clear_imports_for_file(conn: &Connection, file_id: i64) -> Result<()> {
conn.execute("DELETE FROM imports WHERE from_file_id = ?1", [file_id])?;
Ok(())
}
pub fn insert_import(conn: &Connection, from: i64, to: i64, import_type: &str) -> Result<()> {
conn.execute(
"INSERT INTO imports (from_file_id, to_file_id, import_type) VALUES (?1, ?2, ?3)",
params![from, to, import_type],
)?;
Ok(())
}
pub fn all_imports(conn: &Connection) -> Result<Vec<ImportRow>> {
let mut stmt = conn.prepare("SELECT from_file_id, to_file_id FROM imports")?;
let rows = stmt.query_map([], |r| {
Ok(ImportRow {
from_file_id: r.get(0)?,
to_file_id: r.get(1)?,
})
})?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
pub fn symbol_by_id(conn: &Connection, id: i64) -> Result<Option<SymbolRow>> {
conn.query_row(
"SELECT id, file_id, name, qualified_name, kind, signature, line_start, line_end
FROM symbols WHERE id = ?1",
[id],
|r| {
Ok(SymbolRow {
id: r.get(0)?,
file_id: r.get(1)?,
name: r.get(2)?,
qualified_name: r.get(3)?,
kind: r.get(4)?,
signature: r.get(5)?,
line_start: r.get(6)?,
line_end: r.get(7)?,
})
},
)
.optional()
.map_err(Into::into)
}
pub fn symbols_for_file(conn: &Connection, file_id: i64) -> Result<Vec<SymbolRow>> {
let mut stmt = conn.prepare(
"SELECT id, file_id, name, qualified_name, kind, signature, line_start, line_end
FROM symbols WHERE file_id = ?1 ORDER BY line_start",
)?;
let rows = stmt.query_map([file_id], |r| {
Ok(SymbolRow {
id: r.get(0)?,
file_id: r.get(1)?,
name: r.get(2)?,
qualified_name: r.get(3)?,
kind: r.get(4)?,
signature: r.get(5)?,
line_start: r.get(6)?,
line_end: r.get(7)?,
})
})?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
pub fn importers_of(conn: &Connection, file_id: i64) -> Result<Vec<i64>> {
let mut stmt = conn.prepare("SELECT from_file_id FROM imports WHERE to_file_id = ?1")?;
let rows = stmt.query_map([file_id], |r| r.get(0))?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
/// Returns (files, symbols, imports) counts.
pub fn stats(conn: &Connection) -> Result<(i64, i64, i64)> {
let files: i64 = conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))?;
let symbols: i64 = conn.query_row("SELECT COUNT(*) FROM symbols", [], |r| r.get(0))?;
let imports: i64 = conn.query_row("SELECT COUNT(*) FROM imports", [], |r| r.get(0))?;
Ok((files, symbols, imports))
}
pub fn language_counts(conn: &Connection) -> Result<Vec<(String, i64)>> {
let mut stmt =
conn.prepare("SELECT language, COUNT(*) FROM files GROUP BY language ORDER BY 2 DESC")?;
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
#[cfg(test)]
mod tests {
use super::*;
fn mem() -> Connection {
let mut conn = Connection::open_in_memory().unwrap();
conn.execute_batch("PRAGMA foreign_keys=ON;").unwrap();
schema::init(&mut conn).unwrap();
conn
}
#[test]
fn file_insert_query_and_upsert() {
let conn = mem();
let id1 = upsert_file(&conn, "src/lib.rs", "hash1", "rust", 100).unwrap();
let id2 = upsert_file(&conn, "src/lib.rs", "hash2", "rust", 200).unwrap();
assert_eq!(id1, id2, "upsert reuses the same row");
let row = file_by_id(&conn, id1).unwrap().unwrap();
assert_eq!(row.content_hash, "hash2");
assert_eq!(row.language, "rust");
assert_eq!(all_files(&conn).unwrap().len(), 1);
}
#[test]
fn symbols_replace_and_query() {
let conn = mem();
let fid = upsert_file(&conn, "src/lib.rs", "h", "rust", 0).unwrap();
replace_symbols(
&conn,
fid,
&[
NewSymbol {
name: "add".into(),
qualified_name: "add".into(),
kind: "function".into(),
signature: Some("(a: i32, b: i32) -> i32".into()),
line_start: 3,
line_end: 5,
},
NewSymbol {
name: "AuthService".into(),
qualified_name: "AuthService".into(),
kind: "struct".into(),
signature: None,
line_start: 7,
line_end: 7,
},
],
)
.unwrap();
let syms = symbols_for_file(&conn, fid).unwrap();
assert_eq!(syms.len(), 2);
assert_eq!(syms[0].name, "add");
assert_eq!(
symbol_by_id(&conn, syms[1].id).unwrap().unwrap().kind,
"struct"
);
// Replacing drops old symbols and re-inserts.
replace_symbols(
&conn,
fid,
&[NewSymbol {
name: "only".into(),
qualified_name: "only".into(),
kind: "function".into(),
signature: None,
line_start: 1,
line_end: 1,
}],
)
.unwrap();
assert_eq!(symbols_for_file(&conn, fid).unwrap().len(), 1);
}
#[test]
fn imports_roundtrip_and_importers() {
let conn = mem();
let a = upsert_file(&conn, "a.ts", "h", "typescript", 0).unwrap();
let b = upsert_file(&conn, "b.ts", "h", "typescript", 0).unwrap();
insert_import(&conn, a, b, "import").unwrap();
let imports = all_imports(&conn).unwrap();
assert_eq!(imports.len(), 1);
assert_eq!(imports[0].from_file_id, a);
assert_eq!(importers_of(&conn, b).unwrap(), vec![a]);
assert_eq!(stats(&conn).unwrap(), (2, 0, 1));
}
}
+81
View File
@@ -0,0 +1,81 @@
mod cli;
mod config;
mod ctags;
mod discovery;
mod graph;
mod index;
mod mcp;
mod ops;
mod output;
mod search;
use anyhow::Result;
use clap::Parser;
use serde_json::Value;
use cli::{Cli, Command, FindCommand, IndexCommand, InspectCommand, SearchCommand};
fn main() {
if let Err(e) = run() {
if let Some(io) = e.downcast_ref::<std::io::Error>()
&& io.kind() == std::io::ErrorKind::BrokenPipe
{
std::process::exit(0);
}
eprintln!("error: {e:#}");
std::process::exit(1);
}
}
fn run() -> Result<()> {
let cli = Cli::parse();
match &cli.command {
Command::Index { sub } => match sub {
IndexCommand::Create { path } => emit(&ops::index_create(path)?, &cli)?,
IndexCommand::Update { path } => emit(&ops::index_update(path)?, &cli)?,
},
Command::Search { sub } => match sub {
SearchCommand::Symbols { query } => {
emit(&ops::search_symbols(&cli.project, query)?, &cli)?
}
SearchCommand::Text { query } => emit(&ops::search_text(&cli.project, query)?, &cli)?,
},
Command::Inspect { sub } => match sub {
InspectCommand::Symbol { id } => emit(&ops::inspect_symbol(&cli.project, *id)?, &cli)?,
InspectCommand::Outline { file } => {
emit(&ops::inspect_outline(&cli.project, file)?, &cli)?
}
InspectCommand::File { file, lines } => emit(
&ops::inspect_file(&cli.project, file, lines.as_deref())?,
&cli,
)?,
InspectCommand::Tree { depth } => {
emit(&ops::inspect_tree(&cli.project, *depth)?, &cli)?
}
InspectCommand::BlastRadius { file, depth } => emit(
&ops::inspect_blast_radius(&cli.project, file, *depth)?,
&cli,
)?,
InspectCommand::Cycles => emit(&ops::inspect_cycles(&cli.project)?, &cli)?,
},
Command::Find { sub } => match sub {
FindCommand::Importers { file } => {
emit(&ops::find_importers(&cli.project, file)?, &cli)?
}
FindCommand::Dead => emit(&ops::find_dead(&cli.project)?, &cli)?,
},
Command::Serve => mcp::serve(&cli.project)?,
Command::Stats => emit(&ops::stats(&cli.project)?, &cli)?,
}
Ok(())
}
fn emit(value: &Value, cli: &Cli) -> Result<()> {
match cli.format.as_str() {
"table" => {
output::print_table(value);
Ok(())
}
_ => output::print_json(value, cli.compact),
}
}
+254
View File
@@ -0,0 +1,254 @@
//! Minimal MCP server: JSON-RPC over stdio, dispatching to the same ops as the CLI.
use std::io::{BufRead, Write};
use std::path::Path;
use anyhow::Result;
use serde_json::{Value, json};
const PROTOCOL_VERSION: &str = "2024-11-05";
pub fn serve(project: &Path) -> Result<()> {
eprintln!("rustytokens MCP server listening on stdio");
let stdin = std::io::stdin();
let stdout = std::io::stdout();
for line in stdin.lock().lines() {
let line = match line {
Ok(l) => l,
Err(_) => break,
};
if line.trim().is_empty() {
continue;
}
let msg: Value = match serde_json::from_str(&line) {
Ok(m) => m,
Err(_) => continue,
};
if let Some(resp) = handle(&msg, project) {
let mut out = stdout.lock();
serde_json::to_writer(&mut out, &resp)?;
out.write_all(b"\n")?;
out.flush()?;
}
}
Ok(())
}
fn handle(msg: &Value, project: &Path) -> Option<Value> {
let method = msg.get("method")?.as_str()?;
let id = msg.get("id").cloned();
// Notifications have no id -> no response.
let id = id.as_ref()?;
match method {
"initialize" => Some(json!({
"jsonrpc": "2.0",
"id": id,
"result": {
"protocolVersion": PROTOCOL_VERSION,
"capabilities": { "tools": {} },
"serverInfo": { "name": "rustytokens", "version": env!("CARGO_PKG_VERSION") }
}
})),
"ping" => Some(json!({ "jsonrpc": "2.0", "id": id, "result": {} })),
"tools/list" => Some(json!({
"jsonrpc": "2.0",
"id": id,
"result": { "tools": tools() }
})),
"tools/call" => {
let params = msg.get("params").cloned().unwrap_or_else(|| json!({}));
let name = params.get("name").and_then(|n| n.as_str()).unwrap_or("");
let args = params
.get("arguments")
.cloned()
.unwrap_or_else(|| json!({}));
Some(json!({
"jsonrpc": "2.0",
"id": id,
"result": call_tool(name, &args, project)
}))
}
_ => Some(json!({
"jsonrpc": "2.0",
"id": id,
"error": { "code": -32601, "message": format!("method not found: {method}") }
})),
}
}
fn call_tool(name: &str, args: &Value, project: &Path) -> Value {
let result: Result<Value> = dispatch(name, args, project);
match result {
Ok(v) => json!({
"content": [{ "type": "text", "text": serde_json::to_string(&v).unwrap_or_default() }]
}),
Err(e) => json!({
"content": [{ "type": "text", "text": format!("error: {e:#}") }],
"isError": true
}),
}
}
fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
use crate::ops;
let s = |k: &str| args.get(k).and_then(|v| v.as_str()).unwrap_or("");
match name {
"index_create" => ops::index_create(Path::new(s("path"))),
"index_update" => ops::index_update(Path::new(s("path"))),
"search_symbols" => ops::search_symbols(project, s("query")),
"search_text" => ops::search_text(project, s("query")),
"inspect_symbol" => ops::inspect_symbol(
project,
args.get("id").and_then(|v| v.as_i64()).unwrap_or(0),
),
"inspect_outline" => ops::inspect_outline(project, s("file")),
"inspect_file" => ops::inspect_file(
project,
s("file"),
args.get("lines").and_then(|v| v.as_str()),
),
"inspect_tree" => ops::inspect_tree(
project,
args.get("depth")
.and_then(|v| v.as_u64())
.map(|d| d as usize),
),
"find_importers" => ops::find_importers(project, s("file")),
"find_dead" => ops::find_dead(project),
"inspect_cycles" => ops::inspect_cycles(project),
"inspect_blast_radius" => ops::inspect_blast_radius(
project,
s("file"),
args.get("depth")
.and_then(|v| v.as_u64())
.map(|d| d as usize),
),
"stats" => ops::stats(project),
_ => anyhow::bail!("unknown tool: {name}"),
}
}
fn tools() -> Value {
json!([
tool(
"index_create",
"Index a project from scratch",
json!({
"type": "object",
"properties": { "path": { "type": "string", "description": "Project directory to index" } },
"required": ["path"]
})
),
tool(
"index_update",
"Re-index only changed files",
json!({
"type": "object",
"properties": { "path": { "type": "string" } },
"required": ["path"]
})
),
tool(
"search_symbols",
"Search symbols by name",
json!({
"type": "object",
"properties": { "query": { "type": "string" } },
"required": ["query"]
})
),
tool(
"search_text",
"Full-text search over file contents",
json!({
"type": "object",
"properties": { "query": { "type": "string" } },
"required": ["query"]
})
),
tool(
"inspect_symbol",
"Get the source code of a single symbol by id",
json!({
"type": "object",
"properties": { "id": { "type": "integer" } },
"required": ["id"]
})
),
tool(
"inspect_outline",
"Get the symbol outline of a file",
json!({
"type": "object",
"properties": { "file": { "type": "string" } },
"required": ["file"]
})
),
tool(
"inspect_file",
"Read a file or a line range",
json!({
"type": "object",
"properties": {
"file": { "type": "string" },
"lines": { "type": "string", "description": "Optional range, e.g. 10-40" }
},
"required": ["file"]
})
),
tool(
"inspect_tree",
"Show the indexed file tree",
json!({
"type": "object",
"properties": { "depth": { "type": "integer" } }
})
),
tool(
"find_importers",
"List files that import the given file",
json!({
"type": "object",
"properties": { "file": { "type": "string" } },
"required": ["file"]
})
),
tool(
"find_dead",
"List files with no importers",
json!({ "type": "object" })
),
tool(
"inspect_cycles",
"Find circular imports",
json!({ "type": "object" })
),
tool(
"inspect_blast_radius",
"Files transitively affected by a change",
json!({
"type": "object",
"properties": {
"file": { "type": "string" },
"depth": { "type": "integer" }
},
"required": ["file"]
})
),
tool(
"stats",
"Show index statistics",
json!({ "type": "object" })
),
])
}
fn tool(name: &str, description: &str, input_schema: Value) -> Value {
json!({
"name": name,
"description": description,
"inputSchema": input_schema
})
}
+341
View File
@@ -0,0 +1,341 @@
//! High-level operations shared by the CLI and the MCP server.
use std::collections::BTreeMap;
use std::path::Path;
use anyhow::{Context, Result, bail};
use rusqlite::Connection;
use serde::Serialize;
use serde_json::Value;
use crate::config;
use crate::graph::Graph;
use crate::index::{pipeline, store};
use crate::search;
// ---- index ----
pub fn index_create(project: &Path) -> Result<Value> {
let db = config::index_db_path(project);
let stats = pipeline::run(project, &db, true)?;
Ok(serde_json::to_value(stats)?)
}
pub fn index_update(project: &Path) -> Result<Value> {
let db = config::index_db_path(project);
let stats = pipeline::run(project, &db, false)?;
Ok(serde_json::to_value(stats)?)
}
// ---- search ----
pub fn search_symbols(project: &Path, query: &str) -> Result<Value> {
let conn = open_read(project)?;
Ok(serde_json::to_value(search::search_symbols(&conn, query)?)?)
}
pub fn search_text(project: &Path, query: &str) -> Result<Value> {
let conn = open_read(project)?;
Ok(serde_json::to_value(search::search_text(&conn, query)?)?)
}
// ---- inspect ----
pub fn inspect_symbol(project: &Path, id: i64) -> Result<Value> {
let conn = open_read(project)?;
let sym = store::symbol_by_id(&conn, id)?.with_context(|| format!("no symbol with id {id}"))?;
let file = store::file_by_id(&conn, sym.file_id)?
.with_context(|| format!("no file with id {}", sym.file_id))?;
let source = read_lines(project, &file.path, sym.line_start, sym.line_end)?;
Ok(serde_json::json!({
"id": sym.id,
"name": sym.name,
"qualified_name": sym.qualified_name,
"kind": sym.kind,
"signature": sym.signature,
"file": file.path,
"line_start": sym.line_start,
"line_end": sym.line_end,
"source": source,
}))
}
pub fn inspect_outline(project: &Path, file: &str) -> Result<Value> {
let conn = open_read(project)?;
let fid = file_id(&conn, file)?;
let symbols = store::symbols_for_file(&conn, fid)?;
Ok(serde_json::to_value(symbols)?)
}
pub fn inspect_file(project: &Path, file: &str, lines: Option<&str>) -> Result<Value> {
let conn = open_read(project)?;
let fid = file_id(&conn, file)?;
let frow = store::file_by_id(&conn, fid)?.unwrap();
let (start, end) = parse_range(lines);
let all = read_all_lines(project, &frow.path)?;
let total = all.len() as i64;
let (s, e) = (start.unwrap_or(1), end.unwrap_or(total));
let slice = all
.iter()
.enumerate()
.filter(|(i, _)| {
let n = (*i as i64) + 1;
n >= s && n <= e
})
.map(|(_, l)| l.clone())
.collect::<Vec<_>>()
.join("\n");
Ok(serde_json::json!({
"file": frow.path,
"language": frow.language,
"line_start": s,
"line_end": e.min(total),
"total_lines": total,
"content": slice,
}))
}
pub fn inspect_tree(project: &Path, depth: Option<usize>) -> Result<Value> {
let conn = open_read(project)?;
let files = store::all_files(&conn)?;
let paths: Vec<String> = files.into_iter().map(|f| f.path).collect();
let root_name = project
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "project".to_string());
let tree = build_tree(&root_name, &paths, depth);
Ok(serde_json::to_value(tree)?)
}
// ---- find / graph ----
pub fn find_importers(project: &Path, file: &str) -> Result<Value> {
let conn = open_read(project)?;
let fid = file_id(&conn, file)?;
let importers = store::importers_of(&conn, fid)?;
let paths = importers
.into_iter()
.map(|id| store::file_by_id(&conn, id).map(|f| f.map(|f| f.path)))
.collect::<std::result::Result<Vec<_>, _>>()?
.into_iter()
.flatten()
.collect::<Vec<_>>();
Ok(serde_json::to_value(paths)?)
}
pub fn find_dead(project: &Path) -> Result<Value> {
let conn = open_read(project)?;
let files = store::all_files(&conn)?;
let ids: Vec<i64> = files.iter().map(|f| f.id).collect();
let imports = store::all_imports(&conn)?;
let graph = Graph::from_imports(&imports);
let dead = graph.dead_nodes(&ids);
let out: Vec<Value> = dead
.into_iter()
.filter_map(|id| {
let f = files.iter().find(|f| f.id == id)?;
if is_entry_point(&f.path, &f.language) {
return None;
}
Some(serde_json::json!({"id": f.id, "path": f.path, "language": f.language}))
})
.collect();
Ok(serde_json::json!({"count": out.len(), "files": out}))
}
/// Entry-point files are legitimately never imported by anything else.
fn is_entry_point(path: &str, language: &str) -> bool {
let name = path.rsplit('/').next().unwrap_or(path);
let stem = name.split('.').next().unwrap_or(name);
if matches!(stem, "main" | "lib") {
return true;
}
// Cargo compiles every file in these dirs as its own binary/crate.
if language == "rust" {
let top = path.split('/').next().unwrap_or(path);
if matches!(top, "tests" | "benches" | "examples") || path.starts_with("src/bin/") {
return true;
}
}
false
}
pub fn inspect_cycles(project: &Path) -> Result<Value> {
let conn = open_read(project)?;
let files = store::all_files(&conn)?;
let imports = store::all_imports(&conn)?;
let graph = Graph::from_imports(&imports);
let cycles = graph.cycles();
let path_of = |id: i64| {
files
.iter()
.find(|f| f.id == id)
.map(|f| f.path.clone())
.unwrap_or_else(|| format!("<id {id}>"))
};
let out: Vec<Vec<String>> = cycles
.into_iter()
.map(|c| c.into_iter().map(&path_of).collect())
.collect();
Ok(serde_json::json!({"count": out.len(), "cycles": out}))
}
pub fn inspect_blast_radius(project: &Path, file: &str, depth: Option<usize>) -> Result<Value> {
let conn = open_read(project)?;
let fid = file_id(&conn, file)?;
let files = store::all_files(&conn)?;
let imports = store::all_imports(&conn)?;
let graph = Graph::from_imports(&imports);
let max_depth = depth.unwrap_or(3);
let hits = graph.blast_radius(fid, max_depth);
let out: Vec<Value> = hits
.into_iter()
.filter_map(|(id, d)| {
files
.iter()
.find(|f| f.id == id)
.map(|f| serde_json::json!({"id": f.id, "path": f.path, "depth": d}))
})
.collect();
Ok(serde_json::json!({"file": file, "count": out.len(), "affected": out}))
}
// ---- stats ----
pub fn stats(project: &Path) -> Result<Value> {
let conn = open_read(project)?;
let (files, symbols, imports) = store::stats(&conn)?;
let langs = store::language_counts(&conn)?;
Ok(serde_json::json!({
"files": files,
"symbols": symbols,
"imports": imports,
"languages": langs,
}))
}
// ---- helpers ----
fn open_read(project: &Path) -> Result<Connection> {
let db = config::index_db_path(project);
if !db.exists() {
bail!(
"no index found for {}. Run `rustytokens index create <path>` first.",
project.display()
);
}
store::open(&db)
}
fn file_id(conn: &Connection, path: &str) -> Result<i64> {
store::file_id_by_path(conn, path)?.with_context(|| format!("file not indexed: {path}"))
}
fn read_lines(project: &Path, rel: &str, start: i64, end: i64) -> Result<String> {
let all = read_all_lines(project, rel)?;
Ok(all
.iter()
.enumerate()
.filter(|(i, _)| {
let n = (*i as i64) + 1;
n >= start && n <= end
})
.map(|(_, l)| l.clone())
.collect::<Vec<_>>()
.join("\n"))
}
fn read_all_lines(project: &Path, rel: &str) -> Result<Vec<String>> {
let path = project.join(rel);
let content = std::fs::read_to_string(&path)
.with_context(|| format!("cannot read {}", path.display()))?;
Ok(content.lines().map(|l| l.to_string()).collect())
}
fn parse_range(lines: Option<&str>) -> (Option<i64>, Option<i64>) {
let Some(s) = lines else { return (None, None) };
let parts: Vec<&str> = s.split('-').collect();
let start = parts.first().and_then(|p| p.trim().parse::<i64>().ok());
let end = parts.get(1).and_then(|p| p.trim().parse::<i64>().ok());
(start, end)
}
// ---- tree ----
#[derive(Debug, Serialize)]
struct TreeNode {
name: String,
path: String,
#[serde(rename = "type")]
kind: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
children: Vec<TreeNode>,
}
#[derive(Default)]
struct RawNode {
dirs: BTreeMap<String, RawNode>,
files: BTreeMap<String, String>,
}
fn build_tree(root_name: &str, paths: &[String], depth: Option<usize>) -> TreeNode {
let mut root = RawNode::default();
for p in paths {
let segs: Vec<&str> = p.split('/').collect();
insert_raw(&mut root, &segs, p);
}
let children = raw_to_nodes(&root, "", depth);
TreeNode {
name: root_name.to_string(),
path: String::new(),
kind: "dir".to_string(),
children,
}
}
fn insert_raw(node: &mut RawNode, segs: &[&str], full: &str) {
match segs {
[head] => {
node.files
.entry((*head).to_string())
.or_insert_with(|| full.to_string());
}
[head, rest @ ..] => {
let child = node.dirs.entry((*head).to_string()).or_default();
insert_raw(child, rest, full);
}
[] => {}
}
}
fn raw_to_nodes(node: &RawNode, base: &str, depth: Option<usize>) -> Vec<TreeNode> {
let mut out = Vec::new();
for (name, full) in &node.files {
out.push(TreeNode {
name: name.clone(),
path: full.clone(),
kind: "file".to_string(),
children: Vec::new(),
});
}
for (name, child) in &node.dirs {
let path = if base.is_empty() {
name.clone()
} else {
format!("{base}/{name}")
};
let expand = depth.map(|d| d > 0).unwrap_or(true);
let children = if expand {
raw_to_nodes(child, &path, depth.map(|d| d.saturating_sub(1)))
} else {
Vec::new()
};
out.push(TreeNode {
name: name.clone(),
path,
kind: "dir".to_string(),
children,
});
}
out
}
+100
View File
@@ -0,0 +1,100 @@
//! Output formatting: JSON (default) or a minimal table.
use anyhow::Result;
use serde::Serialize;
use serde_json::Value;
use std::io::Write;
pub fn print_json<T: Serialize>(value: &T, compact: bool) -> Result<()> {
let mut buf = Vec::new();
if compact {
serde_json::to_writer(&mut buf, value)?;
} else {
serde_json::to_writer_pretty(&mut buf, value)?;
}
let mut out = std::io::stdout().lock();
out.write_all(&buf)?;
out.write_all(b"\n")?;
Ok(())
}
fn putline(s: &str) {
use std::io::Write;
let _ = writeln!(std::io::stdout().lock(), "{s}");
}
/// Minimal table renderer for array-of-objects and plain objects.
pub fn print_table(value: &Value) {
match value {
Value::Array(items) if !items.is_empty() && items.iter().all(|i| i.is_object()) => {
let cols = collect_columns(items);
let mut rows = vec![cols.clone()];
for item in items {
let obj = item.as_object().unwrap();
rows.push(cols.iter().map(|c| cell_string(obj.get(c))).collect());
}
print_grid(&rows);
}
Value::Object(obj) => {
let mut rows = vec![vec!["key".to_string(), "value".to_string()]];
for (k, v) in obj {
rows.push(vec![k.clone(), cell_string(Some(v))]);
}
print_grid(&rows);
}
Value::Array(items) => {
for i in items {
putline(&cell_string(Some(i)));
}
}
other => putline(&cell_string(Some(other))),
}
}
fn collect_columns(items: &[Value]) -> Vec<String> {
let mut cols: Vec<String> = Vec::new();
for item in items {
if let Some(obj) = item.as_object() {
for k in obj.keys() {
if !cols.contains(k) {
cols.push(k.clone());
}
}
}
}
cols
}
fn cell_string(v: Option<&Value>) -> String {
match v {
None | Some(Value::Null) => String::new(),
Some(Value::String(s)) => s.clone(),
Some(other) => serde_json::to_string(other).unwrap_or_default(),
}
}
fn print_grid(rows: &[Vec<String>]) {
let n = rows.iter().map(|r| r.len()).max().unwrap_or(0);
let mut widths = vec![0usize; n];
for r in rows {
for (i, c) in r.iter().enumerate() {
widths[i] = widths[i].max(c.chars().count());
}
}
for (ri, r) in rows.iter().enumerate() {
let mut line = String::new();
for (i, w) in widths.iter().enumerate() {
let c = r.get(i).cloned().unwrap_or_default();
line.push_str(&c);
if i + 1 < n {
line.push_str(&" ".repeat(w.saturating_sub(c.chars().count())));
line.push_str(" ");
}
}
putline(line.trim_end());
if ri == 0 {
let sep: Vec<String> = widths.iter().map(|w| "-".repeat(*w)).collect();
putline(sep.join(" ").trim_end());
}
}
}
+130
View File
@@ -0,0 +1,130 @@
//! FTS5-backed search with a simple own ranking (exact > prefix > fulltext).
use std::collections::HashSet;
use anyhow::Result;
use rusqlite::{Connection, params};
use serde::Serialize;
#[derive(Debug, Serialize)]
pub struct SymbolHit {
pub id: i64,
pub name: String,
pub qualified_name: String,
pub kind: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub signature: Option<String>,
pub file: String,
pub line_start: i64,
pub line_end: i64,
pub match_type: String,
}
#[derive(Debug, Serialize)]
pub struct TextHit {
pub file: String,
pub snippet: String,
}
const SYMBOL_COLS: &str =
"s.id, s.name, s.qualified_name, s.kind, s.signature, f.path, s.line_start, s.line_end";
pub fn search_symbols(conn: &Connection, query: &str) -> Result<Vec<SymbolHit>> {
let q = query.trim();
let mut seen: HashSet<i64> = HashSet::new();
let mut out: Vec<SymbolHit> = Vec::new();
if q.is_empty() {
return Ok(out);
}
// 1) exact name (case-insensitive)
{
let mut stmt = conn.prepare(&format!(
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
WHERE s.name = ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
))?;
let rows = stmt.query_map(params![q], |r| row_to_hit(r, "exact"))?;
push_hits(&mut out, &mut seen, rows)?;
}
// 2) prefix
{
let pat = format!("{}%", q);
let mut stmt = conn.prepare(&format!(
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
))?;
let rows = stmt.query_map(params![pat], |r| row_to_hit(r, "prefix"))?;
push_hits(&mut out, &mut seen, rows)?;
}
// 3) fulltext over name/qualified_name/kind/signature
{
let phrase = fts_phrase(q);
let mut stmt = conn.prepare(&format!(
"SELECT {SYMBOL_COLS} FROM symbols_fts \
JOIN symbols s ON s.id = symbols_fts.rowid \
JOIN files f ON f.id = s.file_id \
WHERE symbols_fts MATCH ?1 ORDER BY rank LIMIT 200"
))?;
let rows = stmt.query_map(params![phrase], |r| row_to_hit(r, "fulltext"))?;
push_hits(&mut out, &mut seen, rows)?;
}
Ok(out)
}
pub fn search_text(conn: &Connection, query: &str) -> Result<Vec<TextHit>> {
let q = query.trim();
if q.is_empty() {
return Ok(Vec::new());
}
let phrase = fts_phrase(q);
let mut stmt = conn.prepare(
"SELECT path, snippet(content_fts, 1, '«', '»', '…', 24) \
FROM content_fts WHERE content_fts MATCH ?1 ORDER BY rank LIMIT 100",
)?;
let rows = stmt.query_map(params![phrase], |r| {
Ok(TextHit {
file: r.get(0)?,
snippet: r.get(1)?,
})
})?;
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
}
fn row_to_hit(
r: &rusqlite::Row<'_>,
match_type: &str,
) -> std::result::Result<SymbolHit, rusqlite::Error> {
Ok(SymbolHit {
id: r.get(0)?,
name: r.get(1)?,
qualified_name: r.get(2)?,
kind: r.get(3)?,
signature: r.get(4)?,
file: r.get(5)?,
line_start: r.get(6)?,
line_end: r.get(7)?,
match_type: match_type.to_string(),
})
}
fn push_hits(
out: &mut Vec<SymbolHit>,
seen: &mut HashSet<i64>,
rows: impl Iterator<Item = std::result::Result<SymbolHit, rusqlite::Error>>,
) -> Result<()> {
for r in rows {
let hit = r?;
if seen.insert(hit.id) {
out.push(hit);
}
}
Ok(())
}
/// Wrap a user query as a safe FTS5 phrase.
fn fts_phrase(q: &str) -> String {
format!("\"{}\"", q.replace('"', "\"\""))
}