Initial commit: rustytokens — native codebase indexer + MCP server

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

tests/cli.rs

@@ -0,0 +1,480 @@
+//! End-to-end CLI tests against small, self-created fixture repos.
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::OnceLock;
+
+use serde_json::Value;
+
+fn bin() -> &'static str {
+    env!("CARGO_BIN_EXE_rustytokens")
+}
+
+/// Shared temp cache dir so tests never touch the real ~/.cache/rustytokens.
+fn cache_dir() -> &'static Path {
+    static DIR: OnceLock<PathBuf> = OnceLock::new();
+    DIR.get_or_init(|| {
+        let d = std::env::temp_dir().join(format!("rustytokens-test-cache-{}", std::process::id()));
+        std::fs::create_dir_all(&d).unwrap();
+        d
+    })
+}
+
+fn fixture(name: &str) -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures")
+        .join(name)
+}
+
+fn copy_dir(src: &Path, dst: &Path) {
+    std::fs::create_dir_all(dst).unwrap();
+    for entry in std::fs::read_dir(src).unwrap() {
+        let entry = entry.unwrap();
+        let to = dst.join(entry.file_name());
+        if entry.path().is_dir() {
+            copy_dir(&entry.path(), &to);
+        } else {
+            std::fs::copy(entry.path(), &to).unwrap();
+        }
+    }
+}
+
+/// Run the binary with the fixture's cache dir, returning parsed stdout JSON.
+fn run_json(args: &[&str]) -> Value {
+    let out = Command::new(bin())
+        .args(args)
+        .env("XDG_CACHE_HOME", cache_dir())
+        .output()
+        .expect("run rustytokens");
+    assert!(
+        out.status.success(),
+        "command failed: {:?}\nstderr: {}",
+        args,
+        String::from_utf8_lossy(&out.stderr)
+    );
+    serde_json::from_slice(&out.stdout).expect("stdout is valid JSON")
+}
+
+#[test]
+fn rust_index_search_and_dead() {
+    let fx = fixture("rustdemo");
+    run_json(&["index", "create", fx.to_str().unwrap()]);
+
+    let hits = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "search",
+        "symbols",
+        "add",
+    ]);
+    let hit = hits
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|v| v["name"] == "add")
+        .expect("search finds `add`");
+    assert_eq!(hit["kind"], "function");
+
+    // inspect symbol by id returns the exact source.
+    let id = hit["id"].as_i64().unwrap();
+    let sym = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "inspect",
+        "symbol",
+        &id.to_string(),
+    ]);
+    assert!(
+        sym["source"].as_str().unwrap().contains("a + b"),
+        "source contains function body"
+    );
+
+    // dead.rs is flagged; entry points lib.rs/main.rs are not.
+    let dead = run_json(&["--project", fx.to_str().unwrap(), "find", "dead"]);
+    let paths: Vec<&str> = dead["files"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|f| f["path"].as_str().unwrap())
+        .collect();
+    assert!(paths.contains(&"src/dead.rs"), "dead.rs flagged");
+    assert!(!paths.contains(&"src/lib.rs"), "lib.rs is an entry point");
+    assert!(!paths.contains(&"src/main.rs"), "main.rs is an entry point");
+}
+
+#[test]
+fn inspect_blast_radius_finds_affected_files() {
+    let src = fixture("rustdemo");
+    let tmp = cache_dir().join("blast-rustdemo");
+    let _ = std::fs::remove_dir_all(&tmp);
+    copy_dir(&src, &tmp);
+    run_json(&["index", "create", tmp.to_str().unwrap()]);
+
+    let blast = run_json(&[
+        "--project",
+        tmp.to_str().unwrap(),
+        "inspect",
+        "blast-radius",
+        "src/util.rs",
+    ]);
+    assert_eq!(blast["count"], 1);
+    assert_eq!(blast["file"], "src/util.rs");
+    let affected: Vec<&str> = blast["affected"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|f| f["path"].as_str().unwrap())
+        .collect();
+    assert_eq!(affected, ["src/lib.rs"], "lib.rs imports util.rs");
+}
+
+#[test]
+fn python_import_resolution_and_dead() {
+    let fx = fixture("pydemo");
+    run_json(&["index", "create", fx.to_str().unwrap()]);
+
+    let importers = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "find",
+        "importers",
+        "util.py",
+    ]);
+    let names: Vec<&str> = importers
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(names.contains(&"main.py"), "main.py imports util.py");
+
+    let dead = run_json(&["--project", fx.to_str().unwrap(), "find", "dead"]);
+    let paths: Vec<&str> = dead["files"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|f| f["path"].as_str().unwrap())
+        .collect();
+    assert!(paths.contains(&"dead.py"), "dead.py flagged");
+    assert!(!paths.contains(&"util.py"), "util.py is imported");
+}
+
+#[test]
+fn typescript_cycle_detection() {
+    let fx = fixture("tsdemo");
+    run_json(&["index", "create", fx.to_str().unwrap()]);
+
+    let cycles = run_json(&["--project", fx.to_str().unwrap(), "inspect", "cycles"]);
+    assert_eq!(cycles["count"], 1, "one cycle");
+    let cycle = &cycles["cycles"][0];
+    let files: Vec<&str> = cycle
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(
+        files.contains(&"a.ts") && files.contains(&"b.ts"),
+        "a↔b cycle"
+    );
+}
+
+#[test]
+fn java_symbols_and_import_resolution() {
+    let fx = fixture("javademo");
+    run_json(&["index", "create", fx.to_str().unwrap()]);
+
+    let hits = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "search",
+        "symbols",
+        "helper",
+    ]);
+    let hit = hits
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|v| v["name"] == "helper")
+        .expect("search finds `helper`");
+    assert_eq!(hit["kind"], "method");
+
+    // ctags reports Java constructors as `method` with a full `end` span, so
+    // the multi-line `Main` constructor must carry line_end > line_start.
+    let ctor = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "search",
+        "symbols",
+        "Main",
+    ]);
+    let ctor_hit = ctor
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|v| v["name"] == "Main" && v["kind"] == "method")
+        .expect("search finds `Main` constructor");
+    assert!(
+        ctor_hit["line_end"].as_i64().unwrap() > ctor_hit["line_start"].as_i64().unwrap(),
+        "multi-line constructor has an extended span"
+    );
+
+    let importers = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "find",
+        "importers",
+        "Util.java",
+    ]);
+    let names: Vec<&str> = importers
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(names.contains(&"Main.java"), "Main.java imports Util.java");
+}
+
+#[test]
+fn c_and_cpp_symbols_and_imports() {
+    let cdemo = fixture("cdemo");
+    run_json(&["index", "create", cdemo.to_str().unwrap()]);
+
+    let hits = run_json(&[
+        "--project",
+        cdemo.to_str().unwrap(),
+        "search",
+        "symbols",
+        "helper",
+    ]);
+    assert!(
+        hits.as_array()
+            .unwrap()
+            .iter()
+            .any(|v| v["name"] == "helper"),
+        "c demo indexes `helper`"
+    );
+
+    let importers = run_json(&[
+        "--project",
+        cdemo.to_str().unwrap(),
+        "find",
+        "importers",
+        "util.h",
+    ]);
+    let names: Vec<&str> = importers
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(names.contains(&"main.c") && names.contains(&"util.c"));
+
+    let cppdemo = fixture("cppdemo");
+    run_json(&["index", "create", cppdemo.to_str().unwrap()]);
+
+    let hits = run_json(&[
+        "--project",
+        cppdemo.to_str().unwrap(),
+        "search",
+        "symbols",
+        "add",
+    ]);
+    assert!(
+        hits.as_array().unwrap().iter().any(|v| v["name"] == "add"),
+        "cpp demo indexes `add`"
+    );
+
+    // ctags reports C++ constructors/destructors as `function` with an `end`
+    // span; the multi-line `Math` ctor must carry line_end > line_start.
+    let ctor = run_json(&[
+        "--project",
+        cppdemo.to_str().unwrap(),
+        "search",
+        "symbols",
+        "Math",
+    ]);
+    let ctor_hit = ctor
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|v| v["name"] == "Math" && v["kind"] == "function")
+        .expect("search finds `Math` constructor");
+    assert_eq!(ctor_hit["kind"], "function");
+    assert!(
+        ctor_hit["line_end"].as_i64().unwrap() > ctor_hit["line_start"].as_i64().unwrap(),
+        "multi-line constructor has an extended span"
+    );
+
+    // Destructor is also `function` with a ctags-owned `end` span (name `~Math`).
+    let dtor_hit = ctor
+        .as_array()
+        .unwrap()
+        .iter()
+        .find(|v| v["name"] == "~Math" && v["kind"] == "function")
+        .expect("search finds `~Math` destructor");
+    assert!(
+        dtor_hit["line_end"].as_i64().unwrap() > dtor_hit["line_start"].as_i64().unwrap(),
+        "multi-line destructor has an extended span"
+    );
+
+    let importers = run_json(&[
+        "--project",
+        cppdemo.to_str().unwrap(),
+        "find",
+        "importers",
+        "math.hpp",
+    ]);
+    let names: Vec<&str> = importers
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(
+        names.contains(&"main.cpp") && names.contains(&"math.cpp"),
+        "both cpp files include math.hpp"
+    );
+}
+
+#[test]
+fn html_assets_are_discovered_and_imported() {
+    let fx = fixture("htmldemo");
+    run_json(&["index", "create", fx.to_str().unwrap()]);
+
+    let importers = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "find",
+        "importers",
+        "app.js",
+    ]);
+    let names: Vec<&str> = importers
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(
+        names.contains(&"index.html"),
+        "index.html imports app.js via <script src>"
+    );
+
+    let css = run_json(&[
+        "--project",
+        fx.to_str().unwrap(),
+        "find",
+        "importers",
+        "style.css",
+    ]);
+    let names: Vec<&str> = css
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|v| v.as_str().unwrap())
+        .collect();
+    assert!(
+        names.contains(&"index.html"),
+        "index.html imports style.css via <link href>"
+    );
+}
+
+#[test]
+fn mcp_serve_handles_tool_call() {
+    let src = fixture("rustdemo");
+    let tmp = cache_dir().join("mcp-rustdemo");
+    let _ = std::fs::remove_dir_all(&tmp);
+    copy_dir(&src, &tmp);
+    run_json(&["index", "create", tmp.to_str().unwrap()]);
+
+    let mut child = Command::new(bin())
+        .args(["--project", tmp.to_str().unwrap(), "serve"])
+        .env("XDG_CACHE_HOME", cache_dir())
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .spawn()
+        .expect("spawn serve");
+
+    {
+        let mut stdin = child.stdin.take().unwrap();
+        writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize"}}"#).unwrap();
+        writeln!(
+            stdin,
+            r#"{{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{{"name":"search_symbols","arguments":{{"query":"add"}}}}}}"#
+        )
+        .unwrap();
+    } // drop stdin -> closes the pipe, server exits
+
+    let out = child.wait_with_output().unwrap();
+    assert!(out.status.success(), "serve exits cleanly");
+
+    let responses: Vec<Value> = String::from_utf8_lossy(&out.stdout)
+        .lines()
+        .filter_map(|l| serde_json::from_str(l).ok())
+        .collect();
+    let init = responses
+        .iter()
+        .find(|r| r["id"] == 1)
+        .expect("initialize response");
+    assert_eq!(init["result"]["serverInfo"]["name"], "rustytokens");
+
+    let call = responses
+        .iter()
+        .find(|r| r["id"] == 2)
+        .expect("tools/call response");
+    let text = call["result"]["content"][0]["text"].as_str().unwrap();
+    let hits: Value = serde_json::from_str(text).unwrap();
+    assert!(
+        hits.as_array().unwrap().iter().any(|h| h["name"] == "add"),
+        "search_symbols over MCP finds `add`"
+    );
+}
+
+#[test]
+fn index_update_reindexes_changed_files() {
+    let src = fixture("javademo");
+    let tmp = cache_dir().join("update-javademo");
+    let _ = std::fs::remove_dir_all(&tmp);
+    copy_dir(&src, &tmp);
+
+    run_json(&["index", "create", tmp.to_str().unwrap()]);
+
+    let before = run_json(&[
+        "--project",
+        tmp.to_str().unwrap(),
+        "search",
+        "symbols",
+        "brandNew",
+    ]);
+    assert!(
+        before.as_array().unwrap().is_empty(),
+        "new symbol absent before update"
+    );
+
+    let lib = tmp.join("BrandNew.java");
+    std::fs::write(
+        &lib,
+        "package demo;\n\npublic class BrandNew {\n    public static int brandNew() { return 7; }\n}\n",
+    )
+    .unwrap();
+
+    run_json(&["index", "update", tmp.to_str().unwrap()]);
+
+    let after = run_json(&[
+        "--project",
+        tmp.to_str().unwrap(),
+        "search",
+        "symbols",
+        "brandNew",
+    ]);
+    assert!(
+        after
+            .as_array()
+            .unwrap()
+            .iter()
+            .any(|h| h["name"] == "brandNew"),
+        "index update picks up the new symbol"
+    );
+}

tests/fixtures/cdemo/dead.c

@@ -0,0 +1 @@
+int c_dead(void) { return 0; }

tests/fixtures/cdemo/main.c

@@ -0,0 +1,5 @@
+#include "util.h"
+
+int main(void) {
+    return helper();
+}

tests/fixtures/cdemo/util.c

@@ -0,0 +1,3 @@
+#include "util.h"
+
+int helper(void) { return 0; }

tests/fixtures/cdemo/util.h

@@ -0,0 +1 @@
+int helper(void);

tests/fixtures/cppdemo/main.cpp

@@ -0,0 +1,5 @@
+#include "math.hpp"
+
+int main() {
+    return Math::add(1, 2);
+}

tests/fixtures/cppdemo/math.cpp

@@ -0,0 +1,13 @@
+#include "math.hpp"
+
+Math::Math(int scale) : scale_(scale) {
+    scale_ = scale;
+}
+
+Math::~Math() {
+    scale_ = 0;
+}
+
+int Math::add(int a, int b) { return a + b; }
+
+int Math::scaled(int v) const { return v * scale_; }

tests/fixtures/cppdemo/math.hpp

@@ -0,0 +1,11 @@
+#pragma once
+
+class Math {
+public:
+    Math(int scale);
+    ~Math();
+    static int add(int a, int b);
+    int scaled(int v) const;
+private:
+    int scale_;
+};

tests/fixtures/htmldemo/app.js

@@ -0,0 +1 @@
+console.log("hi");

tests/fixtures/htmldemo/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <link rel="stylesheet" href="style.css">
+  <script src="app.js"></script>
+</head>
+<body>
+  <h1>Hello</h1>
+</body>
+</html>

tests/fixtures/htmldemo/style.css

@@ -0,0 +1 @@
+body { color: red; }

tests/fixtures/javademo/Dead.java

@@ -0,0 +1,5 @@
+package demo;
+
+public class Dead {
+    public static void unused() {}
+}

tests/fixtures/javademo/Main.java

@@ -0,0 +1,15 @@
+package demo;
+
+import demo.Util;
+
+public class Main {
+    private final int seed;
+
+    public Main(int seed) {
+        this.seed = seed;
+    }
+
+    public static void main(String[] args) {
+        System.out.println(Util.helper());
+    }
+}

tests/fixtures/javademo/Util.java

@@ -0,0 +1,5 @@
+package demo;
+
+public class Util {
+    public static int helper() { return 42; }
+}

tests/fixtures/pydemo/dead.py

@@ -0,0 +1,2 @@
+def py_dead():
+    return 0

tests/fixtures/pydemo/main.py

@@ -0,0 +1,5 @@
+from util import helper
+
+
+def run():
+    return helper()

tests/fixtures/pydemo/util.py

@@ -0,0 +1,2 @@
+def helper():
+    return 1

tests/fixtures/rustdemo/src/dead.rs

@@ -0,0 +1 @@
+pub fn totally_dead() {}

tests/fixtures/rustdemo/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod util;
+
+pub fn add(a: i32, b: i32) -> i32 {
+    a + b
+}

tests/fixtures/rustdemo/src/main.rs

@@ -0,0 +1,5 @@
+use rustdemo::add;
+
+fn main() {
+    println!("{}", add(2, 3));
+}

tests/fixtures/rustdemo/src/util.rs

@@ -0,0 +1,3 @@
+pub fn helper() -> String {
+    "ok".to_string()
+}

tests/fixtures/tsdemo/a.ts

@@ -0,0 +1,3 @@
+import { b } from "./b";
+
+export const a = 1;

tests/fixtures/tsdemo/b.ts

@@ -0,0 +1,3 @@
+import { a } from "./a";
+
+export const b = a;