Initial commit: rustytokens — native codebase indexer + MCP server
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
endergebnisandCopilot committed
commit
354132e887
43 files changed
+3783
No files matched your search
@@ -0,0 +1 @@
|
||||
/target
|
||||
Generated
+591
@@ -0,0 +1,591 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da"
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17"
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.18.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
|
||||
|
||||
[[package]]
|
||||
name = "globset"
|
||||
version = "0.4.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"bstr",
|
||||
"log",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "ignore"
|
||||
version = "0.4.33"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"globset",
|
||||
"log",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"same-file",
|
||||
"walkdir",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.31.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustytokens"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"ignore",
|
||||
"rayon",
|
||||
"regex",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "xxhash-rust"
|
||||
version = "0.8.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6"
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
[package]
|
||||
name = "rustytokens"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
rusqlite = { version = "0.31", features = ["bundled"] }
|
||||
ignore = "0.4"
|
||||
rayon = "1"
|
||||
xxhash-rust = { version = "0.8", features = ["xxh3"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
anyhow = "1"
|
||||
regex = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json = "1"
|
||||
|
||||
[profile.release]
|
||||
lto = true
|
||||
strip = true
|
||||
@@ -0,0 +1,172 @@
|
||||
# rustytokens
|
||||
|
||||
A native Rust CLI + MCP server that indexes a codebase with
|
||||
[`universal-ctags`](https://github.com/universal-ctags/ctags) and lets an AI
|
||||
coding agent retrieve **symbols and functions** instead of whole files — saving
|
||||
context tokens.
|
||||
|
||||
Language coverage (via ctags): Rust, Java, Python, C/C++, TypeScript,
|
||||
JavaScript, HTML. HTML import edges are extracted directly (not via ctags), and
|
||||
`<link>`/`<script>`/`<img>` references to local `.css`/`.js`/image files are
|
||||
tracked as imports. CSS files are discovered and import-tracked, but ctags has
|
||||
no CSS parser — `search symbols` returns nothing for `.css` files.
|
||||
|
||||
`rustytokens` is an independent, clean-room implementation. It does not share
|
||||
code with any AGPL-licensed project.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Rust (MSRV: current stable; built and tested on nightly)
|
||||
- `universal-ctags` on `PATH` (`ctags --version` must report Universal Ctags)
|
||||
|
||||
```bash
|
||||
# Debian/Ubuntu
|
||||
sudo apt install universal-ctags
|
||||
# macOS
|
||||
brew install universal-ctags
|
||||
```
|
||||
|
||||
No system SQLite needed — `rusqlite` bundles SQLite with FTS5.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
# binary at target/release/rustytokens
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Purpose |
|
||||
|---|---|
|
||||
| `index create <path>` | Index a project from scratch |
|
||||
| `index update <path>` | Re-index only changed files (content-hash diff) |
|
||||
| `search symbols <query>` | Symbol search by name (exact > prefix > full-text) |
|
||||
| `search text <query>` | Full-text search over file contents |
|
||||
| `inspect symbol <id>` | Source of a single symbol |
|
||||
| `inspect outline <file>` | Symbol outline of a file |
|
||||
| `inspect file <file> [--lines a-b]` | Read a file or line range |
|
||||
| `inspect tree [--depth n]` | Indexed file tree |
|
||||
| `inspect blast-radius <file> [--depth n]` | Files transitively affected by a change |
|
||||
| `find importers <file>` | Who imports this file |
|
||||
| `find dead` | Files nobody imports (entry points excluded) |
|
||||
| `inspect cycles` | Circular imports |
|
||||
| `serve` | MCP server over stdio |
|
||||
| `stats` | Index statistics |
|
||||
|
||||
Global options: `--project <dir>` (default `.`), `--format json|table`
|
||||
(default `json`), `--compact`.
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
# Index a project
|
||||
rustytokens index create ./myproject
|
||||
|
||||
# Find the `authenticate` function
|
||||
rustytokens --project ./myproject search symbols authenticate
|
||||
|
||||
# Get only that function's source (id from the search result)
|
||||
rustytokens --project ./myproject inspect symbol 42
|
||||
|
||||
# Which files import auth.rs?
|
||||
rustytokens --project ./myproject find importers src/auth.rs
|
||||
|
||||
# What's dead / circular?
|
||||
rustytokens --project ./myproject find dead
|
||||
rustytokens --project ./myproject inspect cycles
|
||||
|
||||
# Human-readable output
|
||||
rustytokens --project ./myproject --format table search symbols authenticate
|
||||
```
|
||||
|
||||
## MCP integration
|
||||
|
||||
Register the binary as a stdio MCP server (e.g. in Claude Code / Finir
|
||||
`mcp.json`):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"rustytokens": {
|
||||
"command": "/path/to/rustytokens",
|
||||
"args": ["--project", "/path/to/your/project", "serve"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Exposed tools: `index_create`, `index_update`, `search_symbols`,
|
||||
`search_text`, `inspect_symbol`, `inspect_outline`, `inspect_file`,
|
||||
`inspect_tree`, `find_importers`, `find_dead`, `inspect_cycles`,
|
||||
`inspect_blast_radius`, `stats`.
|
||||
|
||||
## Why it saves tokens
|
||||
|
||||
Instead of sending a whole file, the agent fetches only the symbol it needs.
|
||||
Measured on `rustytokens`' own `src/ops.rs` (396 lines, ~11.7 KB):
|
||||
|
||||
- Whole file: 396 lines, 11,675 chars (~2,900 tokens)
|
||||
- `inspect symbol` for `build_tree`: 14 lines, 437 chars (~110 tokens)
|
||||
|
||||
That single retrieval is ~96% smaller than reading the file. Across a large
|
||||
codebase and a long session, per-symbol retrieval keeps context focused on the
|
||||
code actually being edited.
|
||||
|
||||
## Performance
|
||||
|
||||
Measured on a Xeon E5-2667 v3 @ 3.2 GHz (16 cores), `cargo build --release`,
|
||||
`universal-ctags` on `PATH`. Wall time, CPU time and peak RSS via
|
||||
`/usr/bin/time -v`.
|
||||
|
||||
| Operation | Repo (files) | Wall | CPU | Peak RSS |
|
||||
|---|---|---|---|---|
|
||||
| `index create` | nlohmann/json (497) | 1.61 s | 0.98 s | 22 MB |
|
||||
| `index create` | gson (264) | 0.92 s | 0.46 s | 13 MB |
|
||||
| `index create` (warm) | nlohmann/json (497) | 1.55 s | — | — |
|
||||
| `index update` (no-op) | nlohmann/json (497) | 0.02 s | — | — |
|
||||
| `search symbols` | — | 8.7 ms | — | 7.8 MB |
|
||||
| `find dead` | — | 4.4 ms | — | — |
|
||||
| `inspect cycles` | — | 3.7 ms | — | — |
|
||||
|
||||
Notes:
|
||||
|
||||
- `index update` is ~50× faster than a full rebuild when nothing changed: it
|
||||
hashes every file in parallel (rayon) and re-indexes only changed ones, while
|
||||
`index create` drops and rebuilds the whole database. A warm `index create`
|
||||
(≈1.55 s) shows that hashing dominates the cost.
|
||||
- Hardware cycle counters were not available (`perf` tools missing on this
|
||||
kernel), so cycle figures are estimates: CPU time × 3.2 GHz nominal.
|
||||
|
||||
## Development
|
||||
|
||||
```bash
|
||||
cargo fmt
|
||||
cargo clippy
|
||||
cargo test --release # unit (in-memory SQLite) + CLI integration tests
|
||||
```
|
||||
|
||||
Test fixtures live in `tests/fixtures/` (own small Rust/Python/TS repos).
|
||||
|
||||
## Licensing
|
||||
|
||||
`rustytokens` is an independent, clean-room implementation — no code is taken
|
||||
from any AGPL-licensed project, so the project's own source is yours to license
|
||||
however you choose (MIT, proprietary, commercial, closed-source). Decide and
|
||||
set it once: add a `license` field to `Cargo.toml` and, if you ship the source,
|
||||
drop in a `LICENSE` file.
|
||||
|
||||
Dependencies are all permissive (MIT / Apache-2.0 / BSL-1.0 / Unlicense):
|
||||
`clap`, `rusqlite`, `ignore`, `rayon`, `xxhash-rust`, `serde`, `serde_json`,
|
||||
`anyhow`, `regex`. None is copyleft — commercial use and closed-source
|
||||
distribution are fine. The only obligation is to ship each crate's copyright
|
||||
notice and license text (e.g. a `THIRD_PARTY_NOTICES.md`).
|
||||
|
||||
`universal-ctags` is GPL-licensed, but `rustytokens` invokes it only as an
|
||||
external program over `PATH` (subprocess + stdio, no linking, no bundling). That
|
||||
keeps it a separate program and does not impose GPL on `rustytokens` itself. Do
|
||||
not bundle the `ctags` binary into a distributed product — that is what would
|
||||
trigger GPL obligations for that binary. Keep it a user-installed prerequisite.
|
||||
|
||||
Naming: `rustytokens` is distinct from `toktoken`. If this goes commercial,
|
||||
avoid "toktoken-compatible" or similar wording in marketing.
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
//! clap definitions for all subcommands.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "rustytokens",
|
||||
version,
|
||||
about = "Native codebase indexer + MCP server"
|
||||
)]
|
||||
pub struct Cli {
|
||||
/// Project root for index-backed commands (defaults to current directory)
|
||||
#[arg(short, long, global = true, default_value = ".")]
|
||||
pub project: PathBuf,
|
||||
|
||||
/// Output format: json or table
|
||||
#[arg(long, global = true, default_value = "json")]
|
||||
pub format: String,
|
||||
|
||||
/// Compact JSON output
|
||||
#[arg(long, global = true)]
|
||||
pub compact: bool,
|
||||
|
||||
#[command(subcommand)]
|
||||
pub command: Command,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum Command {
|
||||
/// Build or refresh the index
|
||||
Index {
|
||||
#[command(subcommand)]
|
||||
sub: IndexCommand,
|
||||
},
|
||||
/// Search symbols or file contents
|
||||
Search {
|
||||
#[command(subcommand)]
|
||||
sub: SearchCommand,
|
||||
},
|
||||
/// Inspect symbols, files, outlines, the tree, or cycles
|
||||
Inspect {
|
||||
#[command(subcommand)]
|
||||
sub: InspectCommand,
|
||||
},
|
||||
/// Find importers or dead code
|
||||
Find {
|
||||
#[command(subcommand)]
|
||||
sub: FindCommand,
|
||||
},
|
||||
/// Start the MCP server over stdio
|
||||
Serve,
|
||||
/// Show index statistics
|
||||
Stats,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum IndexCommand {
|
||||
/// Index a project from scratch
|
||||
Create { path: PathBuf },
|
||||
/// Re-index only changed files
|
||||
Update { path: PathBuf },
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum SearchCommand {
|
||||
/// Search symbols by name
|
||||
Symbols { query: String },
|
||||
/// Full-text search over file contents
|
||||
Text { query: String },
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum InspectCommand {
|
||||
/// Source code of a single symbol by id
|
||||
Symbol { id: i64 },
|
||||
/// Symbol outline of a file
|
||||
Outline { file: String },
|
||||
/// Read a file or a line range (e.g. --lines 10-40)
|
||||
File {
|
||||
file: String,
|
||||
#[arg(long)]
|
||||
lines: Option<String>,
|
||||
},
|
||||
/// Indexed file tree
|
||||
Tree {
|
||||
#[arg(long)]
|
||||
depth: Option<usize>,
|
||||
},
|
||||
/// Files transitively affected by a change
|
||||
BlastRadius {
|
||||
file: String,
|
||||
#[arg(long)]
|
||||
depth: Option<usize>,
|
||||
},
|
||||
/// Find circular imports
|
||||
Cycles,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
pub enum FindCommand {
|
||||
/// Who imports this file
|
||||
Importers { file: String },
|
||||
/// Files with no importers
|
||||
Dead,
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//! Project configuration: paths, cache dir, ignore patterns, language mapping.
|
||||
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// Directory names skipped during discovery (own list).
|
||||
pub const SKIP_DIRS: &[&str] = &[
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
"node_modules",
|
||||
"target",
|
||||
"vendor",
|
||||
"venv",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"build",
|
||||
".idea",
|
||||
".vscode",
|
||||
".mypy_cache",
|
||||
".pytest_cache",
|
||||
".next",
|
||||
".nuxt",
|
||||
"bower_components",
|
||||
"coverage",
|
||||
"out",
|
||||
];
|
||||
|
||||
/// Map a file extension to a language label, or `None` if unsupported.
|
||||
pub fn language_for(path: &Path) -> Option<&'static str> {
|
||||
let ext = path.extension()?.to_str()?;
|
||||
Some(match ext {
|
||||
"rs" => "rust",
|
||||
"java" => "java",
|
||||
"py" => "python",
|
||||
"c" | "h" => "c",
|
||||
"cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => "cpp",
|
||||
"ts" | "tsx" | "mts" | "cts" => "typescript",
|
||||
"js" | "jsx" | "mjs" | "cjs" => "javascript",
|
||||
"html" | "htm" => "html",
|
||||
"css" => "css",
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Base cache directory for index databases.
|
||||
pub fn cache_dir() -> PathBuf {
|
||||
let base = std::env::var("XDG_CACHE_HOME")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| {
|
||||
std::env::var("HOME")
|
||||
.ok()
|
||||
.map(|h| PathBuf::from(h).join(".cache"))
|
||||
})
|
||||
.unwrap_or_else(|| PathBuf::from("."));
|
||||
base.join("rustytokens")
|
||||
}
|
||||
|
||||
/// Deterministic short key for a project path (used to name the cache DB).
|
||||
fn project_key(path: &str) -> String {
|
||||
let mut h = DefaultHasher::new();
|
||||
path.hash(&mut h);
|
||||
format!("{:016x}", h.finish())
|
||||
}
|
||||
|
||||
/// Path of the SQLite index DB for a project root.
|
||||
pub fn index_db_path(project: &Path) -> PathBuf {
|
||||
let canon = project
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| project.to_path_buf());
|
||||
let key = project_key(&canon.to_string_lossy());
|
||||
let name = project
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| "project".to_string());
|
||||
cache_dir().join(format!("{name}-{key}.db"))
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Thin subprocess wrapper around universal-ctags (no linking, own structs).
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct Tag {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
#[serde(default)]
|
||||
pub line: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub end: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub kind: Option<String>,
|
||||
#[serde(default)]
|
||||
pub scope: Option<String>,
|
||||
#[serde(default)]
|
||||
pub signature: Option<String>,
|
||||
}
|
||||
|
||||
/// Run ctags over `files`, returning parsed JSON tags.
|
||||
pub fn run(files: &[&Path]) -> Result<Vec<Tag>> {
|
||||
if files.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mut child = Command::new("ctags")
|
||||
.args(["--output-format=json", "--fields=+nKeSt", "-L", "-"])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.context("failed to start ctags (is universal-ctags installed?)")?;
|
||||
|
||||
{
|
||||
let stdin = child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.ok_or_else(|| anyhow!("ctags stdin unavailable"))?;
|
||||
for f in files {
|
||||
writeln!(stdin, "{}", f.to_string_lossy())?;
|
||||
}
|
||||
}
|
||||
|
||||
let output = child.wait_with_output().context("ctags failed")?;
|
||||
if !output.status.success() {
|
||||
let err = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(anyhow!("ctags exited with {}: {}", output.status, err));
|
||||
}
|
||||
|
||||
let mut tags = Vec::new();
|
||||
for line in output.stdout.split(|b| *b == b'\n') {
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Ok(t) = serde_json::from_slice::<Tag>(line) {
|
||||
tags.push(t);
|
||||
}
|
||||
}
|
||||
Ok(tags)
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Filesystem walk with smart filtering, producing the list of source files to index.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
use crate::config::{SKIP_DIRS, language_for};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SourceFile {
|
||||
/// Absolute path on disk.
|
||||
pub path: PathBuf,
|
||||
/// Path relative to the project root (forward slashes).
|
||||
pub rel: String,
|
||||
pub language: &'static str,
|
||||
}
|
||||
|
||||
pub fn discover(project: &Path) -> Result<Vec<SourceFile>> {
|
||||
let project_abs = project
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| project.to_path_buf());
|
||||
let mut builder = ignore::WalkBuilder::new(&project_abs);
|
||||
builder.standard_filters(true);
|
||||
builder.require_git(false);
|
||||
builder.filter_entry(|entry| {
|
||||
if entry.file_type().is_some_and(|t| t.is_dir()) {
|
||||
let name = entry.file_name().to_string_lossy();
|
||||
!SKIP_DIRS.iter().any(|d| *d == name)
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
let mut out = Vec::new();
|
||||
for entry in builder.build() {
|
||||
let entry = match entry {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let path = entry.path();
|
||||
if entry.file_type().is_some_and(|t| t.is_dir()) {
|
||||
continue;
|
||||
}
|
||||
let Some(language) = language_for(path) else {
|
||||
continue;
|
||||
};
|
||||
let rel = path
|
||||
.strip_prefix(&project_abs)
|
||||
.map(|p| p.to_string_lossy().replace('\\', "/"))
|
||||
.unwrap_or_else(|_| path.to_string_lossy().replace('\\', "/"));
|
||||
out.push(SourceFile {
|
||||
path: path.to_path_buf(),
|
||||
rel,
|
||||
language,
|
||||
});
|
||||
}
|
||||
out.sort_by(|a, b| a.rel.cmp(&b.rel));
|
||||
Ok(out)
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
//! Import graph: cycles (Tarjan SCC), dead files, blast radius (BFS).
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
use crate::index::store::ImportRow;
|
||||
|
||||
pub struct Graph {
|
||||
adj: HashMap<i64, Vec<i64>>, // from_file_id -> to_file_id
|
||||
rev: HashMap<i64, Vec<i64>>, // to_file_id -> from_file_id
|
||||
}
|
||||
|
||||
impl Graph {
|
||||
pub fn from_imports(imports: &[ImportRow]) -> Self {
|
||||
let mut adj: HashMap<i64, Vec<i64>> = HashMap::new();
|
||||
let mut rev: HashMap<i64, Vec<i64>> = HashMap::new();
|
||||
for imp in imports {
|
||||
adj.entry(imp.from_file_id)
|
||||
.or_default()
|
||||
.push(imp.to_file_id);
|
||||
rev.entry(imp.to_file_id)
|
||||
.or_default()
|
||||
.push(imp.from_file_id);
|
||||
}
|
||||
Graph { adj, rev }
|
||||
}
|
||||
|
||||
/// Strongly-connected components with more than one node (or a self-loop).
|
||||
pub fn cycles(&self) -> Vec<Vec<i64>> {
|
||||
let mut sccs = tarjan_scc(&self.adj);
|
||||
sccs.retain(|scc| {
|
||||
scc.len() > 1
|
||||
|| (scc.len() == 1
|
||||
&& self
|
||||
.adj
|
||||
.get(&scc[0])
|
||||
.map(|v| v.contains(&scc[0]))
|
||||
.unwrap_or(false))
|
||||
});
|
||||
for scc in &mut sccs {
|
||||
scc.sort_unstable();
|
||||
}
|
||||
sccs
|
||||
}
|
||||
|
||||
/// Files with no incoming import edges (nobody imports them).
|
||||
pub fn dead_nodes(&self, all_file_ids: &[i64]) -> Vec<i64> {
|
||||
let mut dead: Vec<i64> = all_file_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| self.rev.get(id).map(|v| v.is_empty()).unwrap_or(true))
|
||||
.collect();
|
||||
dead.sort_unstable();
|
||||
dead
|
||||
}
|
||||
|
||||
/// BFS over the reverse graph: which files (transitively) import `start`.
|
||||
pub fn blast_radius(&self, start: i64, max_depth: usize) -> Vec<(i64, usize)> {
|
||||
let mut out = Vec::new();
|
||||
let mut seen = HashMap::new();
|
||||
seen.insert(start, 0usize);
|
||||
let mut queue = VecDeque::from([(start, 0usize)]);
|
||||
while let Some((node, depth)) = queue.pop_front() {
|
||||
if depth >= max_depth {
|
||||
continue;
|
||||
}
|
||||
if let Some(parents) = self.rev.get(&node) {
|
||||
for &p in parents {
|
||||
if let std::collections::hash_map::Entry::Vacant(e) = seen.entry(p) {
|
||||
e.insert(depth + 1);
|
||||
queue.push_back((p, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (id, d) in seen {
|
||||
if id != start {
|
||||
out.push((id, d));
|
||||
}
|
||||
}
|
||||
out.sort_by_key(|(_, d)| *d);
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterative Tarjan SCC. Returns strongly-connected components.
|
||||
fn tarjan_scc(adj: &HashMap<i64, Vec<i64>>) -> Vec<Vec<i64>> {
|
||||
let mut index_counter = 0usize;
|
||||
let mut indices: HashMap<i64, usize> = HashMap::new();
|
||||
let mut lowlinks: HashMap<i64, usize> = HashMap::new();
|
||||
let mut stack: Vec<i64> = Vec::new();
|
||||
let mut on_stack: HashMap<i64, bool> = HashMap::new();
|
||||
let mut sccs: Vec<Vec<i64>> = Vec::new();
|
||||
|
||||
let nodes: Vec<i64> = {
|
||||
let mut set = std::collections::HashSet::new();
|
||||
for (k, v) in adj {
|
||||
set.insert(*k);
|
||||
for t in v {
|
||||
set.insert(*t);
|
||||
}
|
||||
}
|
||||
let mut v: Vec<i64> = set.into_iter().collect();
|
||||
v.sort_unstable();
|
||||
v
|
||||
};
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn strongconnect(
|
||||
v: i64,
|
||||
adj: &HashMap<i64, Vec<i64>>,
|
||||
index_counter: &mut usize,
|
||||
indices: &mut HashMap<i64, usize>,
|
||||
lowlinks: &mut HashMap<i64, usize>,
|
||||
stack: &mut Vec<i64>,
|
||||
on_stack: &mut HashMap<i64, bool>,
|
||||
sccs: &mut Vec<Vec<i64>>,
|
||||
) {
|
||||
indices.insert(v, *index_counter);
|
||||
lowlinks.insert(v, *index_counter);
|
||||
*index_counter += 1;
|
||||
stack.push(v);
|
||||
on_stack.insert(v, true);
|
||||
|
||||
if let Some(neighbors) = adj.get(&v) {
|
||||
for &w in neighbors {
|
||||
if !indices.contains_key(&w) {
|
||||
strongconnect(
|
||||
w,
|
||||
adj,
|
||||
index_counter,
|
||||
indices,
|
||||
lowlinks,
|
||||
stack,
|
||||
on_stack,
|
||||
sccs,
|
||||
);
|
||||
let lw = lowlinks[&w];
|
||||
let lv = lowlinks[&v];
|
||||
lowlinks.insert(v, lv.min(lw));
|
||||
} else if on_stack.get(&w).copied().unwrap_or(false) {
|
||||
let iw = indices[&w];
|
||||
let lv = lowlinks[&v];
|
||||
lowlinks.insert(v, lv.min(iw));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if lowlinks[&v] == indices[&v] {
|
||||
let mut component = Vec::new();
|
||||
loop {
|
||||
let w = stack.pop().unwrap();
|
||||
on_stack.insert(w, false);
|
||||
component.push(w);
|
||||
if w == v {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sccs.push(component);
|
||||
}
|
||||
}
|
||||
|
||||
for &n in &nodes {
|
||||
if !indices.contains_key(&n) {
|
||||
strongconnect(
|
||||
n,
|
||||
adj,
|
||||
&mut index_counter,
|
||||
&mut indices,
|
||||
&mut lowlinks,
|
||||
&mut stack,
|
||||
&mut on_stack,
|
||||
&mut sccs,
|
||||
);
|
||||
}
|
||||
}
|
||||
sccs
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Content hashing for change detection.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
|
||||
pub fn hash_file(path: &Path) -> Result<String> {
|
||||
let mut f = File::open(path)?;
|
||||
let mut buf = Vec::new();
|
||||
f.read_to_end(&mut buf)?;
|
||||
Ok(format!("{:016x}", xxhash_rust::xxh3::xxh3_64(&buf)))
|
||||
}
|
||||
|
||||
pub fn mtime_secs(path: &Path) -> i64 {
|
||||
std::fs::metadata(path)
|
||||
.and_then(|m| m.modified())
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0)
|
||||
}
|
||||
@@ -0,0 +1,398 @@
|
||||
//! Import extraction (per-language) + best-effort resolution to file ids.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use crate::discovery::SourceFile;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Import {
|
||||
pub target: String,
|
||||
pub import_type: String,
|
||||
/// Directory levels a relative import ascends (Python `from ..x`).
|
||||
pub levels: usize,
|
||||
}
|
||||
|
||||
/// Extract import targets from a source file (own heuristic parsers).
|
||||
pub fn extract(f: &SourceFile) -> Vec<Import> {
|
||||
let content = std::fs::read_to_string(&f.path).unwrap_or_default();
|
||||
match f.language {
|
||||
"python" => extract_python(&content),
|
||||
"rust" => extract_rust(&content),
|
||||
"typescript" | "javascript" => extract_ts(&content),
|
||||
"c" | "cpp" => extract_c(&content),
|
||||
"java" => extract_java(&content),
|
||||
"html" => extract_html(&content),
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_python(content: &str) -> Vec<Import> {
|
||||
let mut out = Vec::new();
|
||||
for line in content.lines() {
|
||||
let l = line.trim();
|
||||
if let Some(rest) = l.strip_prefix("import ") {
|
||||
for part in rest.split(',') {
|
||||
let p = part.trim();
|
||||
if !p.is_empty() {
|
||||
out.push(Import {
|
||||
target: p.to_string(),
|
||||
import_type: "import".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if let Some(rest) = l.strip_prefix("from ") {
|
||||
// `from [..]<mod> import names` — leading dots = parent levels.
|
||||
let module = rest.split_whitespace().next().unwrap_or("");
|
||||
let dots = module.bytes().take_while(|b| *b == b'.').count();
|
||||
let name = &module[dots..];
|
||||
let levels = dots.saturating_sub(1);
|
||||
if name.is_empty() {
|
||||
// `from . import x` -> `x` lives `levels` dirs up.
|
||||
let after = rest.split_once("import ").map(|(_, s)| s).unwrap_or("");
|
||||
for n in after.split(',') {
|
||||
let n = n.trim().trim_end_matches('\\').trim();
|
||||
if !n.is_empty() {
|
||||
out.push(Import {
|
||||
target: n.to_string(),
|
||||
import_type: "from".into(),
|
||||
levels,
|
||||
});
|
||||
}
|
||||
}
|
||||
} else {
|
||||
out.push(Import {
|
||||
target: name.to_string(),
|
||||
import_type: "from".into(),
|
||||
levels,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn extract_rust(content: &str) -> Vec<Import> {
|
||||
let mut out = Vec::new();
|
||||
for line in content.lines() {
|
||||
let l = line.trim();
|
||||
if let Some(rest) = l.strip_prefix("use ") {
|
||||
let rest = rest
|
||||
.trim_end_matches(';')
|
||||
.split('{')
|
||||
.next()
|
||||
.unwrap_or(rest)
|
||||
.trim();
|
||||
let path = rest
|
||||
.trim_start_matches("crate::")
|
||||
.trim_start_matches("self::")
|
||||
.trim_start_matches("super::")
|
||||
.trim_start_matches("::");
|
||||
let mut segs: Vec<&str> = path.split("::").filter(|s| !s.is_empty()).collect();
|
||||
if segs.is_empty() || matches!(segs[0], "std" | "core" | "alloc") {
|
||||
continue;
|
||||
}
|
||||
// A trailing CamelCase segment is a type/symbol, not a module path.
|
||||
if let Some(last) = segs.last()
|
||||
&& last.chars().next().is_some_and(char::is_uppercase)
|
||||
{
|
||||
segs.pop();
|
||||
}
|
||||
if let Some(first) = segs.first() {
|
||||
out.push(Import {
|
||||
target: first.to_string(),
|
||||
import_type: "use".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
if segs.len() > 1 {
|
||||
out.push(Import {
|
||||
target: segs.join("/"),
|
||||
import_type: "use".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
let mod_name = strip_mod_decl(l);
|
||||
if let Some(name) = mod_name {
|
||||
out.push(Import {
|
||||
target: name,
|
||||
import_type: "mod".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Extract the module name from a `mod` declaration, handling visibility
|
||||
/// modifiers such as `pub`, `pub(crate)`, `pub(super)`, and attributes.
|
||||
fn strip_mod_decl(l: &str) -> Option<String> {
|
||||
let mut rest = l;
|
||||
// Strip leading attributes like `#[cfg(...)]`.
|
||||
while rest.starts_with("#[") {
|
||||
let end = rest.find("]\n").or_else(|| rest.find(']'))? + 1;
|
||||
rest = rest[end..].trim_start();
|
||||
}
|
||||
if let Some(r) = rest.strip_prefix("pub ") {
|
||||
rest = r;
|
||||
} else if let Some(r) = rest.strip_prefix("pub(") {
|
||||
let end = r.find(") ")? + 2;
|
||||
rest = r[end..].trim_start();
|
||||
}
|
||||
let name = rest
|
||||
.strip_prefix("mod ")?
|
||||
.trim()
|
||||
.trim_end_matches(';')
|
||||
.trim();
|
||||
if name.is_empty() || name.contains(' ') {
|
||||
None
|
||||
} else {
|
||||
Some(name.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
fn ts_import_re() -> &'static Regex {
|
||||
static RE: OnceLock<Regex> = OnceLock::new();
|
||||
RE.get_or_init(|| {
|
||||
Regex::new(r#"(?:import|export)\s+(?:[^'"]*?from\s+)?['"]([^'"]+)['"]"#).unwrap()
|
||||
})
|
||||
}
|
||||
|
||||
fn ts_require_re() -> &'static Regex {
|
||||
static RE: OnceLock<Regex> = OnceLock::new();
|
||||
RE.get_or_init(|| Regex::new(r#"require\(\s*['"]([^'"]+)['"]\s*\)"#).unwrap())
|
||||
}
|
||||
|
||||
fn extract_ts(content: &str) -> Vec<Import> {
|
||||
let mut out = Vec::new();
|
||||
for cap in ts_import_re().captures_iter(content) {
|
||||
let target = cap[1].to_string();
|
||||
let t = target.trim_start_matches("./");
|
||||
out.push(Import {
|
||||
target: t.to_string(),
|
||||
import_type: "import".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
for cap in ts_require_re().captures_iter(content) {
|
||||
let target = cap[1].to_string();
|
||||
let t = target.trim_start_matches("./");
|
||||
out.push(Import {
|
||||
target: t.to_string(),
|
||||
import_type: "require".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn extract_c(content: &str) -> Vec<Import> {
|
||||
let mut out = Vec::new();
|
||||
for line in content.lines() {
|
||||
let l = line.trim();
|
||||
if let Some(rest) = l.strip_prefix("#include") {
|
||||
let inner = rest
|
||||
.trim()
|
||||
.trim_start_matches('"')
|
||||
.trim_start_matches('<')
|
||||
.split(['"', '>'])
|
||||
.next()
|
||||
.unwrap_or("");
|
||||
if !inner.is_empty() {
|
||||
out.push(Import {
|
||||
target: inner.to_string(),
|
||||
import_type: "include".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn extract_java(content: &str) -> Vec<Import> {
|
||||
let mut out = Vec::new();
|
||||
for line in content.lines() {
|
||||
let l = line.trim();
|
||||
if let Some(rest) = l.strip_prefix("import ") {
|
||||
let rest = rest.trim_end_matches(';').trim_start_matches("static ");
|
||||
let rest = rest.split_whitespace().next().unwrap_or(rest);
|
||||
if !rest.is_empty() {
|
||||
out.push(Import {
|
||||
target: rest.to_string(),
|
||||
import_type: "import".into(),
|
||||
levels: 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn html_attr_re() -> &'static Regex {
|
||||
static RE: OnceLock<Regex> = OnceLock::new();
|
||||
RE.get_or_init(|| Regex::new(r#"(?:src|href)\s*=\s*["']([^"']+)["']"#).unwrap())
|
||||
}
|
||||
|
||||
/// `<script src="...">` / `<link href="...">` as relative imports. ctags' HTML
|
||||
/// parser is weak, so symbol coverage here is best-effort.
|
||||
fn extract_html(content: &str) -> Vec<Import> {
|
||||
html_attr_re()
|
||||
.captures_iter(content)
|
||||
.filter_map(|cap| {
|
||||
let target = cap[1].trim();
|
||||
// Skip absolute URLs, data URIs, and fragments.
|
||||
if target.starts_with("http://")
|
||||
|| target.starts_with("https://")
|
||||
|| target.starts_with("//")
|
||||
|| target.starts_with("data:")
|
||||
|| target.starts_with('#')
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let t = target.trim_start_matches("./");
|
||||
if t.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Import {
|
||||
target: t.to_string(),
|
||||
import_type: "asset".into(),
|
||||
levels: 0,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Best-effort mapping of an import target string to a file id.
|
||||
pub struct Resolver {
|
||||
by_path: HashMap<String, i64>,
|
||||
by_name: HashMap<String, i64>,
|
||||
by_stem: HashMap<String, i64>,
|
||||
}
|
||||
|
||||
/// Normalize an import target string (strip prefixes like `./`, `crate::`).
|
||||
fn clean(target: &str) -> Option<String> {
|
||||
let t = target.trim();
|
||||
if t.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let t = t.trim_start_matches("./");
|
||||
let t = t
|
||||
.trim_start_matches("crate::")
|
||||
.trim_start_matches("self::")
|
||||
.trim_start_matches("super::")
|
||||
.trim_start_matches("::");
|
||||
if t.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(t.to_string())
|
||||
}
|
||||
|
||||
impl Resolver {
|
||||
pub fn build(files: Vec<(String, i64)>) -> Self {
|
||||
let mut by_path = HashMap::new();
|
||||
let mut by_name = HashMap::new();
|
||||
let mut by_stem = HashMap::new();
|
||||
for (rel, id) in files {
|
||||
by_path.insert(rel.clone(), id);
|
||||
let name = rel.rsplit('/').next().unwrap_or(&rel).to_string();
|
||||
by_name.entry(name.clone()).or_insert(id);
|
||||
let stem = name
|
||||
.rsplit_once('.')
|
||||
.map(|(s, _)| s.to_string())
|
||||
.unwrap_or(name);
|
||||
by_stem.entry(stem).or_insert(id);
|
||||
}
|
||||
Resolver {
|
||||
by_path,
|
||||
by_name,
|
||||
by_stem,
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve an import relative to the importing file's directory first,
|
||||
/// ascending `levels` parent dirs for relative imports.
|
||||
pub fn resolve_with_context(&self, target: &str, dir: &str, levels: usize) -> Option<i64> {
|
||||
let t = clean(target)?;
|
||||
let mut base = dir;
|
||||
for _ in 0..levels {
|
||||
base = base.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
|
||||
}
|
||||
let base = if base.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{base}/")
|
||||
};
|
||||
self.resolve_relative(&base, &t)
|
||||
.or_else(|| self.resolve_clean(&t))
|
||||
}
|
||||
|
||||
fn resolve_clean(&self, t: &str) -> Option<i64> {
|
||||
if let Some(id) = self.by_path.get(t) {
|
||||
return Some(*id);
|
||||
}
|
||||
let slashed = t.replace('.', "/");
|
||||
if slashed != t {
|
||||
if let Some(id) = self.by_path.get(&slashed) {
|
||||
return Some(*id);
|
||||
}
|
||||
if let Some(id) = self.try_module(&slashed) {
|
||||
return Some(id);
|
||||
}
|
||||
}
|
||||
if let Some(id) = self.try_module(t) {
|
||||
return Some(id);
|
||||
}
|
||||
if let Some(id) = self.by_stem.get(t) {
|
||||
return Some(*id);
|
||||
}
|
||||
let last = t.rsplit('/').next().unwrap_or(t);
|
||||
if let Some(id) = self.by_stem.get(last) {
|
||||
return Some(*id);
|
||||
}
|
||||
// Java dotted packages (`demo.Util`) — fall back to the final segment.
|
||||
let last_dot = last.rsplit('.').next().unwrap_or(last);
|
||||
if last_dot != last
|
||||
&& let Some(id) = self.by_stem.get(last_dot)
|
||||
{
|
||||
return Some(*id);
|
||||
}
|
||||
self.by_name.get(last).copied()
|
||||
}
|
||||
|
||||
fn resolve_relative(&self, base: &str, t: &str) -> Option<i64> {
|
||||
if let Some(id) = self.by_path.get(&format!("{base}{t}")) {
|
||||
return Some(*id);
|
||||
}
|
||||
for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
|
||||
if let Some(id) = self.by_path.get(&format!("{base}{t}.{ext}")) {
|
||||
return Some(*id);
|
||||
}
|
||||
}
|
||||
for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
|
||||
if let Some(id) = self.by_path.get(&format!("{base}{t}/{entry}")) {
|
||||
return Some(*id);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn try_module(&self, base: &str) -> Option<i64> {
|
||||
for ext in ["rs", "py", "ts", "tsx", "js", "cpp", "c", "h", "java"] {
|
||||
if let Some(id) = self.by_path.get(&format!("{base}.{ext}")) {
|
||||
return Some(*id);
|
||||
}
|
||||
}
|
||||
for entry in ["mod.rs", "__init__.py", "index.ts", "index.js", "index.tsx"] {
|
||||
if let Some(id) = self.by_path.get(&format!("{base}/{entry}")) {
|
||||
return Some(*id);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
pub mod hashing;
|
||||
pub mod imports;
|
||||
pub mod pipeline;
|
||||
pub mod schema;
|
||||
pub mod store;
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Index pipeline: hash, run ctags, extract imports, write to SQLite in one transaction.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use rayon::prelude::*;
|
||||
use rusqlite::params;
|
||||
|
||||
use crate::ctags;
|
||||
use crate::discovery::{self, SourceFile};
|
||||
use crate::index::{hashing, imports, store};
|
||||
|
||||
#[derive(Debug, serde::Serialize)]
|
||||
pub struct IndexStats {
|
||||
pub files: i64,
|
||||
pub symbols: i64,
|
||||
pub imports: i64,
|
||||
}
|
||||
|
||||
/// (Re)index a project. `recreate` drops any existing index first.
|
||||
pub fn run(project: &Path, db_path: &Path, recreate: bool) -> Result<IndexStats> {
|
||||
if recreate && db_path.exists() {
|
||||
std::fs::remove_file(db_path).context("remove old index")?;
|
||||
}
|
||||
let conn = store::open(db_path)?;
|
||||
|
||||
let files = discovery::discover(project)?;
|
||||
|
||||
let existing: HashMap<String, (i64, String)> = store::all_files(&conn)?
|
||||
.into_iter()
|
||||
.map(|f| (f.path.clone(), (f.id, f.content_hash.clone())))
|
||||
.collect();
|
||||
|
||||
// Hash all files in parallel.
|
||||
let hashed: Vec<(String, String, i64, &'static str)> = files
|
||||
.par_iter()
|
||||
.map(|f| {
|
||||
let h = hashing::hash_file(&f.path).unwrap_or_else(|_| "0000000000000000".to_string());
|
||||
let m = hashing::mtime_secs(&f.path);
|
||||
(f.rel.clone(), h, m, f.language)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let by_rel: HashMap<&str, &SourceFile> = files.iter().map(|f| (f.rel.as_str(), f)).collect();
|
||||
let abs_to_rel: HashMap<String, String> = files
|
||||
.iter()
|
||||
.map(|f| (f.path.to_string_lossy().into_owned(), f.rel.clone()))
|
||||
.collect();
|
||||
|
||||
let changed: Vec<&SourceFile> = hashed
|
||||
.iter()
|
||||
.filter(|(rel, h, _, _)| existing.get(rel).map(|(_, oh)| oh != h).unwrap_or(true))
|
||||
.filter_map(|(rel, ..)| by_rel.get(rel.as_str()).copied())
|
||||
.collect();
|
||||
|
||||
let deleted_ids: Vec<i64> = existing
|
||||
.iter()
|
||||
.filter(|(rel, _)| !by_rel.contains_key(rel.as_str()))
|
||||
.map(|(_, (id, _))| *id)
|
||||
.collect();
|
||||
|
||||
conn.execute_batch("BEGIN;")?;
|
||||
|
||||
let mut file_ids: HashMap<String, i64> = HashMap::new();
|
||||
for (rel, h, m, lang) in &hashed {
|
||||
let id = store::upsert_file(&conn, rel, h, lang, *m)?;
|
||||
file_ids.insert(rel.clone(), id);
|
||||
}
|
||||
|
||||
for id in &deleted_ids {
|
||||
store::delete_file(&conn, *id)?;
|
||||
conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [id])?;
|
||||
}
|
||||
|
||||
// Run ctags once over changed files.
|
||||
let changed_paths: Vec<&Path> = changed.iter().map(|f| f.path.as_path()).collect();
|
||||
let tags = ctags::run(&changed_paths)?;
|
||||
let mut tags_by_rel: HashMap<String, Vec<ctags::Tag>> = HashMap::new();
|
||||
for t in tags {
|
||||
if let Some(rel) = abs_to_rel.get(&t.path) {
|
||||
tags_by_rel.entry(rel.clone()).or_default().push(t);
|
||||
}
|
||||
}
|
||||
|
||||
let resolver =
|
||||
imports::Resolver::build(file_ids.iter().map(|(k, v)| (k.clone(), *v)).collect());
|
||||
|
||||
for f in &changed {
|
||||
let fid = file_ids[&f.rel];
|
||||
let content = std::fs::read_to_string(&f.path).unwrap_or_default();
|
||||
let tags = tags_by_rel.get(&f.rel).cloned().unwrap_or_default();
|
||||
store::replace_symbols(&conn, fid, &tags_to_symbols(&tags, &content))?;
|
||||
store::clear_imports_for_file(&conn, fid)?;
|
||||
let dir = f
|
||||
.rel
|
||||
.rsplit_once('/')
|
||||
.map(|(d, _)| d)
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
for imp in imports::extract(f) {
|
||||
if let Some(to) = resolver.resolve_with_context(&imp.target, &dir, imp.levels)
|
||||
&& to != fid
|
||||
{
|
||||
store::insert_import(&conn, fid, to, &imp.import_type)?;
|
||||
}
|
||||
}
|
||||
conn.execute("DELETE FROM content_fts WHERE rowid = ?1", [fid])?;
|
||||
conn.execute(
|
||||
"INSERT INTO content_fts(rowid, path, content) VALUES (?1, ?2, ?3)",
|
||||
params![fid, f.rel, content],
|
||||
)?;
|
||||
}
|
||||
|
||||
conn.execute_batch("COMMIT;")?;
|
||||
|
||||
let (files_n, symbols_n, imports_n) = store::stats(&conn)?;
|
||||
Ok(IndexStats {
|
||||
files: files_n,
|
||||
symbols: symbols_n,
|
||||
imports: imports_n,
|
||||
})
|
||||
}
|
||||
|
||||
fn tags_to_symbols(tags: &[ctags::Tag], content: &str) -> Vec<store::NewSymbol> {
|
||||
let lines: Vec<&str> = content.lines().collect();
|
||||
tags.iter()
|
||||
.filter_map(|t| {
|
||||
let line = t.line? as i64;
|
||||
let end = t.end.map(|e| e as i64).unwrap_or(line);
|
||||
let kind = t.kind.clone().unwrap_or_default();
|
||||
let end = extend_block_end(&lines, line, end, &kind);
|
||||
let name = t.name.clone();
|
||||
let qualified_name = match &t.scope {
|
||||
Some(s) if !name.starts_with(&format!("{s}.")) => format!("{s}.{name}"),
|
||||
_ => name.clone(),
|
||||
};
|
||||
Some(store::NewSymbol {
|
||||
name,
|
||||
qualified_name,
|
||||
kind,
|
||||
signature: t.signature.clone(),
|
||||
line_start: line,
|
||||
line_end: end,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// ctags omits a body span for some kinds (e.g. Rust `fn`), leaving `end ==
|
||||
/// start` even for a multi-line block. Extend brace-based blocks to their
|
||||
/// closing brace so stored spans are correct for every reader.
|
||||
fn extend_block_end(lines: &[&str], start: i64, end: i64, kind: &str) -> i64 {
|
||||
if !is_block_kind(kind) || end > start {
|
||||
return end;
|
||||
}
|
||||
// Semicolon-terminated declarations (`mod foo;`, `struct Foo;`) have no body.
|
||||
if let Some(line) = lines.get((start - 1) as usize)
|
||||
&& line.trim_end().ends_with(';')
|
||||
{
|
||||
return end;
|
||||
}
|
||||
let mut depth = 0i64;
|
||||
let mut opened = false;
|
||||
let mut out = end;
|
||||
// ponytail: naive char brace count; braces in string/comment literals skew
|
||||
// depth. Swap for a tokenizer only if real spans come back wrong.
|
||||
for (i, line) in lines.iter().enumerate() {
|
||||
let n = (i as i64) + 1;
|
||||
if n < start {
|
||||
continue;
|
||||
}
|
||||
for c in line.chars() {
|
||||
match c {
|
||||
'{' => {
|
||||
depth += 1;
|
||||
opened = true;
|
||||
}
|
||||
'}' => depth -= 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if opened {
|
||||
out = n;
|
||||
}
|
||||
if opened && depth <= 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn is_block_kind(kind: &str) -> bool {
|
||||
matches!(
|
||||
kind,
|
||||
"function"
|
||||
| "method"
|
||||
| "struct"
|
||||
| "class"
|
||||
| "interface"
|
||||
| "enum"
|
||||
| "trait"
|
||||
| "impl"
|
||||
| "implementation"
|
||||
| "module"
|
||||
| "namespace"
|
||||
| "union"
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
//! SQLite schema + minimal migration (schema_version in metadata).
|
||||
|
||||
use anyhow::Result;
|
||||
use rusqlite::Connection;
|
||||
|
||||
const SCHEMA_VERSION: i64 = 1;
|
||||
|
||||
const SCHEMA: &str = r#"
|
||||
CREATE TABLE files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
content_hash TEXT NOT NULL,
|
||||
language TEXT NOT NULL,
|
||||
mtime INTEGER NOT NULL
|
||||
);
|
||||
CREATE TABLE symbols (
|
||||
id INTEGER PRIMARY KEY,
|
||||
file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
qualified_name TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
signature TEXT,
|
||||
line_start INTEGER NOT NULL,
|
||||
line_end INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_symbols_file ON symbols(file_id);
|
||||
CREATE INDEX idx_symbols_name ON symbols(name);
|
||||
CREATE TABLE imports (
|
||||
id INTEGER PRIMARY KEY,
|
||||
from_file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
to_file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,
|
||||
import_type TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX idx_imports_from ON imports(from_file_id);
|
||||
CREATE INDEX idx_imports_to ON imports(to_file_id);
|
||||
CREATE VIRTUAL TABLE symbols_fts USING fts5(
|
||||
name, qualified_name, kind, signature,
|
||||
content='symbols', content_rowid='id'
|
||||
);
|
||||
CREATE TRIGGER symbols_ai AFTER INSERT ON symbols BEGIN
|
||||
INSERT INTO symbols_fts(rowid, name, qualified_name, kind, signature)
|
||||
VALUES (new.id, new.name, new.qualified_name, new.kind, new.signature);
|
||||
END;
|
||||
CREATE TRIGGER symbols_ad AFTER DELETE ON symbols BEGIN
|
||||
INSERT INTO symbols_fts(symbols_fts, rowid, name, qualified_name, kind, signature)
|
||||
VALUES ('delete', old.id, old.name, old.qualified_name, old.kind, old.signature);
|
||||
END;
|
||||
CREATE VIRTUAL TABLE content_fts USING fts5(path UNINDEXED, content);
|
||||
"#;
|
||||
|
||||
/// Create schema if missing, or migrate to the current version.
|
||||
pub fn init(conn: &mut Connection) -> Result<()> {
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);",
|
||||
)?;
|
||||
let version: i64 = conn
|
||||
.query_row(
|
||||
"SELECT COALESCE(CAST(value AS INTEGER), 0) FROM metadata WHERE key='schema_version'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap_or(0);
|
||||
if version < SCHEMA_VERSION {
|
||||
conn.execute_batch(SCHEMA)?;
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO metadata (key, value) VALUES ('schema_version', ?1)",
|
||||
[SCHEMA_VERSION.to_string()],
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
//! CRUD on files/symbols/imports.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::index::schema;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FileRow {
|
||||
pub id: i64,
|
||||
pub path: String,
|
||||
pub content_hash: String,
|
||||
pub language: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct SymbolRow {
|
||||
pub id: i64,
|
||||
pub file_id: i64,
|
||||
pub name: String,
|
||||
pub qualified_name: String,
|
||||
pub kind: String,
|
||||
pub signature: Option<String>,
|
||||
pub line_start: i64,
|
||||
pub line_end: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NewSymbol {
|
||||
pub name: String,
|
||||
pub qualified_name: String,
|
||||
pub kind: String,
|
||||
pub signature: Option<String>,
|
||||
pub line_start: i64,
|
||||
pub line_end: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ImportRow {
|
||||
pub from_file_id: i64,
|
||||
pub to_file_id: i64,
|
||||
}
|
||||
|
||||
/// Open (creating if needed) the index DB and apply schema.
|
||||
pub fn open(path: &Path) -> Result<Connection> {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut conn = Connection::open(path)?;
|
||||
conn.execute_batch(
|
||||
"PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL;",
|
||||
)?;
|
||||
schema::init(&mut conn)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
pub fn all_files(conn: &Connection) -> Result<Vec<FileRow>> {
|
||||
let mut stmt = conn.prepare("SELECT id, path, content_hash, language FROM files")?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
Ok(FileRow {
|
||||
id: r.get(0)?,
|
||||
path: r.get(1)?,
|
||||
content_hash: r.get(2)?,
|
||||
language: r.get(3)?,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
pub fn upsert_file(
|
||||
conn: &Connection,
|
||||
path: &str,
|
||||
hash: &str,
|
||||
language: &str,
|
||||
mtime: i64,
|
||||
) -> Result<i64> {
|
||||
conn.execute(
|
||||
"INSERT INTO files (path, content_hash, language, mtime) VALUES (?1, ?2, ?3, ?4)
|
||||
ON CONFLICT(path) DO UPDATE SET content_hash=?2, language=?3, mtime=?4",
|
||||
params![path, hash, language, mtime],
|
||||
)?;
|
||||
// last_insert_rowid() is unreliable after an upsert that updates; query it.
|
||||
Ok(conn.query_row("SELECT id FROM files WHERE path = ?1", [path], |r| r.get(0))?)
|
||||
}
|
||||
|
||||
pub fn delete_file(conn: &Connection, id: i64) -> Result<()> {
|
||||
conn.execute("DELETE FROM files WHERE id = ?1", [id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn file_by_id(conn: &Connection, id: i64) -> Result<Option<FileRow>> {
|
||||
conn.query_row(
|
||||
"SELECT id, path, content_hash, language FROM files WHERE id = ?1",
|
||||
[id],
|
||||
|r| {
|
||||
Ok(FileRow {
|
||||
id: r.get(0)?,
|
||||
path: r.get(1)?,
|
||||
content_hash: r.get(2)?,
|
||||
language: r.get(3)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
pub fn file_id_by_path(conn: &Connection, path: &str) -> Result<Option<i64>> {
|
||||
Ok(conn
|
||||
.query_row("SELECT id FROM files WHERE path = ?1", [path], |r| r.get(0))
|
||||
.optional()?)
|
||||
}
|
||||
|
||||
pub fn replace_symbols(conn: &Connection, file_id: i64, symbols: &[NewSymbol]) -> Result<()> {
|
||||
conn.execute("DELETE FROM symbols WHERE file_id = ?1", [file_id])?;
|
||||
let mut stmt = conn.prepare(
|
||||
"INSERT INTO symbols (file_id, name, qualified_name, kind, signature, line_start, line_end)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||
)?;
|
||||
for s in symbols {
|
||||
stmt.execute(params![
|
||||
file_id,
|
||||
s.name,
|
||||
s.qualified_name,
|
||||
s.kind,
|
||||
s.signature,
|
||||
s.line_start,
|
||||
s.line_end
|
||||
])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn clear_imports_for_file(conn: &Connection, file_id: i64) -> Result<()> {
|
||||
conn.execute("DELETE FROM imports WHERE from_file_id = ?1", [file_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn insert_import(conn: &Connection, from: i64, to: i64, import_type: &str) -> Result<()> {
|
||||
conn.execute(
|
||||
"INSERT INTO imports (from_file_id, to_file_id, import_type) VALUES (?1, ?2, ?3)",
|
||||
params![from, to, import_type],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn all_imports(conn: &Connection) -> Result<Vec<ImportRow>> {
|
||||
let mut stmt = conn.prepare("SELECT from_file_id, to_file_id FROM imports")?;
|
||||
let rows = stmt.query_map([], |r| {
|
||||
Ok(ImportRow {
|
||||
from_file_id: r.get(0)?,
|
||||
to_file_id: r.get(1)?,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
pub fn symbol_by_id(conn: &Connection, id: i64) -> Result<Option<SymbolRow>> {
|
||||
conn.query_row(
|
||||
"SELECT id, file_id, name, qualified_name, kind, signature, line_start, line_end
|
||||
FROM symbols WHERE id = ?1",
|
||||
[id],
|
||||
|r| {
|
||||
Ok(SymbolRow {
|
||||
id: r.get(0)?,
|
||||
file_id: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
qualified_name: r.get(3)?,
|
||||
kind: r.get(4)?,
|
||||
signature: r.get(5)?,
|
||||
line_start: r.get(6)?,
|
||||
line_end: r.get(7)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
pub fn symbols_for_file(conn: &Connection, file_id: i64) -> Result<Vec<SymbolRow>> {
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT id, file_id, name, qualified_name, kind, signature, line_start, line_end
|
||||
FROM symbols WHERE file_id = ?1 ORDER BY line_start",
|
||||
)?;
|
||||
let rows = stmt.query_map([file_id], |r| {
|
||||
Ok(SymbolRow {
|
||||
id: r.get(0)?,
|
||||
file_id: r.get(1)?,
|
||||
name: r.get(2)?,
|
||||
qualified_name: r.get(3)?,
|
||||
kind: r.get(4)?,
|
||||
signature: r.get(5)?,
|
||||
line_start: r.get(6)?,
|
||||
line_end: r.get(7)?,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
pub fn importers_of(conn: &Connection, file_id: i64) -> Result<Vec<i64>> {
|
||||
let mut stmt = conn.prepare("SELECT from_file_id FROM imports WHERE to_file_id = ?1")?;
|
||||
let rows = stmt.query_map([file_id], |r| r.get(0))?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
/// Returns (files, symbols, imports) counts.
|
||||
pub fn stats(conn: &Connection) -> Result<(i64, i64, i64)> {
|
||||
let files: i64 = conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))?;
|
||||
let symbols: i64 = conn.query_row("SELECT COUNT(*) FROM symbols", [], |r| r.get(0))?;
|
||||
let imports: i64 = conn.query_row("SELECT COUNT(*) FROM imports", [], |r| r.get(0))?;
|
||||
Ok((files, symbols, imports))
|
||||
}
|
||||
|
||||
pub fn language_counts(conn: &Connection) -> Result<Vec<(String, i64)>> {
|
||||
let mut stmt =
|
||||
conn.prepare("SELECT language, COUNT(*) FROM files GROUP BY language ORDER BY 2 DESC")?;
|
||||
let rows = stmt.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn mem() -> Connection {
|
||||
let mut conn = Connection::open_in_memory().unwrap();
|
||||
conn.execute_batch("PRAGMA foreign_keys=ON;").unwrap();
|
||||
schema::init(&mut conn).unwrap();
|
||||
conn
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_insert_query_and_upsert() {
|
||||
let conn = mem();
|
||||
let id1 = upsert_file(&conn, "src/lib.rs", "hash1", "rust", 100).unwrap();
|
||||
let id2 = upsert_file(&conn, "src/lib.rs", "hash2", "rust", 200).unwrap();
|
||||
assert_eq!(id1, id2, "upsert reuses the same row");
|
||||
let row = file_by_id(&conn, id1).unwrap().unwrap();
|
||||
assert_eq!(row.content_hash, "hash2");
|
||||
assert_eq!(row.language, "rust");
|
||||
assert_eq!(all_files(&conn).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn symbols_replace_and_query() {
|
||||
let conn = mem();
|
||||
let fid = upsert_file(&conn, "src/lib.rs", "h", "rust", 0).unwrap();
|
||||
replace_symbols(
|
||||
&conn,
|
||||
fid,
|
||||
&[
|
||||
NewSymbol {
|
||||
name: "add".into(),
|
||||
qualified_name: "add".into(),
|
||||
kind: "function".into(),
|
||||
signature: Some("(a: i32, b: i32) -> i32".into()),
|
||||
line_start: 3,
|
||||
line_end: 5,
|
||||
},
|
||||
NewSymbol {
|
||||
name: "AuthService".into(),
|
||||
qualified_name: "AuthService".into(),
|
||||
kind: "struct".into(),
|
||||
signature: None,
|
||||
line_start: 7,
|
||||
line_end: 7,
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let syms = symbols_for_file(&conn, fid).unwrap();
|
||||
assert_eq!(syms.len(), 2);
|
||||
assert_eq!(syms[0].name, "add");
|
||||
assert_eq!(
|
||||
symbol_by_id(&conn, syms[1].id).unwrap().unwrap().kind,
|
||||
"struct"
|
||||
);
|
||||
|
||||
// Replacing drops old symbols and re-inserts.
|
||||
replace_symbols(
|
||||
&conn,
|
||||
fid,
|
||||
&[NewSymbol {
|
||||
name: "only".into(),
|
||||
qualified_name: "only".into(),
|
||||
kind: "function".into(),
|
||||
signature: None,
|
||||
line_start: 1,
|
||||
line_end: 1,
|
||||
}],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(symbols_for_file(&conn, fid).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imports_roundtrip_and_importers() {
|
||||
let conn = mem();
|
||||
let a = upsert_file(&conn, "a.ts", "h", "typescript", 0).unwrap();
|
||||
let b = upsert_file(&conn, "b.ts", "h", "typescript", 0).unwrap();
|
||||
insert_import(&conn, a, b, "import").unwrap();
|
||||
let imports = all_imports(&conn).unwrap();
|
||||
assert_eq!(imports.len(), 1);
|
||||
assert_eq!(imports[0].from_file_id, a);
|
||||
assert_eq!(importers_of(&conn, b).unwrap(), vec![a]);
|
||||
assert_eq!(stats(&conn).unwrap(), (2, 0, 1));
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
mod cli;
|
||||
mod config;
|
||||
mod ctags;
|
||||
mod discovery;
|
||||
mod graph;
|
||||
mod index;
|
||||
mod mcp;
|
||||
mod ops;
|
||||
mod output;
|
||||
mod search;
|
||||
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use serde_json::Value;
|
||||
|
||||
use cli::{Cli, Command, FindCommand, IndexCommand, InspectCommand, SearchCommand};
|
||||
|
||||
fn main() {
|
||||
if let Err(e) = run() {
|
||||
if let Some(io) = e.downcast_ref::<std::io::Error>()
|
||||
&& io.kind() == std::io::ErrorKind::BrokenPipe
|
||||
{
|
||||
std::process::exit(0);
|
||||
}
|
||||
eprintln!("error: {e:#}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<()> {
|
||||
let cli = Cli::parse();
|
||||
match &cli.command {
|
||||
Command::Index { sub } => match sub {
|
||||
IndexCommand::Create { path } => emit(&ops::index_create(path)?, &cli)?,
|
||||
IndexCommand::Update { path } => emit(&ops::index_update(path)?, &cli)?,
|
||||
},
|
||||
Command::Search { sub } => match sub {
|
||||
SearchCommand::Symbols { query } => {
|
||||
emit(&ops::search_symbols(&cli.project, query)?, &cli)?
|
||||
}
|
||||
SearchCommand::Text { query } => emit(&ops::search_text(&cli.project, query)?, &cli)?,
|
||||
},
|
||||
Command::Inspect { sub } => match sub {
|
||||
InspectCommand::Symbol { id } => emit(&ops::inspect_symbol(&cli.project, *id)?, &cli)?,
|
||||
InspectCommand::Outline { file } => {
|
||||
emit(&ops::inspect_outline(&cli.project, file)?, &cli)?
|
||||
}
|
||||
InspectCommand::File { file, lines } => emit(
|
||||
&ops::inspect_file(&cli.project, file, lines.as_deref())?,
|
||||
&cli,
|
||||
)?,
|
||||
InspectCommand::Tree { depth } => {
|
||||
emit(&ops::inspect_tree(&cli.project, *depth)?, &cli)?
|
||||
}
|
||||
InspectCommand::BlastRadius { file, depth } => emit(
|
||||
&ops::inspect_blast_radius(&cli.project, file, *depth)?,
|
||||
&cli,
|
||||
)?,
|
||||
InspectCommand::Cycles => emit(&ops::inspect_cycles(&cli.project)?, &cli)?,
|
||||
},
|
||||
Command::Find { sub } => match sub {
|
||||
FindCommand::Importers { file } => {
|
||||
emit(&ops::find_importers(&cli.project, file)?, &cli)?
|
||||
}
|
||||
FindCommand::Dead => emit(&ops::find_dead(&cli.project)?, &cli)?,
|
||||
},
|
||||
Command::Serve => mcp::serve(&cli.project)?,
|
||||
Command::Stats => emit(&ops::stats(&cli.project)?, &cli)?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn emit(value: &Value, cli: &Cli) -> Result<()> {
|
||||
match cli.format.as_str() {
|
||||
"table" => {
|
||||
output::print_table(value);
|
||||
Ok(())
|
||||
}
|
||||
_ => output::print_json(value, cli.compact),
|
||||
}
|
||||
}
|
||||
+254
@@ -0,0 +1,254 @@
|
||||
//! Minimal MCP server: JSON-RPC over stdio, dispatching to the same ops as the CLI.
|
||||
|
||||
use std::io::{BufRead, Write};
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const PROTOCOL_VERSION: &str = "2024-11-05";
|
||||
|
||||
pub fn serve(project: &Path) -> Result<()> {
|
||||
eprintln!("rustytokens MCP server listening on stdio");
|
||||
let stdin = std::io::stdin();
|
||||
let stdout = std::io::stdout();
|
||||
for line in stdin.lock().lines() {
|
||||
let line = match line {
|
||||
Ok(l) => l,
|
||||
Err(_) => break,
|
||||
};
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let msg: Value = match serde_json::from_str(&line) {
|
||||
Ok(m) => m,
|
||||
Err(_) => continue,
|
||||
};
|
||||
if let Some(resp) = handle(&msg, project) {
|
||||
let mut out = stdout.lock();
|
||||
serde_json::to_writer(&mut out, &resp)?;
|
||||
out.write_all(b"\n")?;
|
||||
out.flush()?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle(msg: &Value, project: &Path) -> Option<Value> {
|
||||
let method = msg.get("method")?.as_str()?;
|
||||
let id = msg.get("id").cloned();
|
||||
|
||||
// Notifications have no id -> no response.
|
||||
let id = id.as_ref()?;
|
||||
|
||||
match method {
|
||||
"initialize" => Some(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"result": {
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"capabilities": { "tools": {} },
|
||||
"serverInfo": { "name": "rustytokens", "version": env!("CARGO_PKG_VERSION") }
|
||||
}
|
||||
})),
|
||||
"ping" => Some(json!({ "jsonrpc": "2.0", "id": id, "result": {} })),
|
||||
"tools/list" => Some(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"result": { "tools": tools() }
|
||||
})),
|
||||
"tools/call" => {
|
||||
let params = msg.get("params").cloned().unwrap_or_else(|| json!({}));
|
||||
let name = params.get("name").and_then(|n| n.as_str()).unwrap_or("");
|
||||
let args = params
|
||||
.get("arguments")
|
||||
.cloned()
|
||||
.unwrap_or_else(|| json!({}));
|
||||
Some(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"result": call_tool(name, &args, project)
|
||||
}))
|
||||
}
|
||||
_ => Some(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"error": { "code": -32601, "message": format!("method not found: {method}") }
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
fn call_tool(name: &str, args: &Value, project: &Path) -> Value {
|
||||
let result: Result<Value> = dispatch(name, args, project);
|
||||
match result {
|
||||
Ok(v) => json!({
|
||||
"content": [{ "type": "text", "text": serde_json::to_string(&v).unwrap_or_default() }]
|
||||
}),
|
||||
Err(e) => json!({
|
||||
"content": [{ "type": "text", "text": format!("error: {e:#}") }],
|
||||
"isError": true
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch(name: &str, args: &Value, project: &Path) -> Result<Value> {
|
||||
use crate::ops;
|
||||
let s = |k: &str| args.get(k).and_then(|v| v.as_str()).unwrap_or("");
|
||||
match name {
|
||||
"index_create" => ops::index_create(Path::new(s("path"))),
|
||||
"index_update" => ops::index_update(Path::new(s("path"))),
|
||||
"search_symbols" => ops::search_symbols(project, s("query")),
|
||||
"search_text" => ops::search_text(project, s("query")),
|
||||
"inspect_symbol" => ops::inspect_symbol(
|
||||
project,
|
||||
args.get("id").and_then(|v| v.as_i64()).unwrap_or(0),
|
||||
),
|
||||
"inspect_outline" => ops::inspect_outline(project, s("file")),
|
||||
"inspect_file" => ops::inspect_file(
|
||||
project,
|
||||
s("file"),
|
||||
args.get("lines").and_then(|v| v.as_str()),
|
||||
),
|
||||
"inspect_tree" => ops::inspect_tree(
|
||||
project,
|
||||
args.get("depth")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|d| d as usize),
|
||||
),
|
||||
"find_importers" => ops::find_importers(project, s("file")),
|
||||
"find_dead" => ops::find_dead(project),
|
||||
"inspect_cycles" => ops::inspect_cycles(project),
|
||||
"inspect_blast_radius" => ops::inspect_blast_radius(
|
||||
project,
|
||||
s("file"),
|
||||
args.get("depth")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|d| d as usize),
|
||||
),
|
||||
"stats" => ops::stats(project),
|
||||
_ => anyhow::bail!("unknown tool: {name}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn tools() -> Value {
|
||||
json!([
|
||||
tool(
|
||||
"index_create",
|
||||
"Index a project from scratch",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "path": { "type": "string", "description": "Project directory to index" } },
|
||||
"required": ["path"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"index_update",
|
||||
"Re-index only changed files",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "path": { "type": "string" } },
|
||||
"required": ["path"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"search_symbols",
|
||||
"Search symbols by name",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "query": { "type": "string" } },
|
||||
"required": ["query"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"search_text",
|
||||
"Full-text search over file contents",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "query": { "type": "string" } },
|
||||
"required": ["query"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"inspect_symbol",
|
||||
"Get the source code of a single symbol by id",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "id": { "type": "integer" } },
|
||||
"required": ["id"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"inspect_outline",
|
||||
"Get the symbol outline of a file",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "file": { "type": "string" } },
|
||||
"required": ["file"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"inspect_file",
|
||||
"Read a file or a line range",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": { "type": "string" },
|
||||
"lines": { "type": "string", "description": "Optional range, e.g. 10-40" }
|
||||
},
|
||||
"required": ["file"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"inspect_tree",
|
||||
"Show the indexed file tree",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "depth": { "type": "integer" } }
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"find_importers",
|
||||
"List files that import the given file",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": { "file": { "type": "string" } },
|
||||
"required": ["file"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"find_dead",
|
||||
"List files with no importers",
|
||||
json!({ "type": "object" })
|
||||
),
|
||||
tool(
|
||||
"inspect_cycles",
|
||||
"Find circular imports",
|
||||
json!({ "type": "object" })
|
||||
),
|
||||
tool(
|
||||
"inspect_blast_radius",
|
||||
"Files transitively affected by a change",
|
||||
json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file": { "type": "string" },
|
||||
"depth": { "type": "integer" }
|
||||
},
|
||||
"required": ["file"]
|
||||
})
|
||||
),
|
||||
tool(
|
||||
"stats",
|
||||
"Show index statistics",
|
||||
json!({ "type": "object" })
|
||||
),
|
||||
])
|
||||
}
|
||||
|
||||
fn tool(name: &str, description: &str, input_schema: Value) -> Value {
|
||||
json!({
|
||||
"name": name,
|
||||
"description": description,
|
||||
"inputSchema": input_schema
|
||||
})
|
||||
}
|
||||
+341
@@ -0,0 +1,341 @@
|
||||
//! High-level operations shared by the CLI and the MCP server.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use rusqlite::Connection;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::config;
|
||||
use crate::graph::Graph;
|
||||
use crate::index::{pipeline, store};
|
||||
use crate::search;
|
||||
|
||||
// ---- index ----
|
||||
|
||||
pub fn index_create(project: &Path) -> Result<Value> {
|
||||
let db = config::index_db_path(project);
|
||||
let stats = pipeline::run(project, &db, true)?;
|
||||
Ok(serde_json::to_value(stats)?)
|
||||
}
|
||||
|
||||
pub fn index_update(project: &Path) -> Result<Value> {
|
||||
let db = config::index_db_path(project);
|
||||
let stats = pipeline::run(project, &db, false)?;
|
||||
Ok(serde_json::to_value(stats)?)
|
||||
}
|
||||
|
||||
// ---- search ----
|
||||
|
||||
pub fn search_symbols(project: &Path, query: &str) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
Ok(serde_json::to_value(search::search_symbols(&conn, query)?)?)
|
||||
}
|
||||
|
||||
pub fn search_text(project: &Path, query: &str) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
Ok(serde_json::to_value(search::search_text(&conn, query)?)?)
|
||||
}
|
||||
|
||||
// ---- inspect ----
|
||||
|
||||
pub fn inspect_symbol(project: &Path, id: i64) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let sym = store::symbol_by_id(&conn, id)?.with_context(|| format!("no symbol with id {id}"))?;
|
||||
let file = store::file_by_id(&conn, sym.file_id)?
|
||||
.with_context(|| format!("no file with id {}", sym.file_id))?;
|
||||
let source = read_lines(project, &file.path, sym.line_start, sym.line_end)?;
|
||||
Ok(serde_json::json!({
|
||||
"id": sym.id,
|
||||
"name": sym.name,
|
||||
"qualified_name": sym.qualified_name,
|
||||
"kind": sym.kind,
|
||||
"signature": sym.signature,
|
||||
"file": file.path,
|
||||
"line_start": sym.line_start,
|
||||
"line_end": sym.line_end,
|
||||
"source": source,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn inspect_outline(project: &Path, file: &str) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let fid = file_id(&conn, file)?;
|
||||
let symbols = store::symbols_for_file(&conn, fid)?;
|
||||
Ok(serde_json::to_value(symbols)?)
|
||||
}
|
||||
|
||||
pub fn inspect_file(project: &Path, file: &str, lines: Option<&str>) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let fid = file_id(&conn, file)?;
|
||||
let frow = store::file_by_id(&conn, fid)?.unwrap();
|
||||
let (start, end) = parse_range(lines);
|
||||
let all = read_all_lines(project, &frow.path)?;
|
||||
let total = all.len() as i64;
|
||||
let (s, e) = (start.unwrap_or(1), end.unwrap_or(total));
|
||||
let slice = all
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| {
|
||||
let n = (*i as i64) + 1;
|
||||
n >= s && n <= e
|
||||
})
|
||||
.map(|(_, l)| l.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
Ok(serde_json::json!({
|
||||
"file": frow.path,
|
||||
"language": frow.language,
|
||||
"line_start": s,
|
||||
"line_end": e.min(total),
|
||||
"total_lines": total,
|
||||
"content": slice,
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn inspect_tree(project: &Path, depth: Option<usize>) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let files = store::all_files(&conn)?;
|
||||
let paths: Vec<String> = files.into_iter().map(|f| f.path).collect();
|
||||
let root_name = project
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().into_owned())
|
||||
.unwrap_or_else(|| "project".to_string());
|
||||
let tree = build_tree(&root_name, &paths, depth);
|
||||
Ok(serde_json::to_value(tree)?)
|
||||
}
|
||||
|
||||
// ---- find / graph ----
|
||||
|
||||
pub fn find_importers(project: &Path, file: &str) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let fid = file_id(&conn, file)?;
|
||||
let importers = store::importers_of(&conn, fid)?;
|
||||
let paths = importers
|
||||
.into_iter()
|
||||
.map(|id| store::file_by_id(&conn, id).map(|f| f.map(|f| f.path)))
|
||||
.collect::<std::result::Result<Vec<_>, _>>()?
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>();
|
||||
Ok(serde_json::to_value(paths)?)
|
||||
}
|
||||
|
||||
pub fn find_dead(project: &Path) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let files = store::all_files(&conn)?;
|
||||
let ids: Vec<i64> = files.iter().map(|f| f.id).collect();
|
||||
let imports = store::all_imports(&conn)?;
|
||||
let graph = Graph::from_imports(&imports);
|
||||
let dead = graph.dead_nodes(&ids);
|
||||
let out: Vec<Value> = dead
|
||||
.into_iter()
|
||||
.filter_map(|id| {
|
||||
let f = files.iter().find(|f| f.id == id)?;
|
||||
if is_entry_point(&f.path, &f.language) {
|
||||
return None;
|
||||
}
|
||||
Some(serde_json::json!({"id": f.id, "path": f.path, "language": f.language}))
|
||||
})
|
||||
.collect();
|
||||
Ok(serde_json::json!({"count": out.len(), "files": out}))
|
||||
}
|
||||
|
||||
/// Entry-point files are legitimately never imported by anything else.
|
||||
fn is_entry_point(path: &str, language: &str) -> bool {
|
||||
let name = path.rsplit('/').next().unwrap_or(path);
|
||||
let stem = name.split('.').next().unwrap_or(name);
|
||||
if matches!(stem, "main" | "lib") {
|
||||
return true;
|
||||
}
|
||||
// Cargo compiles every file in these dirs as its own binary/crate.
|
||||
if language == "rust" {
|
||||
let top = path.split('/').next().unwrap_or(path);
|
||||
if matches!(top, "tests" | "benches" | "examples") || path.starts_with("src/bin/") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
pub fn inspect_cycles(project: &Path) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let files = store::all_files(&conn)?;
|
||||
let imports = store::all_imports(&conn)?;
|
||||
let graph = Graph::from_imports(&imports);
|
||||
let cycles = graph.cycles();
|
||||
let path_of = |id: i64| {
|
||||
files
|
||||
.iter()
|
||||
.find(|f| f.id == id)
|
||||
.map(|f| f.path.clone())
|
||||
.unwrap_or_else(|| format!("<id {id}>"))
|
||||
};
|
||||
let out: Vec<Vec<String>> = cycles
|
||||
.into_iter()
|
||||
.map(|c| c.into_iter().map(&path_of).collect())
|
||||
.collect();
|
||||
Ok(serde_json::json!({"count": out.len(), "cycles": out}))
|
||||
}
|
||||
|
||||
pub fn inspect_blast_radius(project: &Path, file: &str, depth: Option<usize>) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let fid = file_id(&conn, file)?;
|
||||
let files = store::all_files(&conn)?;
|
||||
let imports = store::all_imports(&conn)?;
|
||||
let graph = Graph::from_imports(&imports);
|
||||
let max_depth = depth.unwrap_or(3);
|
||||
let hits = graph.blast_radius(fid, max_depth);
|
||||
let out: Vec<Value> = hits
|
||||
.into_iter()
|
||||
.filter_map(|(id, d)| {
|
||||
files
|
||||
.iter()
|
||||
.find(|f| f.id == id)
|
||||
.map(|f| serde_json::json!({"id": f.id, "path": f.path, "depth": d}))
|
||||
})
|
||||
.collect();
|
||||
Ok(serde_json::json!({"file": file, "count": out.len(), "affected": out}))
|
||||
}
|
||||
|
||||
// ---- stats ----
|
||||
|
||||
pub fn stats(project: &Path) -> Result<Value> {
|
||||
let conn = open_read(project)?;
|
||||
let (files, symbols, imports) = store::stats(&conn)?;
|
||||
let langs = store::language_counts(&conn)?;
|
||||
Ok(serde_json::json!({
|
||||
"files": files,
|
||||
"symbols": symbols,
|
||||
"imports": imports,
|
||||
"languages": langs,
|
||||
}))
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
fn open_read(project: &Path) -> Result<Connection> {
|
||||
let db = config::index_db_path(project);
|
||||
if !db.exists() {
|
||||
bail!(
|
||||
"no index found for {}. Run `rustytokens index create <path>` first.",
|
||||
project.display()
|
||||
);
|
||||
}
|
||||
store::open(&db)
|
||||
}
|
||||
|
||||
fn file_id(conn: &Connection, path: &str) -> Result<i64> {
|
||||
store::file_id_by_path(conn, path)?.with_context(|| format!("file not indexed: {path}"))
|
||||
}
|
||||
|
||||
fn read_lines(project: &Path, rel: &str, start: i64, end: i64) -> Result<String> {
|
||||
let all = read_all_lines(project, rel)?;
|
||||
Ok(all
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(i, _)| {
|
||||
let n = (*i as i64) + 1;
|
||||
n >= start && n <= end
|
||||
})
|
||||
.map(|(_, l)| l.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n"))
|
||||
}
|
||||
|
||||
fn read_all_lines(project: &Path, rel: &str) -> Result<Vec<String>> {
|
||||
let path = project.join(rel);
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.with_context(|| format!("cannot read {}", path.display()))?;
|
||||
Ok(content.lines().map(|l| l.to_string()).collect())
|
||||
}
|
||||
|
||||
fn parse_range(lines: Option<&str>) -> (Option<i64>, Option<i64>) {
|
||||
let Some(s) = lines else { return (None, None) };
|
||||
let parts: Vec<&str> = s.split('-').collect();
|
||||
let start = parts.first().and_then(|p| p.trim().parse::<i64>().ok());
|
||||
let end = parts.get(1).and_then(|p| p.trim().parse::<i64>().ok());
|
||||
(start, end)
|
||||
}
|
||||
|
||||
// ---- tree ----
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TreeNode {
|
||||
name: String,
|
||||
path: String,
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
children: Vec<TreeNode>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct RawNode {
|
||||
dirs: BTreeMap<String, RawNode>,
|
||||
files: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
fn build_tree(root_name: &str, paths: &[String], depth: Option<usize>) -> TreeNode {
|
||||
let mut root = RawNode::default();
|
||||
for p in paths {
|
||||
let segs: Vec<&str> = p.split('/').collect();
|
||||
insert_raw(&mut root, &segs, p);
|
||||
}
|
||||
let children = raw_to_nodes(&root, "", depth);
|
||||
TreeNode {
|
||||
name: root_name.to_string(),
|
||||
path: String::new(),
|
||||
kind: "dir".to_string(),
|
||||
children,
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_raw(node: &mut RawNode, segs: &[&str], full: &str) {
|
||||
match segs {
|
||||
[head] => {
|
||||
node.files
|
||||
.entry((*head).to_string())
|
||||
.or_insert_with(|| full.to_string());
|
||||
}
|
||||
[head, rest @ ..] => {
|
||||
let child = node.dirs.entry((*head).to_string()).or_default();
|
||||
insert_raw(child, rest, full);
|
||||
}
|
||||
[] => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_to_nodes(node: &RawNode, base: &str, depth: Option<usize>) -> Vec<TreeNode> {
|
||||
let mut out = Vec::new();
|
||||
for (name, full) in &node.files {
|
||||
out.push(TreeNode {
|
||||
name: name.clone(),
|
||||
path: full.clone(),
|
||||
kind: "file".to_string(),
|
||||
children: Vec::new(),
|
||||
});
|
||||
}
|
||||
for (name, child) in &node.dirs {
|
||||
let path = if base.is_empty() {
|
||||
name.clone()
|
||||
} else {
|
||||
format!("{base}/{name}")
|
||||
};
|
||||
let expand = depth.map(|d| d > 0).unwrap_or(true);
|
||||
let children = if expand {
|
||||
raw_to_nodes(child, &path, depth.map(|d| d.saturating_sub(1)))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
out.push(TreeNode {
|
||||
name: name.clone(),
|
||||
path,
|
||||
kind: "dir".to_string(),
|
||||
children,
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
//! Output formatting: JSON (default) or a minimal table.
|
||||
|
||||
use anyhow::Result;
|
||||
use serde::Serialize;
|
||||
use serde_json::Value;
|
||||
use std::io::Write;
|
||||
|
||||
pub fn print_json<T: Serialize>(value: &T, compact: bool) -> Result<()> {
|
||||
let mut buf = Vec::new();
|
||||
if compact {
|
||||
serde_json::to_writer(&mut buf, value)?;
|
||||
} else {
|
||||
serde_json::to_writer_pretty(&mut buf, value)?;
|
||||
}
|
||||
let mut out = std::io::stdout().lock();
|
||||
out.write_all(&buf)?;
|
||||
out.write_all(b"\n")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn putline(s: &str) {
|
||||
use std::io::Write;
|
||||
let _ = writeln!(std::io::stdout().lock(), "{s}");
|
||||
}
|
||||
|
||||
/// Minimal table renderer for array-of-objects and plain objects.
|
||||
pub fn print_table(value: &Value) {
|
||||
match value {
|
||||
Value::Array(items) if !items.is_empty() && items.iter().all(|i| i.is_object()) => {
|
||||
let cols = collect_columns(items);
|
||||
let mut rows = vec![cols.clone()];
|
||||
for item in items {
|
||||
let obj = item.as_object().unwrap();
|
||||
rows.push(cols.iter().map(|c| cell_string(obj.get(c))).collect());
|
||||
}
|
||||
print_grid(&rows);
|
||||
}
|
||||
Value::Object(obj) => {
|
||||
let mut rows = vec![vec!["key".to_string(), "value".to_string()]];
|
||||
for (k, v) in obj {
|
||||
rows.push(vec![k.clone(), cell_string(Some(v))]);
|
||||
}
|
||||
print_grid(&rows);
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for i in items {
|
||||
putline(&cell_string(Some(i)));
|
||||
}
|
||||
}
|
||||
other => putline(&cell_string(Some(other))),
|
||||
}
|
||||
}
|
||||
|
||||
fn collect_columns(items: &[Value]) -> Vec<String> {
|
||||
let mut cols: Vec<String> = Vec::new();
|
||||
for item in items {
|
||||
if let Some(obj) = item.as_object() {
|
||||
for k in obj.keys() {
|
||||
if !cols.contains(k) {
|
||||
cols.push(k.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
cols
|
||||
}
|
||||
|
||||
fn cell_string(v: Option<&Value>) -> String {
|
||||
match v {
|
||||
None | Some(Value::Null) => String::new(),
|
||||
Some(Value::String(s)) => s.clone(),
|
||||
Some(other) => serde_json::to_string(other).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn print_grid(rows: &[Vec<String>]) {
|
||||
let n = rows.iter().map(|r| r.len()).max().unwrap_or(0);
|
||||
let mut widths = vec![0usize; n];
|
||||
for r in rows {
|
||||
for (i, c) in r.iter().enumerate() {
|
||||
widths[i] = widths[i].max(c.chars().count());
|
||||
}
|
||||
}
|
||||
for (ri, r) in rows.iter().enumerate() {
|
||||
let mut line = String::new();
|
||||
for (i, w) in widths.iter().enumerate() {
|
||||
let c = r.get(i).cloned().unwrap_or_default();
|
||||
line.push_str(&c);
|
||||
if i + 1 < n {
|
||||
line.push_str(&" ".repeat(w.saturating_sub(c.chars().count())));
|
||||
line.push_str(" ");
|
||||
}
|
||||
}
|
||||
putline(line.trim_end());
|
||||
if ri == 0 {
|
||||
let sep: Vec<String> = widths.iter().map(|w| "-".repeat(*w)).collect();
|
||||
putline(sep.join(" ").trim_end());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
//! FTS5-backed search with a simple own ranking (exact > prefix > fulltext).
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::Result;
|
||||
use rusqlite::{Connection, params};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct SymbolHit {
|
||||
pub id: i64,
|
||||
pub name: String,
|
||||
pub qualified_name: String,
|
||||
pub kind: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub signature: Option<String>,
|
||||
pub file: String,
|
||||
pub line_start: i64,
|
||||
pub line_end: i64,
|
||||
pub match_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct TextHit {
|
||||
pub file: String,
|
||||
pub snippet: String,
|
||||
}
|
||||
|
||||
const SYMBOL_COLS: &str =
|
||||
"s.id, s.name, s.qualified_name, s.kind, s.signature, f.path, s.line_start, s.line_end";
|
||||
|
||||
pub fn search_symbols(conn: &Connection, query: &str) -> Result<Vec<SymbolHit>> {
|
||||
let q = query.trim();
|
||||
let mut seen: HashSet<i64> = HashSet::new();
|
||||
let mut out: Vec<SymbolHit> = Vec::new();
|
||||
if q.is_empty() {
|
||||
return Ok(out);
|
||||
}
|
||||
|
||||
// 1) exact name (case-insensitive)
|
||||
{
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
|
||||
WHERE s.name = ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
|
||||
))?;
|
||||
let rows = stmt.query_map(params![q], |r| row_to_hit(r, "exact"))?;
|
||||
push_hits(&mut out, &mut seen, rows)?;
|
||||
}
|
||||
|
||||
// 2) prefix
|
||||
{
|
||||
let pat = format!("{}%", q);
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
|
||||
WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
|
||||
))?;
|
||||
let rows = stmt.query_map(params![pat], |r| row_to_hit(r, "prefix"))?;
|
||||
push_hits(&mut out, &mut seen, rows)?;
|
||||
}
|
||||
|
||||
// 3) fulltext over name/qualified_name/kind/signature
|
||||
{
|
||||
let phrase = fts_phrase(q);
|
||||
let mut stmt = conn.prepare(&format!(
|
||||
"SELECT {SYMBOL_COLS} FROM symbols_fts \
|
||||
JOIN symbols s ON s.id = symbols_fts.rowid \
|
||||
JOIN files f ON f.id = s.file_id \
|
||||
WHERE symbols_fts MATCH ?1 ORDER BY rank LIMIT 200"
|
||||
))?;
|
||||
let rows = stmt.query_map(params![phrase], |r| row_to_hit(r, "fulltext"))?;
|
||||
push_hits(&mut out, &mut seen, rows)?;
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn search_text(conn: &Connection, query: &str) -> Result<Vec<TextHit>> {
|
||||
let q = query.trim();
|
||||
if q.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let phrase = fts_phrase(q);
|
||||
let mut stmt = conn.prepare(
|
||||
"SELECT path, snippet(content_fts, 1, '«', '»', '…', 24) \
|
||||
FROM content_fts WHERE content_fts MATCH ?1 ORDER BY rank LIMIT 100",
|
||||
)?;
|
||||
let rows = stmt.query_map(params![phrase], |r| {
|
||||
Ok(TextHit {
|
||||
file: r.get(0)?,
|
||||
snippet: r.get(1)?,
|
||||
})
|
||||
})?;
|
||||
Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||
}
|
||||
|
||||
fn row_to_hit(
|
||||
r: &rusqlite::Row<'_>,
|
||||
match_type: &str,
|
||||
) -> std::result::Result<SymbolHit, rusqlite::Error> {
|
||||
Ok(SymbolHit {
|
||||
id: r.get(0)?,
|
||||
name: r.get(1)?,
|
||||
qualified_name: r.get(2)?,
|
||||
kind: r.get(3)?,
|
||||
signature: r.get(4)?,
|
||||
file: r.get(5)?,
|
||||
line_start: r.get(6)?,
|
||||
line_end: r.get(7)?,
|
||||
match_type: match_type.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn push_hits(
|
||||
out: &mut Vec<SymbolHit>,
|
||||
seen: &mut HashSet<i64>,
|
||||
rows: impl Iterator<Item = std::result::Result<SymbolHit, rusqlite::Error>>,
|
||||
) -> Result<()> {
|
||||
for r in rows {
|
||||
let hit = r?;
|
||||
if seen.insert(hit.id) {
|
||||
out.push(hit);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wrap a user query as a safe FTS5 phrase.
|
||||
fn fts_phrase(q: &str) -> String {
|
||||
format!("\"{}\"", q.replace('"', "\"\""))
|
||||
}
|
||||
+480
@@ -0,0 +1,480 @@
|
||||
//! End-to-end CLI tests against small, self-created fixture repos.
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
fn bin() -> &'static str {
|
||||
env!("CARGO_BIN_EXE_rustytokens")
|
||||
}
|
||||
|
||||
/// Shared temp cache dir so tests never touch the real ~/.cache/rustytokens.
|
||||
fn cache_dir() -> &'static Path {
|
||||
static DIR: OnceLock<PathBuf> = OnceLock::new();
|
||||
DIR.get_or_init(|| {
|
||||
let d = std::env::temp_dir().join(format!("rustytokens-test-cache-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
})
|
||||
}
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
fn copy_dir(src: &Path, dst: &Path) {
|
||||
std::fs::create_dir_all(dst).unwrap();
|
||||
for entry in std::fs::read_dir(src).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let to = dst.join(entry.file_name());
|
||||
if entry.path().is_dir() {
|
||||
copy_dir(&entry.path(), &to);
|
||||
} else {
|
||||
std::fs::copy(entry.path(), &to).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the binary with the fixture's cache dir, returning parsed stdout JSON.
|
||||
fn run_json(args: &[&str]) -> Value {
|
||||
let out = Command::new(bin())
|
||||
.args(args)
|
||||
.env("XDG_CACHE_HOME", cache_dir())
|
||||
.output()
|
||||
.expect("run rustytokens");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"command failed: {:?}\nstderr: {}",
|
||||
args,
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
serde_json::from_slice(&out.stdout).expect("stdout is valid JSON")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rust_index_search_and_dead() {
|
||||
let fx = fixture("rustdemo");
|
||||
run_json(&["index", "create", fx.to_str().unwrap()]);
|
||||
|
||||
let hits = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"add",
|
||||
]);
|
||||
let hit = hits
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|v| v["name"] == "add")
|
||||
.expect("search finds `add`");
|
||||
assert_eq!(hit["kind"], "function");
|
||||
|
||||
// inspect symbol by id returns the exact source.
|
||||
let id = hit["id"].as_i64().unwrap();
|
||||
let sym = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"inspect",
|
||||
"symbol",
|
||||
&id.to_string(),
|
||||
]);
|
||||
assert!(
|
||||
sym["source"].as_str().unwrap().contains("a + b"),
|
||||
"source contains function body"
|
||||
);
|
||||
|
||||
// dead.rs is flagged; entry points lib.rs/main.rs are not.
|
||||
let dead = run_json(&["--project", fx.to_str().unwrap(), "find", "dead"]);
|
||||
let paths: Vec<&str> = dead["files"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|f| f["path"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(paths.contains(&"src/dead.rs"), "dead.rs flagged");
|
||||
assert!(!paths.contains(&"src/lib.rs"), "lib.rs is an entry point");
|
||||
assert!(!paths.contains(&"src/main.rs"), "main.rs is an entry point");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inspect_blast_radius_finds_affected_files() {
|
||||
let src = fixture("rustdemo");
|
||||
let tmp = cache_dir().join("blast-rustdemo");
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
copy_dir(&src, &tmp);
|
||||
run_json(&["index", "create", tmp.to_str().unwrap()]);
|
||||
|
||||
let blast = run_json(&[
|
||||
"--project",
|
||||
tmp.to_str().unwrap(),
|
||||
"inspect",
|
||||
"blast-radius",
|
||||
"src/util.rs",
|
||||
]);
|
||||
assert_eq!(blast["count"], 1);
|
||||
assert_eq!(blast["file"], "src/util.rs");
|
||||
let affected: Vec<&str> = blast["affected"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|f| f["path"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(affected, ["src/lib.rs"], "lib.rs imports util.rs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn python_import_resolution_and_dead() {
|
||||
let fx = fixture("pydemo");
|
||||
run_json(&["index", "create", fx.to_str().unwrap()]);
|
||||
|
||||
let importers = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"util.py",
|
||||
]);
|
||||
let names: Vec<&str> = importers
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(names.contains(&"main.py"), "main.py imports util.py");
|
||||
|
||||
let dead = run_json(&["--project", fx.to_str().unwrap(), "find", "dead"]);
|
||||
let paths: Vec<&str> = dead["files"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|f| f["path"].as_str().unwrap())
|
||||
.collect();
|
||||
assert!(paths.contains(&"dead.py"), "dead.py flagged");
|
||||
assert!(!paths.contains(&"util.py"), "util.py is imported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn typescript_cycle_detection() {
|
||||
let fx = fixture("tsdemo");
|
||||
run_json(&["index", "create", fx.to_str().unwrap()]);
|
||||
|
||||
let cycles = run_json(&["--project", fx.to_str().unwrap(), "inspect", "cycles"]);
|
||||
assert_eq!(cycles["count"], 1, "one cycle");
|
||||
let cycle = &cycles["cycles"][0];
|
||||
let files: Vec<&str> = cycle
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(
|
||||
files.contains(&"a.ts") && files.contains(&"b.ts"),
|
||||
"a↔b cycle"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn java_symbols_and_import_resolution() {
|
||||
let fx = fixture("javademo");
|
||||
run_json(&["index", "create", fx.to_str().unwrap()]);
|
||||
|
||||
let hits = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"helper",
|
||||
]);
|
||||
let hit = hits
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|v| v["name"] == "helper")
|
||||
.expect("search finds `helper`");
|
||||
assert_eq!(hit["kind"], "method");
|
||||
|
||||
// ctags reports Java constructors as `method` with a full `end` span, so
|
||||
// the multi-line `Main` constructor must carry line_end > line_start.
|
||||
let ctor = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"Main",
|
||||
]);
|
||||
let ctor_hit = ctor
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|v| v["name"] == "Main" && v["kind"] == "method")
|
||||
.expect("search finds `Main` constructor");
|
||||
assert!(
|
||||
ctor_hit["line_end"].as_i64().unwrap() > ctor_hit["line_start"].as_i64().unwrap(),
|
||||
"multi-line constructor has an extended span"
|
||||
);
|
||||
|
||||
let importers = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"Util.java",
|
||||
]);
|
||||
let names: Vec<&str> = importers
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(names.contains(&"Main.java"), "Main.java imports Util.java");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn c_and_cpp_symbols_and_imports() {
|
||||
let cdemo = fixture("cdemo");
|
||||
run_json(&["index", "create", cdemo.to_str().unwrap()]);
|
||||
|
||||
let hits = run_json(&[
|
||||
"--project",
|
||||
cdemo.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"helper",
|
||||
]);
|
||||
assert!(
|
||||
hits.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|v| v["name"] == "helper"),
|
||||
"c demo indexes `helper`"
|
||||
);
|
||||
|
||||
let importers = run_json(&[
|
||||
"--project",
|
||||
cdemo.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"util.h",
|
||||
]);
|
||||
let names: Vec<&str> = importers
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(names.contains(&"main.c") && names.contains(&"util.c"));
|
||||
|
||||
let cppdemo = fixture("cppdemo");
|
||||
run_json(&["index", "create", cppdemo.to_str().unwrap()]);
|
||||
|
||||
let hits = run_json(&[
|
||||
"--project",
|
||||
cppdemo.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"add",
|
||||
]);
|
||||
assert!(
|
||||
hits.as_array().unwrap().iter().any(|v| v["name"] == "add"),
|
||||
"cpp demo indexes `add`"
|
||||
);
|
||||
|
||||
// ctags reports C++ constructors/destructors as `function` with an `end`
|
||||
// span; the multi-line `Math` ctor must carry line_end > line_start.
|
||||
let ctor = run_json(&[
|
||||
"--project",
|
||||
cppdemo.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"Math",
|
||||
]);
|
||||
let ctor_hit = ctor
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|v| v["name"] == "Math" && v["kind"] == "function")
|
||||
.expect("search finds `Math` constructor");
|
||||
assert_eq!(ctor_hit["kind"], "function");
|
||||
assert!(
|
||||
ctor_hit["line_end"].as_i64().unwrap() > ctor_hit["line_start"].as_i64().unwrap(),
|
||||
"multi-line constructor has an extended span"
|
||||
);
|
||||
|
||||
// Destructor is also `function` with a ctags-owned `end` span (name `~Math`).
|
||||
let dtor_hit = ctor
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|v| v["name"] == "~Math" && v["kind"] == "function")
|
||||
.expect("search finds `~Math` destructor");
|
||||
assert!(
|
||||
dtor_hit["line_end"].as_i64().unwrap() > dtor_hit["line_start"].as_i64().unwrap(),
|
||||
"multi-line destructor has an extended span"
|
||||
);
|
||||
|
||||
let importers = run_json(&[
|
||||
"--project",
|
||||
cppdemo.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"math.hpp",
|
||||
]);
|
||||
let names: Vec<&str> = importers
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(
|
||||
names.contains(&"main.cpp") && names.contains(&"math.cpp"),
|
||||
"both cpp files include math.hpp"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_assets_are_discovered_and_imported() {
|
||||
let fx = fixture("htmldemo");
|
||||
run_json(&["index", "create", fx.to_str().unwrap()]);
|
||||
|
||||
let importers = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"app.js",
|
||||
]);
|
||||
let names: Vec<&str> = importers
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(
|
||||
names.contains(&"index.html"),
|
||||
"index.html imports app.js via <script src>"
|
||||
);
|
||||
|
||||
let css = run_json(&[
|
||||
"--project",
|
||||
fx.to_str().unwrap(),
|
||||
"find",
|
||||
"importers",
|
||||
"style.css",
|
||||
]);
|
||||
let names: Vec<&str> = css
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_str().unwrap())
|
||||
.collect();
|
||||
assert!(
|
||||
names.contains(&"index.html"),
|
||||
"index.html imports style.css via <link href>"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mcp_serve_handles_tool_call() {
|
||||
let src = fixture("rustdemo");
|
||||
let tmp = cache_dir().join("mcp-rustdemo");
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
copy_dir(&src, &tmp);
|
||||
run_json(&["index", "create", tmp.to_str().unwrap()]);
|
||||
|
||||
let mut child = Command::new(bin())
|
||||
.args(["--project", tmp.to_str().unwrap(), "serve"])
|
||||
.env("XDG_CACHE_HOME", cache_dir())
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("spawn serve");
|
||||
|
||||
{
|
||||
let mut stdin = child.stdin.take().unwrap();
|
||||
writeln!(stdin, r#"{{"jsonrpc":"2.0","id":1,"method":"initialize"}}"#).unwrap();
|
||||
writeln!(
|
||||
stdin,
|
||||
r#"{{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{{"name":"search_symbols","arguments":{{"query":"add"}}}}}}"#
|
||||
)
|
||||
.unwrap();
|
||||
} // drop stdin -> closes the pipe, server exits
|
||||
|
||||
let out = child.wait_with_output().unwrap();
|
||||
assert!(out.status.success(), "serve exits cleanly");
|
||||
|
||||
let responses: Vec<Value> = String::from_utf8_lossy(&out.stdout)
|
||||
.lines()
|
||||
.filter_map(|l| serde_json::from_str(l).ok())
|
||||
.collect();
|
||||
let init = responses
|
||||
.iter()
|
||||
.find(|r| r["id"] == 1)
|
||||
.expect("initialize response");
|
||||
assert_eq!(init["result"]["serverInfo"]["name"], "rustytokens");
|
||||
|
||||
let call = responses
|
||||
.iter()
|
||||
.find(|r| r["id"] == 2)
|
||||
.expect("tools/call response");
|
||||
let text = call["result"]["content"][0]["text"].as_str().unwrap();
|
||||
let hits: Value = serde_json::from_str(text).unwrap();
|
||||
assert!(
|
||||
hits.as_array().unwrap().iter().any(|h| h["name"] == "add"),
|
||||
"search_symbols over MCP finds `add`"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_update_reindexes_changed_files() {
|
||||
let src = fixture("javademo");
|
||||
let tmp = cache_dir().join("update-javademo");
|
||||
let _ = std::fs::remove_dir_all(&tmp);
|
||||
copy_dir(&src, &tmp);
|
||||
|
||||
run_json(&["index", "create", tmp.to_str().unwrap()]);
|
||||
|
||||
let before = run_json(&[
|
||||
"--project",
|
||||
tmp.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"brandNew",
|
||||
]);
|
||||
assert!(
|
||||
before.as_array().unwrap().is_empty(),
|
||||
"new symbol absent before update"
|
||||
);
|
||||
|
||||
let lib = tmp.join("BrandNew.java");
|
||||
std::fs::write(
|
||||
&lib,
|
||||
"package demo;\n\npublic class BrandNew {\n public static int brandNew() { return 7; }\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
run_json(&["index", "update", tmp.to_str().unwrap()]);
|
||||
|
||||
let after = run_json(&[
|
||||
"--project",
|
||||
tmp.to_str().unwrap(),
|
||||
"search",
|
||||
"symbols",
|
||||
"brandNew",
|
||||
]);
|
||||
assert!(
|
||||
after
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|h| h["name"] == "brandNew"),
|
||||
"index update picks up the new symbol"
|
||||
);
|
||||
}
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
int c_dead(void) { return 0; }
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
#include "util.h"
|
||||
|
||||
int main(void) {
|
||||
return helper();
|
||||
}
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
#include "util.h"
|
||||
|
||||
int helper(void) { return 0; }
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
int helper(void);
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
#include "math.hpp"
|
||||
|
||||
int main() {
|
||||
return Math::add(1, 2);
|
||||
}
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
#include "math.hpp"
|
||||
|
||||
Math::Math(int scale) : scale_(scale) {
|
||||
scale_ = scale;
|
||||
}
|
||||
|
||||
Math::~Math() {
|
||||
scale_ = 0;
|
||||
}
|
||||
|
||||
int Math::add(int a, int b) { return a + b; }
|
||||
|
||||
int Math::scaled(int v) const { return v * scale_; }
|
||||
Vendored
+11
@@ -0,0 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
class Math {
|
||||
public:
|
||||
Math(int scale);
|
||||
~Math();
|
||||
static int add(int a, int b);
|
||||
int scaled(int v) const;
|
||||
private:
|
||||
int scale_;
|
||||
};
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
console.log("hi");
|
||||
Vendored
+10
@@ -0,0 +1,10 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
<script src="app.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Hello</h1>
|
||||
</body>
|
||||
</html>
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
body { color: red; }
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
package demo;
|
||||
|
||||
public class Dead {
|
||||
public static void unused() {}
|
||||
}
|
||||
Vendored
+15
@@ -0,0 +1,15 @@
|
||||
package demo;
|
||||
|
||||
import demo.Util;
|
||||
|
||||
public class Main {
|
||||
private final int seed;
|
||||
|
||||
public Main(int seed) {
|
||||
this.seed = seed;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(Util.helper());
|
||||
}
|
||||
}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
package demo;
|
||||
|
||||
public class Util {
|
||||
public static int helper() { return 42; }
|
||||
}
|
||||
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
def py_dead():
|
||||
return 0
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
from util import helper
|
||||
|
||||
|
||||
def run():
|
||||
return helper()
|
||||
Vendored
+2
@@ -0,0 +1,2 @@
|
||||
def helper():
|
||||
return 1
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
pub fn totally_dead() {}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
pub mod util;
|
||||
|
||||
pub fn add(a: i32, b: i32) -> i32 {
|
||||
a + b
|
||||
}
|
||||
Vendored
+5
@@ -0,0 +1,5 @@
|
||||
use rustdemo::add;
|
||||
|
||||
fn main() {
|
||||
println!("{}", add(2, 3));
|
||||
}
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
pub fn helper() -> String {
|
||||
"ok".to_string()
|
||||
}
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
import { b } from "./b";
|
||||
|
||||
export const a = 1;
|
||||
Vendored
+3
@@ -0,0 +1,3 @@
|
||||
import { a } from "./a";
|
||||
|
||||
export const b = a;
|
||||
Reference in new issue
Block a user