From 354132e8874185bce6228d55b0489303ee0abc2f Mon Sep 17 00:00:00 2001 From: endergebnis Date: Fri, 4 Sep 2026 02:43:00 +0200 Subject: [PATCH] =?UTF-8?q?Initial=20commit:=20rustytokens=20=E2=80=94=20n?= =?UTF-8?q?ative=20codebase=20indexer=20+=20MCP=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 1 + Cargo.lock | 591 ++++++++++++++++++++++++++++ Cargo.toml | 22 ++ README.md | 172 ++++++++ src/cli.rs | 107 +++++ src/config.rs | 81 ++++ src/ctags/mod.rs | 65 +++ src/discovery/mod.rs | 59 +++ src/graph/mod.rs | 177 +++++++++ src/index/hashing.rs | 23 ++ src/index/imports.rs | 398 +++++++++++++++++++ src/index/mod.rs | 5 + src/index/pipeline.rs | 209 ++++++++++ src/index/schema.rs | 71 ++++ src/index/store.rs | 311 +++++++++++++++ src/main.rs | 81 ++++ src/mcp/mod.rs | 254 ++++++++++++ src/ops.rs | 341 ++++++++++++++++ src/output.rs | 100 +++++ src/search/mod.rs | 130 ++++++ tests/cli.rs | 480 ++++++++++++++++++++++ tests/fixtures/cdemo/dead.c | 1 + tests/fixtures/cdemo/main.c | 5 + tests/fixtures/cdemo/util.c | 3 + tests/fixtures/cdemo/util.h | 1 + tests/fixtures/cppdemo/main.cpp | 5 + tests/fixtures/cppdemo/math.cpp | 13 + tests/fixtures/cppdemo/math.hpp | 11 + tests/fixtures/htmldemo/app.js | 1 + tests/fixtures/htmldemo/index.html | 10 + tests/fixtures/htmldemo/style.css | 1 + tests/fixtures/javademo/Dead.java | 5 + tests/fixtures/javademo/Main.java | 15 + tests/fixtures/javademo/Util.java | 5 + tests/fixtures/pydemo/dead.py | 2 + tests/fixtures/pydemo/main.py | 5 + tests/fixtures/pydemo/util.py | 2 + tests/fixtures/rustdemo/src/dead.rs | 1 + tests/fixtures/rustdemo/src/lib.rs | 5 + tests/fixtures/rustdemo/src/main.rs | 5 + tests/fixtures/rustdemo/src/util.rs | 3 + tests/fixtures/tsdemo/a.ts | 3 + tests/fixtures/tsdemo/b.ts | 3 + 43 files changed, 3783 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 src/cli.rs create mode 100644 src/config.rs create mode 100644 src/ctags/mod.rs create mode 100644 src/discovery/mod.rs create mode 100644 src/graph/mod.rs create mode 100644 src/index/hashing.rs create mode 100644 src/index/imports.rs create mode 100644 src/index/mod.rs create mode 100644 src/index/pipeline.rs create mode 100644 src/index/schema.rs create mode 100644 src/index/store.rs create mode 100644 src/main.rs create mode 100644 src/mcp/mod.rs create mode 100644 src/ops.rs create mode 100644 src/output.rs create mode 100644 src/search/mod.rs create mode 100644 tests/cli.rs create mode 100644 tests/fixtures/cdemo/dead.c create mode 100644 tests/fixtures/cdemo/main.c create mode 100644 tests/fixtures/cdemo/util.c create mode 100644 tests/fixtures/cdemo/util.h create mode 100644 tests/fixtures/cppdemo/main.cpp create mode 100644 tests/fixtures/cppdemo/math.cpp create mode 100644 tests/fixtures/cppdemo/math.hpp create mode 100644 tests/fixtures/htmldemo/app.js create mode 100644 tests/fixtures/htmldemo/index.html create mode 100644 tests/fixtures/htmldemo/style.css create mode 100644 tests/fixtures/javademo/Dead.java create mode 100644 tests/fixtures/javademo/Main.java create mode 100644 tests/fixtures/javademo/Util.java create mode 100644 tests/fixtures/pydemo/dead.py create mode 100644 tests/fixtures/pydemo/main.py create mode 100644 tests/fixtures/pydemo/util.py create mode 100644 tests/fixtures/rustdemo/src/dead.rs create mode 100644 tests/fixtures/rustdemo/src/lib.rs create mode 100644 tests/fixtures/rustdemo/src/main.rs create mode 100644 tests/fixtures/rustdemo/src/util.rs create mode 100644 tests/fixtures/tsdemo/a.ts create mode 100644 tests/fixtures/tsdemo/b.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c92d91a --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,591 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ignore" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustytokens" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "ignore", + "rayon", + "regex", + "rusqlite", + "serde", + "serde_json", + "xxhash-rust", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..441e039 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustytokens" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4", features = ["derive"] } +rusqlite = { version = "0.31", features = ["bundled"] } +ignore = "0.4" +rayon = "1" +xxhash-rust = { version = "0.8", features = ["xxh3"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +anyhow = "1" +regex = "1" + +[dev-dependencies] +serde_json = "1" + +[profile.release] +lto = true +strip = true diff --git a/README.md b/README.md new file mode 100644 index 0000000..672d313 --- /dev/null +++ b/README.md @@ -0,0 +1,172 @@ +# rustytokens + +A native Rust CLI + MCP server that indexes a codebase with +[`universal-ctags`](https://github.com/universal-ctags/ctags) and lets an AI +coding agent retrieve **symbols and functions** instead of whole files — saving +context tokens. + +Language coverage (via ctags): Rust, Java, Python, C/C++, TypeScript, +JavaScript, HTML. HTML import edges are extracted directly (not via ctags), and +``/` + + +

Hello

+ + diff --git a/tests/fixtures/htmldemo/style.css b/tests/fixtures/htmldemo/style.css new file mode 100644 index 0000000..991912d --- /dev/null +++ b/tests/fixtures/htmldemo/style.css @@ -0,0 +1 @@ +body { color: red; } diff --git a/tests/fixtures/javademo/Dead.java b/tests/fixtures/javademo/Dead.java new file mode 100644 index 0000000..48a6329 --- /dev/null +++ b/tests/fixtures/javademo/Dead.java @@ -0,0 +1,5 @@ +package demo; + +public class Dead { + public static void unused() {} +} diff --git a/tests/fixtures/javademo/Main.java b/tests/fixtures/javademo/Main.java new file mode 100644 index 0000000..73ae77c --- /dev/null +++ b/tests/fixtures/javademo/Main.java @@ -0,0 +1,15 @@ +package demo; + +import demo.Util; + +public class Main { + private final int seed; + + public Main(int seed) { + this.seed = seed; + } + + public static void main(String[] args) { + System.out.println(Util.helper()); + } +} diff --git a/tests/fixtures/javademo/Util.java b/tests/fixtures/javademo/Util.java new file mode 100644 index 0000000..656f099 --- /dev/null +++ b/tests/fixtures/javademo/Util.java @@ -0,0 +1,5 @@ +package demo; + +public class Util { + public static int helper() { return 42; } +} diff --git a/tests/fixtures/pydemo/dead.py b/tests/fixtures/pydemo/dead.py new file mode 100644 index 0000000..9e7a5a9 --- /dev/null +++ b/tests/fixtures/pydemo/dead.py @@ -0,0 +1,2 @@ +def py_dead(): + return 0 diff --git a/tests/fixtures/pydemo/main.py b/tests/fixtures/pydemo/main.py new file mode 100644 index 0000000..bc851f8 --- /dev/null +++ b/tests/fixtures/pydemo/main.py @@ -0,0 +1,5 @@ +from util import helper + + +def run(): + return helper() diff --git a/tests/fixtures/pydemo/util.py b/tests/fixtures/pydemo/util.py new file mode 100644 index 0000000..4c8b581 --- /dev/null +++ b/tests/fixtures/pydemo/util.py @@ -0,0 +1,2 @@ +def helper(): + return 1 diff --git a/tests/fixtures/rustdemo/src/dead.rs b/tests/fixtures/rustdemo/src/dead.rs new file mode 100644 index 0000000..7c1e328 --- /dev/null +++ b/tests/fixtures/rustdemo/src/dead.rs @@ -0,0 +1 @@ +pub fn totally_dead() {} diff --git a/tests/fixtures/rustdemo/src/lib.rs b/tests/fixtures/rustdemo/src/lib.rs new file mode 100644 index 0000000..f3257c4 --- /dev/null +++ b/tests/fixtures/rustdemo/src/lib.rs @@ -0,0 +1,5 @@ +pub mod util; + +pub fn add(a: i32, b: i32) -> i32 { + a + b +} diff --git a/tests/fixtures/rustdemo/src/main.rs b/tests/fixtures/rustdemo/src/main.rs new file mode 100644 index 0000000..07cc682 --- /dev/null +++ b/tests/fixtures/rustdemo/src/main.rs @@ -0,0 +1,5 @@ +use rustdemo::add; + +fn main() { + println!("{}", add(2, 3)); +} diff --git a/tests/fixtures/rustdemo/src/util.rs b/tests/fixtures/rustdemo/src/util.rs new file mode 100644 index 0000000..7d099f6 --- /dev/null +++ b/tests/fixtures/rustdemo/src/util.rs @@ -0,0 +1,3 @@ +pub fn helper() -> String { + "ok".to_string() +} diff --git a/tests/fixtures/tsdemo/a.ts b/tests/fixtures/tsdemo/a.ts new file mode 100644 index 0000000..9896a79 --- /dev/null +++ b/tests/fixtures/tsdemo/a.ts @@ -0,0 +1,3 @@ +import { b } from "./b"; + +export const a = 1; diff --git a/tests/fixtures/tsdemo/b.ts b/tests/fixtures/tsdemo/b.ts new file mode 100644 index 0000000..9847a66 --- /dev/null +++ b/tests/fixtures/tsdemo/b.ts @@ -0,0 +1,3 @@ +import { a } from "./a"; + +export const b = a;