Initial commit: rustytokens — native codebase indexer + MCP server

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

src/discovery/mod.rs

@@ -0,0 +1,59 @@
+//! Filesystem walk with smart filtering, producing the list of source files to index.
+
+use std::path::{Path, PathBuf};
+
+use anyhow::Result;
+
+use crate::config::{SKIP_DIRS, language_for};
+
+#[derive(Debug, Clone)]
+pub struct SourceFile {
+    /// Absolute path on disk.
+    pub path: PathBuf,
+    /// Path relative to the project root (forward slashes).
+    pub rel: String,
+    pub language: &'static str,
+}
+
+pub fn discover(project: &Path) -> Result<Vec<SourceFile>> {
+    let project_abs = project
+        .canonicalize()
+        .unwrap_or_else(|_| project.to_path_buf());
+    let mut builder = ignore::WalkBuilder::new(&project_abs);
+    builder.standard_filters(true);
+    builder.require_git(false);
+    builder.filter_entry(|entry| {
+        if entry.file_type().is_some_and(|t| t.is_dir()) {
+            let name = entry.file_name().to_string_lossy();
+            !SKIP_DIRS.iter().any(|d| *d == name)
+        } else {
+            true
+        }
+    });
+
+    let mut out = Vec::new();
+    for entry in builder.build() {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        let path = entry.path();
+        if entry.file_type().is_some_and(|t| t.is_dir()) {
+            continue;
+        }
+        let Some(language) = language_for(path) else {
+            continue;
+        };
+        let rel = path
+            .strip_prefix(&project_abs)
+            .map(|p| p.to_string_lossy().replace('\\', "/"))
+            .unwrap_or_else(|_| path.to_string_lossy().replace('\\', "/"));
+        out.push(SourceFile {
+            path: path.to_path_buf(),
+            rel,
+            language,
+        });
+    }
+    out.sort_by(|a, b| a.rel.cmp(&b.rel));
+    Ok(out)
+}