Initial commit: rustytokens — native codebase indexer + MCP server

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

src/search/mod.rs

@@ -0,0 +1,130 @@
+//! FTS5-backed search with a simple own ranking (exact > prefix > fulltext).
+
+use std::collections::HashSet;
+
+use anyhow::Result;
+use rusqlite::{Connection, params};
+use serde::Serialize;
+
+#[derive(Debug, Serialize)]
+pub struct SymbolHit {
+    pub id: i64,
+    pub name: String,
+    pub qualified_name: String,
+    pub kind: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub signature: Option<String>,
+    pub file: String,
+    pub line_start: i64,
+    pub line_end: i64,
+    pub match_type: String,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TextHit {
+    pub file: String,
+    pub snippet: String,
+}
+
+const SYMBOL_COLS: &str =
+    "s.id, s.name, s.qualified_name, s.kind, s.signature, f.path, s.line_start, s.line_end";
+
+pub fn search_symbols(conn: &Connection, query: &str) -> Result<Vec<SymbolHit>> {
+    let q = query.trim();
+    let mut seen: HashSet<i64> = HashSet::new();
+    let mut out: Vec<SymbolHit> = Vec::new();
+    if q.is_empty() {
+        return Ok(out);
+    }
+
+    // 1) exact name (case-insensitive)
+    {
+        let mut stmt = conn.prepare(&format!(
+            "SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
+             WHERE s.name = ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
+        ))?;
+        let rows = stmt.query_map(params![q], |r| row_to_hit(r, "exact"))?;
+        push_hits(&mut out, &mut seen, rows)?;
+    }
+
+    // 2) prefix
+    {
+        let pat = format!("{}%", q);
+        let mut stmt = conn.prepare(&format!(
+            "SELECT {SYMBOL_COLS} FROM symbols s JOIN files f ON f.id = s.file_id \
+             WHERE s.name LIKE ?1 COLLATE NOCASE ORDER BY s.id LIMIT 200"
+        ))?;
+        let rows = stmt.query_map(params![pat], |r| row_to_hit(r, "prefix"))?;
+        push_hits(&mut out, &mut seen, rows)?;
+    }
+
+    // 3) fulltext over name/qualified_name/kind/signature
+    {
+        let phrase = fts_phrase(q);
+        let mut stmt = conn.prepare(&format!(
+            "SELECT {SYMBOL_COLS} FROM symbols_fts \
+             JOIN symbols s ON s.id = symbols_fts.rowid \
+             JOIN files f ON f.id = s.file_id \
+             WHERE symbols_fts MATCH ?1 ORDER BY rank LIMIT 200"
+        ))?;
+        let rows = stmt.query_map(params![phrase], |r| row_to_hit(r, "fulltext"))?;
+        push_hits(&mut out, &mut seen, rows)?;
+    }
+
+    Ok(out)
+}
+
+pub fn search_text(conn: &Connection, query: &str) -> Result<Vec<TextHit>> {
+    let q = query.trim();
+    if q.is_empty() {
+        return Ok(Vec::new());
+    }
+    let phrase = fts_phrase(q);
+    let mut stmt = conn.prepare(
+        "SELECT path, snippet(content_fts, 1, '«', '»', '…', 24) \
+         FROM content_fts WHERE content_fts MATCH ?1 ORDER BY rank LIMIT 100",
+    )?;
+    let rows = stmt.query_map(params![phrase], |r| {
+        Ok(TextHit {
+            file: r.get(0)?,
+            snippet: r.get(1)?,
+        })
+    })?;
+    Ok(rows.collect::<std::result::Result<Vec<_>, _>>()?)
+}
+
+fn row_to_hit(
+    r: &rusqlite::Row<'_>,
+    match_type: &str,
+) -> std::result::Result<SymbolHit, rusqlite::Error> {
+    Ok(SymbolHit {
+        id: r.get(0)?,
+        name: r.get(1)?,
+        qualified_name: r.get(2)?,
+        kind: r.get(3)?,
+        signature: r.get(4)?,
+        file: r.get(5)?,
+        line_start: r.get(6)?,
+        line_end: r.get(7)?,
+        match_type: match_type.to_string(),
+    })
+}
+
+fn push_hits(
+    out: &mut Vec<SymbolHit>,
+    seen: &mut HashSet<i64>,
+    rows: impl Iterator<Item = std::result::Result<SymbolHit, rusqlite::Error>>,
+) -> Result<()> {
+    for r in rows {
+        let hit = r?;
+        if seen.insert(hit.id) {
+            out.push(hit);
+        }
+    }
+    Ok(())
+}
+
+/// Wrap a user query as a safe FTS5 phrase.
+fn fts_phrase(q: &str) -> String {
+    format!("\"{}\"", q.replace('"', "\"\""))
+}