diff options
Diffstat (limited to 'src/main.rs')
| -rw-r--r-- | src/main.rs | 588 |
1 files changed, 588 insertions, 0 deletions
diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9824351 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,588 @@ +//! elel: Emacs Lexical Embedding Link. A notes engine. +//! +//! This program only reads the note files. It never writes to them. +//! Everything it stores lives in the index directory and can be rebuilt. + +mod adapter; +mod chunk; +mod graph; +mod note; +mod query; +mod text; +mod vec; + +use anyhow::{Context, Result, bail}; +use clap::{Parser, Subcommand}; +use graph::{FileEntry, Graph, Manifest}; +use note::Note; +use query::Query; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use text::Text; +use vec::{Ollama, Store}; + +#[derive(Parser)] +#[command(version, about = "Search and link notes kept in plain files")] +struct Cli { + /// Directory that holds the note files. + #[arg(long, env = "ELEL_ROOT", global = true)] + root: Option<PathBuf>, + /// Directory for the derived index. + #[arg(long, env = "ELEL_INDEX", global = true)] + index: Option<PathBuf>, + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Scan the note files and update the index. + Index { + /// Re-read every file, even the unchanged ones. + #[arg(long)] + force: bool, + /// Do not compute embeddings (no Ollama needed). + #[arg(long)] + no_embed: bool, + }, + /// Search notes. The query may hold filters such as `tag:X`, + /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README); + /// the other words go to the full text index. An empty query lists + /// every note. + Search { + query: Option<String>, + #[arg(short, default_value_t = 50)] + n: usize, + /// Also rank by meaning: fuse the text search with the embeddings. + #[arg(long)] + hybrid: bool, + }, + /// Print how many notes each query matches, one line per query. + Count { queries: Vec<String> }, + /// Show the neighbourhood of a note. + Hood { + id: String, + #[arg(long, default_value_t = 1)] + hops: usize, + }, + /// Print a note with its body, read fresh from its file. + Get { id: String }, + /// Report notes without IDs, duplicate IDs and dangling links. + Check, + /// List the tags with the number of notes that carry each. + Tags { + /// Rank the tags by how close their notes are to this note. + #[arg(long)] + like: Option<String>, + }, +} + +struct Ctx { + root: PathBuf, + index: PathBuf, +} + +impl Ctx { + fn new(cli: &Cli) -> Result<Ctx> { + let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?); + let root = cli.root.clone().unwrap_or_else(|| home.join("notes")); + let index = cli + .index + .clone() + .unwrap_or_else(|| home.join(".local/share/elel")); + Ok(Ctx { root, index }) + } + + fn manifest_path(&self) -> PathBuf { + self.index.join("graph.json") + } + + fn tantivy_dir(&self) -> PathBuf { + self.index.join("tantivy") + } +} + +/// A note as shown to the front end. +#[derive(Serialize)] +struct View<'a> { + #[serde(flatten)] + note: &'a Note, + #[serde(skip_serializing_if = "Option::is_none")] + score: Option<f32>, + #[serde(skip_serializing_if = "Option::is_none")] + snippet: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + dist: Option<usize>, + #[serde(skip_serializing_if = "Option::is_none")] + rel: Option<&'static str>, + /// For hits by meaning: the line of the body where the best chunk starts. + #[serde(skip_serializing_if = "Option::is_none")] + chunk_line: Option<usize>, + /// Number of notes that link here. + backlinks: usize, +} + +fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> { + View { + note: n, + score: None, + snippet: None, + dist: None, + rel: None, + chunk_line: None, + backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0), + } +} + +fn print<T: Serialize>(v: &T) -> Result<()> { + println!("{}", serde_json::to_string(v)?); + Ok(()) +} + +fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> { + let mut entries: Vec<_> = std::fs::read_dir(dir) + .with_context(|| format!("cannot read {}", dir.display()))? + .filter_map(|e| e.ok()) + .collect(); + entries.sort_by_key(|e| e.file_name()); + for e in entries { + let p = e.path(); + let name = e.file_name().to_string_lossy().to_string(); + if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') { + continue; + } + if p.is_dir() { + collect_files(&p, out)?; + } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) { + out.push(p); + } + } + Ok(()) +} + +fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> { + std::fs::create_dir_all(&ctx.index)?; + let text = Text::open(&ctx.tantivy_dir())?; + let old = if text.fresh || force { + Manifest::default() + } else { + Manifest::load(&ctx.manifest_path()) + }; + let root = ctx + .root + .canonicalize() + .with_context(|| format!("no notes directory at {}", ctx.root.display()))?; + let mut files = Vec::new(); + collect_files(&root, &mut files)?; + + let mut new = Manifest::default(); + let mut changed: Vec<(String, Vec<Note>)> = Vec::new(); + for f in &files { + let key = f.to_string_lossy().to_string(); + let content = match std::fs::read_to_string(f) { + Ok(c) => c, + Err(e) => { + eprintln!("skipping {}: {}", key, e); + continue; + } + }; + let hash = blake3::hash(content.as_bytes()).to_hex().to_string(); + match old.files.get(&key) { + Some(e) if e.hash == hash => { + new.files.insert( + key, + FileEntry { + hash, + notes: e.notes.clone(), + }, + ); + } + _ => { + let notes = adapter::parse_file(f, &content).unwrap_or_default(); + changed.push((key.clone(), notes.clone())); + new.files.insert(key, FileEntry { hash, notes }); + } + } + } + let removed: Vec<String> = old + .files + .keys() + .filter(|k| !new.files.contains_key(*k)) + .cloned() + .collect(); + text.update(&removed, &changed)?; + new.save(&ctx.manifest_path())?; + let g = Graph::build(&new); + let mut report = serde_json::json!({ + "files": new.files.len(), + "changed": changed.len(), + "removed": removed.len(), + "notes": g.notes.len(), + }); + if !no_embed { + let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?; + report["embedded"] = r.embedded.into(); + report["pending"] = r.pending.into(); + if let Some(e) = r.error { + eprintln!("elel: embeddings stopped: {}", e); + report["embed_error"] = e.into(); + } + } + print(&report) +} + +fn load_graph(ctx: &Ctx) -> Result<Graph> { + let p = ctx.manifest_path(); + if !p.exists() { + bail!("no index yet; run `elel index` first"); + } + Ok(Graph::build(&Manifest::load(&p))) +} + +/// One answer of a query: the note, and for text queries the score +/// and the snippet. +struct Hit<'a> { + note: &'a Note, + score: Option<f32>, + snippet: Option<String>, + /// The best chunk, for hits by meaning. + row: Option<vec::Row>, +} + +/// The words of a text query that can be embedded: no `-x`, `OR`, or +/// field prefixes, and no quotes. +fn plain_words(words: &str) -> String { + words + .split_whitespace() + .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND") + .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"')) + .filter(|w| !w.is_empty()) + .collect::<Vec<_>>() + .join(" ") +} + +/// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words). +fn semantic( + ctx: &Ctx, + g: &Graph, + q: &Query, + m: &query::Matcher, + hybrid: bool, +) -> Result<Vec<vec::VecHit>> { + let store = Store::load(&ctx.index); + if store.is_empty() { + bail!("no embeddings yet; start Ollama and run `elel index`"); + } + let ollama = Ollama::from_env(); + if store.model != ollama.model { + bail!( + "the embeddings were made with {}, but the model is {}; run `elel index`", + store.model, + ollama.model + ); + } + let mut by_chunk = false; + let mut exclude: HashSet<String> = HashSet::new(); + let mut resolve = |s: &str| -> Result<Vec<f32>> { + if g.get(s).is_some() { + exclude.insert(s.to_string()); + store + .note_vec(s) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s)) + } else { + by_chunk = true; + ollama.embed_query(s) + } + }; + let mut pos = Vec::new(); + for s in &q.like { + pos.push(resolve(s)?); + } + let mut neg = Vec::new(); + for s in &q.unlike { + neg.push(resolve(s)?); + } + if pos.is_empty() { + if !neg.is_empty() { + bail!("-like: needs a like: to go with"); + } + by_chunk = true; + pos.push(ollama.embed_query(&plain_words(&q.words))?); + } + let qv = vec::rocchio(&pos, &neg).unwrap(); + let _ = hybrid; + let allow = |id: &str| match g.get(id) { + Some(n) => !exclude.contains(id) && m.pass(n), + None => false, + }; + Ok(store.rank(&qv, by_chunk, &allow)) +} + +fn run_query<'a>( + ctx: &Ctx, + g: &'a Graph, + query: &str, + n: usize, + hybrid: bool, +) -> Result<Vec<Hit<'a>>> { + let q = Query::parse(query)?; + let root = ctx + .root + .canonicalize() + .unwrap_or_else(|_| ctx.root.clone()) + .to_string_lossy() + .to_string(); + let m = q.matcher(g, &root); + let has_words = !q.words.trim().is_empty(); + let has_like = !q.like.is_empty() || !q.unlike.is_empty(); + let mut out = Vec::new(); + + // The ranking by meaning, if the query asks for it. + let mut by_meaning: Option<Vec<vec::VecHit>> = None; + if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) { + match semantic(ctx, g, &q, &m, hybrid) { + Ok(v) => by_meaning = Some(v), + Err(e) if has_like => return Err(e), + Err(e) => eprintln!("elel: {}; using the text search only", e), + } + } + + // The ranking by words. + let mut lexical: Vec<(&'a Note, f32, Option<String>)> = Vec::new(); + if has_words { + let text = Text::open(&ctx.tantivy_dir())?; + // Filters apply after the text search, so ask for plenty. + for f in text.search(&q.words, 100_000)? { + let note = match g.get(&f.id) { + Some(note) => note, + None => continue, + }; + if m.pass(note) { + lexical.push((note, f.score, f.snippet)); + } + } + } + + match by_meaning { + None if has_words => { + for (note, score, snippet) in lexical.into_iter().take(n) { + out.push(Hit { note, score: Some(score), snippet, row: None }); + } + } + None => { + let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect(); + // Newest first; notes without a date go last. + all.sort_by(|a, b| b.date.cmp(&a.date)); + for note in all.into_iter().take(n) { + out.push(Hit { note, score: None, snippet: None, row: None }); + } + } + Some(v) if !has_words => { + for h in v.into_iter().take(n) { + if let Some(note) = g.get(&h.id) { + out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row }); + } + } + } + Some(v) => { + let lists = vec![ + lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::<Vec<_>>(), + v.iter().map(|h| h.id.clone()).collect::<Vec<_>>(), + ]; + let mut rows: HashMap<String, vec::Row> = HashMap::new(); + for h in v { + if let Some(r) = h.row { + rows.insert(h.id, r); + } + } + let mut snippets: HashMap<String, Option<String>> = HashMap::new(); + for (note, _, s) in lexical { + if let Some(id) = ¬e.id { + snippets.insert(id.clone(), s); + } + } + for (id, score) in vec::rrf(&lists).into_iter().take(n) { + if let Some(note) = g.get(&id) { + out.push(Hit { + note, + score: Some(score), + snippet: snippets.remove(&id).flatten(), + row: rows.remove(&id), + }); + } + } + } + } + Ok(out) +} + +/// The first text of a chunk, for a hit by meaning. +fn chunk_snippet(h: &Hit, cache: &mut HashMap<String, Vec<Note>>) -> Option<String> { + let row = h.row.as_ref()?; + let path = &h.note.path; + if !cache.contains_key(path) { + let content = std::fs::read_to_string(path).ok()?; + let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default(); + cache.insert(path.clone(), v); + } + let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?; + let cs = vec::note_chunks(fresh); + let text = &cs.get(row.chunk)?.text; + let body = text.strip_prefix(&fresh.title).unwrap_or(text); + let flat = body.split_whitespace().collect::<Vec<_>>().join(" "); + Some(flat.chars().take(200).collect()) +} + +fn cmd_search(ctx: &Ctx, query: Option<String>, n: usize, hybrid: bool) -> Result<()> { + let g = load_graph(ctx)?; + let mut cache = HashMap::new(); + for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? { + let mut v = view(&g, h.note); + v.score = h.score; + v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache)); + v.chunk_line = h.row.as_ref().map(|r| r.line); + print(&v)?; + } + Ok(()) +} + +fn cmd_count(ctx: &Ctx, queries: Vec<String>) -> Result<()> { + let g = load_graph(ctx)?; + for q in queries { + let n = run_query(ctx, &g, &q, usize::MAX, false)?.len(); + print(&serde_json::json!({ "count": n }))?; + } + Ok(()) +} + +fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> { + let g = load_graph(ctx)?; + let center = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + print(&serde_json::json!({ "center": view(&g, center) }))?; + for (n, d, rel) in g.reach(id, hops.max(1)) { + let mut v = view(&g, n); + v.dist = Some(d); + v.rel = Some(rel.name()); + print(&v)?; + } + Ok(()) +} + +fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> { + let g = load_graph(ctx)?; + let n = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + // Read the file again, so the body is never stale. + let path = PathBuf::from(&n.path); + let content = std::fs::read_to_string(&path) + .with_context(|| format!("index out of date: cannot read {}", path.display()))?; + let fresh = adapter::parse_file(&path, &content) + .unwrap_or_default() + .into_iter() + .find(|x| x.id.as_deref() == Some(id)); + let note = fresh.as_ref().unwrap_or(n); + print(&serde_json::json!({ + "note": view(&g, note), + "body": note.body, + // The exact text of the note in its file, for display. + "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()), + "stale": fresh.is_none(), + "parent": g.parent_of(id).map(|x| view(&g, x)), + "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + })) +} + +fn cmd_check(ctx: &Ctx) -> Result<()> { + let g = load_graph(ctx)?; + let mut issues: Vec<serde_json::Value> = Vec::new(); + for n in &g.notes { + match &n.id { + None => issues.push(serde_json::json!({ + "issue": "missing-id", "path": n.path, "line": n.line, + "start": n.start, "title": n.title, + })), + Some(id) => { + if let Some(p) = &n.parent { + if g.get(p).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-parent", "id": id, "target": p, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for l in &n.links { + if g.get(&l.dst).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-link", "id": id, "target": l.dst, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + } + } + } + let mut by_path: BTreeMap<&str, usize> = BTreeMap::new(); + for (id, idxs) in &g.duplicates { + for &i in idxs { + let n = &g.notes[i]; + *by_path.entry(&n.path).or_default() += 1; + issues.push(serde_json::json!({ + "issue": "duplicate-id", "id": id, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for i in issues { + print(&i)?; + } + Ok(()) +} + +fn cmd_tags(ctx: &Ctx, like: Option<String>) -> Result<()> { + let g = load_graph(ctx)?; + if let Some(id) = like { + let store = Store::load(&ctx.index); + let v = store + .note_vec(&id) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?; + for (tag, count, score, has) in store.tag_scores(&g, &id, &v) { + print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?; + } + return Ok(()); + } + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for n in g.notes.iter().filter(|n| n.id.is_some()) { + for t in &n.tags { + *counts.entry(t).or_default() += 1; + } + } + let mut v: Vec<(&str, usize)> = counts.into_iter().collect(); + // Most used first, then alphabetical. + v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + for (tag, count) in v { + print(&serde_json::json!({ "tag": tag, "count": count }))?; + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let ctx = Ctx::new(&cli)?; + match cli.cmd { + Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed), + Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid), + Cmd::Count { queries } => cmd_count(&ctx, queries), + Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops), + Cmd::Get { id } => cmd_get(&ctx, &id), + Cmd::Check => cmd_check(&ctx), + Cmd::Tags { like } => cmd_tags(&ctx, like), + } +} |
