//! elel: Emacs Lexical Embedding Link. A notes engine. //! //! This program only reads the note files. It never writes to them. //! Everything it stores lives in the index directory and can be rebuilt. mod adapter; mod chunk; mod graph; mod note; mod query; mod text; mod vec; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use graph::{FileEntry, Graph, Manifest}; use note::Note; use query::Query; use serde::Serialize; use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Path, PathBuf}; use text::Text; use vec::{Ollama, Store}; #[derive(Parser)] #[command(version, about = "Search and link notes kept in plain files")] struct Cli { /// Directory that holds the note files. #[arg(long, env = "ELEL_ROOT", global = true)] root: Option, /// Directory for the derived index. #[arg(long, env = "ELEL_INDEX", global = true)] index: Option, #[command(subcommand)] cmd: Cmd, } #[derive(Subcommand)] enum Cmd { /// Scan the note files and update the index. Index { /// Re-read every file, even the unchanged ones. #[arg(long)] force: bool, /// Do not compute embeddings (no Ollama needed). #[arg(long)] no_embed: bool, }, /// Search notes. The query may hold filters such as `tag:X`, /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README); /// the other words go to the full text index. An empty query lists /// every note. Search { query: Option, #[arg(short, default_value_t = 50)] n: usize, /// Also rank by meaning: fuse the text search with the embeddings. #[arg(long)] hybrid: bool, }, /// Print how many notes each query matches, one line per query. Count { queries: Vec }, /// Show the neighbourhood of a note. Hood { id: String, #[arg(long, default_value_t = 1)] hops: usize, }, /// Print a note with its body, read fresh from its file. Get { id: String }, /// Report notes without IDs, duplicate IDs and dangling links. Check, /// List the tags with the number of notes that carry each. Tags { /// Rank the tags by how close their notes are to this note. #[arg(long)] like: Option, }, } struct Ctx { root: PathBuf, index: PathBuf, } impl Ctx { fn new(cli: &Cli) -> Result { let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?); let root = cli.root.clone().unwrap_or_else(|| home.join("notes")); let index = cli .index .clone() .unwrap_or_else(|| home.join(".local/share/elel")); Ok(Ctx { root, index }) } fn manifest_path(&self) -> PathBuf { self.index.join("graph.json") } fn tantivy_dir(&self) -> PathBuf { self.index.join("tantivy") } } /// A note as shown to the front end. #[derive(Serialize)] struct View<'a> { #[serde(flatten)] note: &'a Note, #[serde(skip_serializing_if = "Option::is_none")] score: Option, #[serde(skip_serializing_if = "Option::is_none")] snippet: Option, #[serde(skip_serializing_if = "Option::is_none")] dist: Option, #[serde(skip_serializing_if = "Option::is_none")] rel: Option<&'static str>, /// For hits by meaning: the line of the body where the best chunk starts. #[serde(skip_serializing_if = "Option::is_none")] chunk_line: Option, /// Number of notes that link here. backlinks: usize, } fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> { View { note: n, score: None, snippet: None, dist: None, rel: None, chunk_line: None, backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0), } } fn print(v: &T) -> Result<()> { println!("{}", serde_json::to_string(v)?); Ok(()) } fn collect_files(dir: &Path, out: &mut Vec) -> Result<()> { let mut entries: Vec<_> = std::fs::read_dir(dir) .with_context(|| format!("cannot read {}", dir.display()))? .filter_map(|e| e.ok()) .collect(); entries.sort_by_key(|e| e.file_name()); for e in entries { let p = e.path(); let name = e.file_name().to_string_lossy().to_string(); if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') { continue; } if p.is_dir() { collect_files(&p, out)?; } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) { out.push(p); } } Ok(()) } fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> { std::fs::create_dir_all(&ctx.index)?; let text = Text::open(&ctx.tantivy_dir())?; let old = if text.fresh || force { Manifest::default() } else { Manifest::load(&ctx.manifest_path()) }; let root = ctx .root .canonicalize() .with_context(|| format!("no notes directory at {}", ctx.root.display()))?; let mut files = Vec::new(); collect_files(&root, &mut files)?; let mut new = Manifest::default(); let mut changed: Vec<(String, Vec)> = Vec::new(); for f in &files { let key = f.to_string_lossy().to_string(); let content = match std::fs::read_to_string(f) { Ok(c) => c, Err(e) => { eprintln!("skipping {}: {}", key, e); continue; } }; let hash = blake3::hash(content.as_bytes()).to_hex().to_string(); match old.files.get(&key) { Some(e) if e.hash == hash => { new.files.insert( key, FileEntry { hash, notes: e.notes.clone(), }, ); } _ => { let notes = adapter::parse_file(f, &content).unwrap_or_default(); changed.push((key.clone(), notes.clone())); new.files.insert(key, FileEntry { hash, notes }); } } } let removed: Vec = old .files .keys() .filter(|k| !new.files.contains_key(*k)) .cloned() .collect(); text.update(&removed, &changed)?; new.save(&ctx.manifest_path())?; let g = Graph::build(&new); let mut report = serde_json::json!({ "files": new.files.len(), "changed": changed.len(), "removed": removed.len(), "notes": g.notes.len(), }); if !no_embed { let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?; report["embedded"] = r.embedded.into(); report["pending"] = r.pending.into(); if let Some(e) = r.error { eprintln!("elel: embeddings stopped: {}", e); report["embed_error"] = e.into(); } } print(&report) } fn load_graph(ctx: &Ctx) -> Result { let p = ctx.manifest_path(); if !p.exists() { bail!("no index yet; run `elel index` first"); } Ok(Graph::build(&Manifest::load(&p))) } /// One answer of a query: the note, and for text queries the score /// and the snippet. struct Hit<'a> { note: &'a Note, score: Option, snippet: Option, /// The best chunk, for hits by meaning. row: Option, } /// The words of a text query that can be embedded: no `-x`, `OR`, or /// field prefixes, and no quotes. fn plain_words(words: &str) -> String { words .split_whitespace() .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND") .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"')) .filter(|w| !w.is_empty()) .collect::>() .join(" ") } /// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words). fn semantic( ctx: &Ctx, g: &Graph, q: &Query, m: &query::Matcher, hybrid: bool, ) -> Result> { let store = Store::load(&ctx.index); if store.is_empty() { bail!("no embeddings yet; start Ollama and run `elel index`"); } let ollama = Ollama::from_env(); if store.model != ollama.model { bail!( "the embeddings were made with {}, but the model is {}; run `elel index`", store.model, ollama.model ); } let mut by_chunk = false; let mut exclude: HashSet = HashSet::new(); let mut resolve = |s: &str| -> Result> { if g.get(s).is_some() { exclude.insert(s.to_string()); store .note_vec(s) .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s)) } else { by_chunk = true; ollama.embed_query(s) } }; let mut pos = Vec::new(); for s in &q.like { pos.push(resolve(s)?); } let mut neg = Vec::new(); for s in &q.unlike { neg.push(resolve(s)?); } if pos.is_empty() { if !neg.is_empty() { bail!("-like: needs a like: to go with"); } by_chunk = true; pos.push(ollama.embed_query(&plain_words(&q.words))?); } let qv = vec::rocchio(&pos, &neg).unwrap(); let _ = hybrid; let allow = |id: &str| match g.get(id) { Some(n) => !exclude.contains(id) && m.pass(n), None => false, }; Ok(store.rank(&qv, by_chunk, &allow)) } fn run_query<'a>( ctx: &Ctx, g: &'a Graph, query: &str, n: usize, hybrid: bool, ) -> Result>> { let q = Query::parse(query)?; let root = ctx .root .canonicalize() .unwrap_or_else(|_| ctx.root.clone()) .to_string_lossy() .to_string(); let m = q.matcher(g, &root); let has_words = !q.words.trim().is_empty(); let has_like = !q.like.is_empty() || !q.unlike.is_empty(); let mut out = Vec::new(); // The ranking by meaning, if the query asks for it. let mut by_meaning: Option> = None; if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) { match semantic(ctx, g, &q, &m, hybrid) { Ok(v) => by_meaning = Some(v), Err(e) if has_like => return Err(e), Err(e) => eprintln!("elel: {}; using the text search only", e), } } // The ranking by words. let mut lexical: Vec<(&'a Note, f32, Option)> = Vec::new(); if has_words { let text = Text::open(&ctx.tantivy_dir())?; // Filters apply after the text search, so ask for plenty. for f in text.search(&q.words, 100_000)? { let note = match g.get(&f.id) { Some(note) => note, None => continue, }; if m.pass(note) { lexical.push((note, f.score, f.snippet)); } } } match by_meaning { None if has_words => { for (note, score, snippet) in lexical.into_iter().take(n) { out.push(Hit { note, score: Some(score), snippet, row: None }); } } None => { let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect(); // Newest first; notes without a date go last. all.sort_by(|a, b| b.date.cmp(&a.date)); for note in all.into_iter().take(n) { out.push(Hit { note, score: None, snippet: None, row: None }); } } Some(v) if !has_words => { for h in v.into_iter().take(n) { if let Some(note) = g.get(&h.id) { out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row }); } } } Some(v) => { let lists = vec![ lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::>(), v.iter().map(|h| h.id.clone()).collect::>(), ]; let mut rows: HashMap = HashMap::new(); for h in v { if let Some(r) = h.row { rows.insert(h.id, r); } } let mut snippets: HashMap> = HashMap::new(); for (note, _, s) in lexical { if let Some(id) = ¬e.id { snippets.insert(id.clone(), s); } } for (id, score) in vec::rrf(&lists).into_iter().take(n) { if let Some(note) = g.get(&id) { out.push(Hit { note, score: Some(score), snippet: snippets.remove(&id).flatten(), row: rows.remove(&id), }); } } } } Ok(out) } /// The first text of a chunk, for a hit by meaning. fn chunk_snippet(h: &Hit, cache: &mut HashMap>) -> Option { let row = h.row.as_ref()?; let path = &h.note.path; if !cache.contains_key(path) { let content = std::fs::read_to_string(path).ok()?; let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default(); cache.insert(path.clone(), v); } let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?; let cs = vec::note_chunks(fresh); let text = &cs.get(row.chunk)?.text; let body = text.strip_prefix(&fresh.title).unwrap_or(text); let flat = body.split_whitespace().collect::>().join(" "); Some(flat.chars().take(200).collect()) } fn cmd_search(ctx: &Ctx, query: Option, n: usize, hybrid: bool) -> Result<()> { let g = load_graph(ctx)?; let mut cache = HashMap::new(); for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? { let mut v = view(&g, h.note); v.score = h.score; v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache)); v.chunk_line = h.row.as_ref().map(|r| r.line); print(&v)?; } Ok(()) } fn cmd_count(ctx: &Ctx, queries: Vec) -> Result<()> { let g = load_graph(ctx)?; for q in queries { let n = run_query(ctx, &g, &q, usize::MAX, false)?.len(); print(&serde_json::json!({ "count": n }))?; } Ok(()) } fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> { let g = load_graph(ctx)?; let center = match g.get(id) { Some(n) => n, None => bail!("no note with id {}", id), }; print(&serde_json::json!({ "center": view(&g, center) }))?; for (n, d, rel) in g.reach(id, hops.max(1)) { let mut v = view(&g, n); v.dist = Some(d); v.rel = Some(rel.name()); print(&v)?; } Ok(()) } fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> { let g = load_graph(ctx)?; let n = match g.get(id) { Some(n) => n, None => bail!("no note with id {}", id), }; // Read the file again, so the body is never stale. let path = PathBuf::from(&n.path); let content = std::fs::read_to_string(&path) .with_context(|| format!("index out of date: cannot read {}", path.display()))?; let fresh = adapter::parse_file(&path, &content) .unwrap_or_default() .into_iter() .find(|x| x.id.as_deref() == Some(id)); let note = fresh.as_ref().unwrap_or(n); print(&serde_json::json!({ "note": view(&g, note), "body": note.body, // The exact text of the note in its file, for display. "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()), "stale": fresh.is_none(), "parent": g.parent_of(id).map(|x| view(&g, x)), "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::>(), "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::>(), "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::>(), })) } fn cmd_check(ctx: &Ctx) -> Result<()> { let g = load_graph(ctx)?; let mut issues: Vec = Vec::new(); for n in &g.notes { match &n.id { None => issues.push(serde_json::json!({ "issue": "missing-id", "path": n.path, "line": n.line, "start": n.start, "title": n.title, })), Some(id) => { if let Some(p) = &n.parent { if g.get(p).is_none() { issues.push(serde_json::json!({ "issue": "dangling-parent", "id": id, "target": p, "path": n.path, "line": n.line, "title": n.title, })); } } for l in &n.links { if g.get(&l.dst).is_none() { issues.push(serde_json::json!({ "issue": "dangling-link", "id": id, "target": l.dst, "path": n.path, "line": n.line, "title": n.title, })); } } } } } let mut by_path: BTreeMap<&str, usize> = BTreeMap::new(); for (id, idxs) in &g.duplicates { for &i in idxs { let n = &g.notes[i]; *by_path.entry(&n.path).or_default() += 1; issues.push(serde_json::json!({ "issue": "duplicate-id", "id": id, "path": n.path, "line": n.line, "title": n.title, })); } } for i in issues { print(&i)?; } Ok(()) } fn cmd_tags(ctx: &Ctx, like: Option) -> Result<()> { let g = load_graph(ctx)?; if let Some(id) = like { let store = Store::load(&ctx.index); let v = store .note_vec(&id) .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?; for (tag, count, score, has) in store.tag_scores(&g, &id, &v) { print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?; } return Ok(()); } let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); for n in g.notes.iter().filter(|n| n.id.is_some()) { for t in &n.tags { *counts.entry(t).or_default() += 1; } } let mut v: Vec<(&str, usize)> = counts.into_iter().collect(); // Most used first, then alphabetical. v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); for (tag, count) in v { print(&serde_json::json!({ "tag": tag, "count": count }))?; } Ok(()) } fn main() -> Result<()> { let cli = Cli::parse(); let ctx = Ctx::new(&cli)?; match cli.cmd { Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed), Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid), Cmd::Count { queries } => cmd_count(&ctx, queries), Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops), Cmd::Get { id } => cmd_get(&ctx, &id), Cmd::Check => cmd_check(&ctx), Cmd::Tags { like } => cmd_tags(&ctx, like), } }