aboutsummaryrefslogtreecommitdiff
path: root/src/main.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/main.rs')
-rw-r--r--src/main.rs588
1 files changed, 588 insertions, 0 deletions
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 0000000..9824351
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,588 @@
+//! elel: Emacs Lexical Embedding Link. A notes engine.
+//!
+//! This program only reads the note files. It never writes to them.
+//! Everything it stores lives in the index directory and can be rebuilt.
+
+mod adapter;
+mod chunk;
+mod graph;
+mod note;
+mod query;
+mod text;
+mod vec;
+
+use anyhow::{Context, Result, bail};
+use clap::{Parser, Subcommand};
+use graph::{FileEntry, Graph, Manifest};
+use note::Note;
+use query::Query;
+use serde::Serialize;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::path::{Path, PathBuf};
+use text::Text;
+use vec::{Ollama, Store};
+
+#[derive(Parser)]
+#[command(version, about = "Search and link notes kept in plain files")]
+struct Cli {
+ /// Directory that holds the note files.
+ #[arg(long, env = "ELEL_ROOT", global = true)]
+ root: Option<PathBuf>,
+ /// Directory for the derived index.
+ #[arg(long, env = "ELEL_INDEX", global = true)]
+ index: Option<PathBuf>,
+ #[command(subcommand)]
+ cmd: Cmd,
+}
+
+#[derive(Subcommand)]
+enum Cmd {
+ /// Scan the note files and update the index.
+ Index {
+ /// Re-read every file, even the unchanged ones.
+ #[arg(long)]
+ force: bool,
+ /// Do not compute embeddings (no Ollama needed).
+ #[arg(long)]
+ no_embed: bool,
+ },
+ /// Search notes. The query may hold filters such as `tag:X`,
+ /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README);
+ /// the other words go to the full text index. An empty query lists
+ /// every note.
+ Search {
+ query: Option<String>,
+ #[arg(short, default_value_t = 50)]
+ n: usize,
+ /// Also rank by meaning: fuse the text search with the embeddings.
+ #[arg(long)]
+ hybrid: bool,
+ },
+ /// Print how many notes each query matches, one line per query.
+ Count { queries: Vec<String> },
+ /// Show the neighbourhood of a note.
+ Hood {
+ id: String,
+ #[arg(long, default_value_t = 1)]
+ hops: usize,
+ },
+ /// Print a note with its body, read fresh from its file.
+ Get { id: String },
+ /// Report notes without IDs, duplicate IDs and dangling links.
+ Check,
+ /// List the tags with the number of notes that carry each.
+ Tags {
+ /// Rank the tags by how close their notes are to this note.
+ #[arg(long)]
+ like: Option<String>,
+ },
+}
+
+struct Ctx {
+ root: PathBuf,
+ index: PathBuf,
+}
+
+impl Ctx {
+ fn new(cli: &Cli) -> Result<Ctx> {
+ let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?);
+ let root = cli.root.clone().unwrap_or_else(|| home.join("notes"));
+ let index = cli
+ .index
+ .clone()
+ .unwrap_or_else(|| home.join(".local/share/elel"));
+ Ok(Ctx { root, index })
+ }
+
+ fn manifest_path(&self) -> PathBuf {
+ self.index.join("graph.json")
+ }
+
+ fn tantivy_dir(&self) -> PathBuf {
+ self.index.join("tantivy")
+ }
+}
+
+/// A note as shown to the front end.
+#[derive(Serialize)]
+struct View<'a> {
+ #[serde(flatten)]
+ note: &'a Note,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ score: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ snippet: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ dist: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ rel: Option<&'static str>,
+ /// For hits by meaning: the line of the body where the best chunk starts.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ chunk_line: Option<usize>,
+ /// Number of notes that link here.
+ backlinks: usize,
+}
+
+fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> {
+ View {
+ note: n,
+ score: None,
+ snippet: None,
+ dist: None,
+ rel: None,
+ chunk_line: None,
+ backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0),
+ }
+}
+
+fn print<T: Serialize>(v: &T) -> Result<()> {
+ println!("{}", serde_json::to_string(v)?);
+ Ok(())
+}
+
+fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
+ let mut entries: Vec<_> = std::fs::read_dir(dir)
+ .with_context(|| format!("cannot read {}", dir.display()))?
+ .filter_map(|e| e.ok())
+ .collect();
+ entries.sort_by_key(|e| e.file_name());
+ for e in entries {
+ let p = e.path();
+ let name = e.file_name().to_string_lossy().to_string();
+ if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') {
+ continue;
+ }
+ if p.is_dir() {
+ collect_files(&p, out)?;
+ } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) {
+ out.push(p);
+ }
+ }
+ Ok(())
+}
+
+fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> {
+ std::fs::create_dir_all(&ctx.index)?;
+ let text = Text::open(&ctx.tantivy_dir())?;
+ let old = if text.fresh || force {
+ Manifest::default()
+ } else {
+ Manifest::load(&ctx.manifest_path())
+ };
+ let root = ctx
+ .root
+ .canonicalize()
+ .with_context(|| format!("no notes directory at {}", ctx.root.display()))?;
+ let mut files = Vec::new();
+ collect_files(&root, &mut files)?;
+
+ let mut new = Manifest::default();
+ let mut changed: Vec<(String, Vec<Note>)> = Vec::new();
+ for f in &files {
+ let key = f.to_string_lossy().to_string();
+ let content = match std::fs::read_to_string(f) {
+ Ok(c) => c,
+ Err(e) => {
+ eprintln!("skipping {}: {}", key, e);
+ continue;
+ }
+ };
+ let hash = blake3::hash(content.as_bytes()).to_hex().to_string();
+ match old.files.get(&key) {
+ Some(e) if e.hash == hash => {
+ new.files.insert(
+ key,
+ FileEntry {
+ hash,
+ notes: e.notes.clone(),
+ },
+ );
+ }
+ _ => {
+ let notes = adapter::parse_file(f, &content).unwrap_or_default();
+ changed.push((key.clone(), notes.clone()));
+ new.files.insert(key, FileEntry { hash, notes });
+ }
+ }
+ }
+ let removed: Vec<String> = old
+ .files
+ .keys()
+ .filter(|k| !new.files.contains_key(*k))
+ .cloned()
+ .collect();
+ text.update(&removed, &changed)?;
+ new.save(&ctx.manifest_path())?;
+ let g = Graph::build(&new);
+ let mut report = serde_json::json!({
+ "files": new.files.len(),
+ "changed": changed.len(),
+ "removed": removed.len(),
+ "notes": g.notes.len(),
+ });
+ if !no_embed {
+ let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?;
+ report["embedded"] = r.embedded.into();
+ report["pending"] = r.pending.into();
+ if let Some(e) = r.error {
+ eprintln!("elel: embeddings stopped: {}", e);
+ report["embed_error"] = e.into();
+ }
+ }
+ print(&report)
+}
+
+fn load_graph(ctx: &Ctx) -> Result<Graph> {
+ let p = ctx.manifest_path();
+ if !p.exists() {
+ bail!("no index yet; run `elel index` first");
+ }
+ Ok(Graph::build(&Manifest::load(&p)))
+}
+
+/// One answer of a query: the note, and for text queries the score
+/// and the snippet.
+struct Hit<'a> {
+ note: &'a Note,
+ score: Option<f32>,
+ snippet: Option<String>,
+ /// The best chunk, for hits by meaning.
+ row: Option<vec::Row>,
+}
+
+/// The words of a text query that can be embedded: no `-x`, `OR`, or
+/// field prefixes, and no quotes.
+fn plain_words(words: &str) -> String {
+ words
+ .split_whitespace()
+ .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND")
+ .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"'))
+ .filter(|w| !w.is_empty())
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+/// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words).
+fn semantic(
+ ctx: &Ctx,
+ g: &Graph,
+ q: &Query,
+ m: &query::Matcher,
+ hybrid: bool,
+) -> Result<Vec<vec::VecHit>> {
+ let store = Store::load(&ctx.index);
+ if store.is_empty() {
+ bail!("no embeddings yet; start Ollama and run `elel index`");
+ }
+ let ollama = Ollama::from_env();
+ if store.model != ollama.model {
+ bail!(
+ "the embeddings were made with {}, but the model is {}; run `elel index`",
+ store.model,
+ ollama.model
+ );
+ }
+ let mut by_chunk = false;
+ let mut exclude: HashSet<String> = HashSet::new();
+ let mut resolve = |s: &str| -> Result<Vec<f32>> {
+ if g.get(s).is_some() {
+ exclude.insert(s.to_string());
+ store
+ .note_vec(s)
+ .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s))
+ } else {
+ by_chunk = true;
+ ollama.embed_query(s)
+ }
+ };
+ let mut pos = Vec::new();
+ for s in &q.like {
+ pos.push(resolve(s)?);
+ }
+ let mut neg = Vec::new();
+ for s in &q.unlike {
+ neg.push(resolve(s)?);
+ }
+ if pos.is_empty() {
+ if !neg.is_empty() {
+ bail!("-like: needs a like: to go with");
+ }
+ by_chunk = true;
+ pos.push(ollama.embed_query(&plain_words(&q.words))?);
+ }
+ let qv = vec::rocchio(&pos, &neg).unwrap();
+ let _ = hybrid;
+ let allow = |id: &str| match g.get(id) {
+ Some(n) => !exclude.contains(id) && m.pass(n),
+ None => false,
+ };
+ Ok(store.rank(&qv, by_chunk, &allow))
+}
+
+fn run_query<'a>(
+ ctx: &Ctx,
+ g: &'a Graph,
+ query: &str,
+ n: usize,
+ hybrid: bool,
+) -> Result<Vec<Hit<'a>>> {
+ let q = Query::parse(query)?;
+ let root = ctx
+ .root
+ .canonicalize()
+ .unwrap_or_else(|_| ctx.root.clone())
+ .to_string_lossy()
+ .to_string();
+ let m = q.matcher(g, &root);
+ let has_words = !q.words.trim().is_empty();
+ let has_like = !q.like.is_empty() || !q.unlike.is_empty();
+ let mut out = Vec::new();
+
+ // The ranking by meaning, if the query asks for it.
+ let mut by_meaning: Option<Vec<vec::VecHit>> = None;
+ if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) {
+ match semantic(ctx, g, &q, &m, hybrid) {
+ Ok(v) => by_meaning = Some(v),
+ Err(e) if has_like => return Err(e),
+ Err(e) => eprintln!("elel: {}; using the text search only", e),
+ }
+ }
+
+ // The ranking by words.
+ let mut lexical: Vec<(&'a Note, f32, Option<String>)> = Vec::new();
+ if has_words {
+ let text = Text::open(&ctx.tantivy_dir())?;
+ // Filters apply after the text search, so ask for plenty.
+ for f in text.search(&q.words, 100_000)? {
+ let note = match g.get(&f.id) {
+ Some(note) => note,
+ None => continue,
+ };
+ if m.pass(note) {
+ lexical.push((note, f.score, f.snippet));
+ }
+ }
+ }
+
+ match by_meaning {
+ None if has_words => {
+ for (note, score, snippet) in lexical.into_iter().take(n) {
+ out.push(Hit { note, score: Some(score), snippet, row: None });
+ }
+ }
+ None => {
+ let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect();
+ // Newest first; notes without a date go last.
+ all.sort_by(|a, b| b.date.cmp(&a.date));
+ for note in all.into_iter().take(n) {
+ out.push(Hit { note, score: None, snippet: None, row: None });
+ }
+ }
+ Some(v) if !has_words => {
+ for h in v.into_iter().take(n) {
+ if let Some(note) = g.get(&h.id) {
+ out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row });
+ }
+ }
+ }
+ Some(v) => {
+ let lists = vec![
+ lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::<Vec<_>>(),
+ v.iter().map(|h| h.id.clone()).collect::<Vec<_>>(),
+ ];
+ let mut rows: HashMap<String, vec::Row> = HashMap::new();
+ for h in v {
+ if let Some(r) = h.row {
+ rows.insert(h.id, r);
+ }
+ }
+ let mut snippets: HashMap<String, Option<String>> = HashMap::new();
+ for (note, _, s) in lexical {
+ if let Some(id) = &note.id {
+ snippets.insert(id.clone(), s);
+ }
+ }
+ for (id, score) in vec::rrf(&lists).into_iter().take(n) {
+ if let Some(note) = g.get(&id) {
+ out.push(Hit {
+ note,
+ score: Some(score),
+ snippet: snippets.remove(&id).flatten(),
+ row: rows.remove(&id),
+ });
+ }
+ }
+ }
+ }
+ Ok(out)
+}
+
+/// The first text of a chunk, for a hit by meaning.
+fn chunk_snippet(h: &Hit, cache: &mut HashMap<String, Vec<Note>>) -> Option<String> {
+ let row = h.row.as_ref()?;
+ let path = &h.note.path;
+ if !cache.contains_key(path) {
+ let content = std::fs::read_to_string(path).ok()?;
+ let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default();
+ cache.insert(path.clone(), v);
+ }
+ let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?;
+ let cs = vec::note_chunks(fresh);
+ let text = &cs.get(row.chunk)?.text;
+ let body = text.strip_prefix(&fresh.title).unwrap_or(text);
+ let flat = body.split_whitespace().collect::<Vec<_>>().join(" ");
+ Some(flat.chars().take(200).collect())
+}
+
+fn cmd_search(ctx: &Ctx, query: Option<String>, n: usize, hybrid: bool) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let mut cache = HashMap::new();
+ for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? {
+ let mut v = view(&g, h.note);
+ v.score = h.score;
+ v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache));
+ v.chunk_line = h.row.as_ref().map(|r| r.line);
+ print(&v)?;
+ }
+ Ok(())
+}
+
+fn cmd_count(ctx: &Ctx, queries: Vec<String>) -> Result<()> {
+ let g = load_graph(ctx)?;
+ for q in queries {
+ let n = run_query(ctx, &g, &q, usize::MAX, false)?.len();
+ print(&serde_json::json!({ "count": n }))?;
+ }
+ Ok(())
+}
+
+fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let center = match g.get(id) {
+ Some(n) => n,
+ None => bail!("no note with id {}", id),
+ };
+ print(&serde_json::json!({ "center": view(&g, center) }))?;
+ for (n, d, rel) in g.reach(id, hops.max(1)) {
+ let mut v = view(&g, n);
+ v.dist = Some(d);
+ v.rel = Some(rel.name());
+ print(&v)?;
+ }
+ Ok(())
+}
+
+fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let n = match g.get(id) {
+ Some(n) => n,
+ None => bail!("no note with id {}", id),
+ };
+ // Read the file again, so the body is never stale.
+ let path = PathBuf::from(&n.path);
+ let content = std::fs::read_to_string(&path)
+ .with_context(|| format!("index out of date: cannot read {}", path.display()))?;
+ let fresh = adapter::parse_file(&path, &content)
+ .unwrap_or_default()
+ .into_iter()
+ .find(|x| x.id.as_deref() == Some(id));
+ let note = fresh.as_ref().unwrap_or(n);
+ print(&serde_json::json!({
+ "note": view(&g, note),
+ "body": note.body,
+ // The exact text of the note in its file, for display.
+ "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()),
+ "stale": fresh.is_none(),
+ "parent": g.parent_of(id).map(|x| view(&g, x)),
+ "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ }))
+}
+
+fn cmd_check(ctx: &Ctx) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let mut issues: Vec<serde_json::Value> = Vec::new();
+ for n in &g.notes {
+ match &n.id {
+ None => issues.push(serde_json::json!({
+ "issue": "missing-id", "path": n.path, "line": n.line,
+ "start": n.start, "title": n.title,
+ })),
+ Some(id) => {
+ if let Some(p) = &n.parent {
+ if g.get(p).is_none() {
+ issues.push(serde_json::json!({
+ "issue": "dangling-parent", "id": id, "target": p,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ for l in &n.links {
+ if g.get(&l.dst).is_none() {
+ issues.push(serde_json::json!({
+ "issue": "dangling-link", "id": id, "target": l.dst,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ }
+ }
+ }
+ let mut by_path: BTreeMap<&str, usize> = BTreeMap::new();
+ for (id, idxs) in &g.duplicates {
+ for &i in idxs {
+ let n = &g.notes[i];
+ *by_path.entry(&n.path).or_default() += 1;
+ issues.push(serde_json::json!({
+ "issue": "duplicate-id", "id": id,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ for i in issues {
+ print(&i)?;
+ }
+ Ok(())
+}
+
+fn cmd_tags(ctx: &Ctx, like: Option<String>) -> Result<()> {
+ let g = load_graph(ctx)?;
+ if let Some(id) = like {
+ let store = Store::load(&ctx.index);
+ let v = store
+ .note_vec(&id)
+ .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?;
+ for (tag, count, score, has) in store.tag_scores(&g, &id, &v) {
+ print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?;
+ }
+ return Ok(());
+ }
+ let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
+ for n in g.notes.iter().filter(|n| n.id.is_some()) {
+ for t in &n.tags {
+ *counts.entry(t).or_default() += 1;
+ }
+ }
+ let mut v: Vec<(&str, usize)> = counts.into_iter().collect();
+ // Most used first, then alphabetical.
+ v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
+ for (tag, count) in v {
+ print(&serde_json::json!({ "tag": tag, "count": count }))?;
+ }
+ Ok(())
+}
+
+fn main() -> Result<()> {
+ let cli = Cli::parse();
+ let ctx = Ctx::new(&cli)?;
+ match cli.cmd {
+ Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed),
+ Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid),
+ Cmd::Count { queries } => cmd_count(&ctx, queries),
+ Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops),
+ Cmd::Get { id } => cmd_get(&ctx, &id),
+ Cmd::Check => cmd_check(&ctx),
+ Cmd::Tags { like } => cmd_tags(&ctx, like),
+ }
+}