diff options
| author | JSDurand <mmemmew@gmail.com> | 2026-11-19 09:29:47 +0800 |
|---|---|---|
| committer | JSDurand <mmemmew@gmail.com> | 2026-11-19 09:29:47 +0800 |
| commit | 5b9a806af13d38163156302097e405a7b145b6cc (patch) | |
| tree | c079f3c4e027dc730892630f667b0e6401736043 /src | |
Tantivy full text index, link graph, inline query filters, Ollama
embeddings with paragraph chunks, and an Elisp summary/article UI.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Diffstat (limited to 'src')
| -rw-r--r-- | src/adapter/mod.rs | 275 | ||||
| -rw-r--r-- | src/adapter/org.rs | 84 | ||||
| -rw-r--r-- | src/adapter/txt.rs | 85 | ||||
| -rw-r--r-- | src/chunk.rs | 314 | ||||
| -rw-r--r-- | src/graph.rs | 187 | ||||
| -rw-r--r-- | src/main.rs | 588 | ||||
| -rw-r--r-- | src/note.rs | 36 | ||||
| -rw-r--r-- | src/query.rs | 335 | ||||
| -rw-r--r-- | src/text.rs | 188 | ||||
| -rw-r--r-- | src/vec.rs | 651 |
10 files changed, 2743 insertions, 0 deletions
diff --git a/src/adapter/mod.rs b/src/adapter/mod.rs new file mode 100644 index 0000000..8f56b9b --- /dev/null +++ b/src/adapter/mod.rs @@ -0,0 +1,275 @@ +//! Format adapters. Each adapter turns the text of a file into notes. + +pub mod org; +pub mod txt; + +use crate::note::{Link, Note}; +use std::path::Path; + +/// Parse the file at `path`, whose content is `text`. Returns `None` +/// when no adapter handles the extension. +pub fn parse_file(path: &Path, text: &str) -> Option<Vec<Note>> { + let p = path.to_string_lossy().to_string(); + match path.extension().and_then(|e| e.to_str()) { + Some("org") => Some(org::parse(&p, text)), + Some("txt") => Some(txt::parse(&p, text)), + _ => None, + } +} + +/// The fields every adapter collects before building a note. +#[derive(Default)] +pub struct Fields { + pub id: Option<String>, + pub title: String, + pub date: Option<String>, + pub kind: Option<String>, + pub tags: Vec<String>, + pub parent: Option<String>, + pub links: Option<String>, + pub context: Option<String>, + pub summary: Option<String>, +} + +impl Fields { + /// Store the value of a metadata key. Returns false for unknown keys. + pub fn set(&mut self, key: &str, value: &str) -> bool { + let value = value.trim(); + let some = |v: &str| { + if v.is_empty() { + None + } else { + Some(v.to_string()) + } + }; + match key.to_ascii_lowercase().as_str() { + "id" => self.id = some(value), + "title" => self.title = value.to_string(), + "date" => self.date = some(value), + "kind" => self.kind = some(value), + "tags" => { + for t in split_list(value) { + if !self.tags.contains(&t) { + self.tags.push(t); + } + } + } + "parent" => self.parent = some(value), + "links" => self.links = some(value), + "context" => self.context = some(value), + "summary" => self.summary = some(value), + _ => return false, + } + true + } +} + +/// Assemble a note from its fields, its body, and its place in the file. +#[allow(clippy::too_many_arguments)] +pub fn build( + path: &str, + text: &str, + start: usize, + end: usize, + line: usize, + fields: Fields, + body: String, +) -> Note { + let mut links = fields + .links + .as_deref() + .map(parse_links_prop) + .unwrap_or_default(); + for l in body_links(&body) { + if !links.iter().any(|x| x.dst == l.dst) { + links.push(l); + } + } + Note { + id: fields.id, + path: path.to_string(), + start, + end, + line, + hash: blake3::hash(text[start..end].as_bytes()).to_hex().to_string(), + title: fields.title, + date: fields.date, + kind: fields.kind, + tags: fields.tags, + parent: fields.parent, + links, + context: fields.context, + summary: fields.summary, + body, + } +} + +/// Split on whitespace, commas and colons. +pub fn split_list(s: &str) -> Vec<String> { + s.split(|c: char| c.is_whitespace() || c == ',' || c == ':') + .filter(|t| !t.is_empty()) + .map(|t| t.to_string()) + .collect() +} + +fn is_kind(s: &str) -> bool { + !s.is_empty() + && s + .chars() + .all(|c| c.is_ascii_alphabetic() || c == '-' || c == '_') +} + +/// Parse the value of a LINKS property: tokens `ID` or `kind:ID`. +pub fn parse_links_prop(s: &str) -> Vec<Link> { + let mut out = Vec::new(); + for tok in s.split_whitespace() { + let link = match tok.split_once(':') { + Some(("notes" | "note", rest)) if !rest.is_empty() => Link { + dst: rest.to_string(), + kind: None, + }, + Some((k, rest)) if is_kind(k) && !rest.is_empty() => Link { + dst: rest.to_string(), + kind: Some(k.to_string()), + }, + _ => Link { + dst: tok.to_string(), + kind: None, + }, + }; + out.push(link); + } + out +} + +/// Split the path of a `note:` link into its kind and its ID. +/// +/// The path is `ID` for an untyped link and `:KIND:ID` for a typed one, +/// so `[[note:ID]]` and `[[note::KIND:ID]]` are the two forms. +fn split_note_path(p: &str) -> Option<(Option<String>, String)> { + if p.is_empty() || p.contains(char::is_whitespace) { + return None; + } + match p.strip_prefix(':') { + Some(rest) => { + let (k, id) = rest.split_once(':')?; + let ok = !k.is_empty() + && k.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + (ok && !id.is_empty()).then(|| (Some(k.to_string()), id.to_string())) + } + None => Some((None, p.to_string())), + } +} + +/// Find the `[[note:ID][description]]` links in a body, and their kinds. +/// The description is for humans (usually the title of the note); only +/// the ID is used to locate the note. +pub fn body_links(body: &str) -> Vec<Link> { + let mut out: Vec<Link> = Vec::new(); + let mut pos = 0; + while let Some(i) = body[pos..].find("[[note:") { + let start = pos + i + "[[note:".len(); + let rest = &body[start..]; + match rest.find(']') { + Some(j) if !rest[..j].contains('\n') => { + let closes = rest[j..].starts_with("]]") || rest[j..].starts_with("]["); + if closes { + if let Some((kind, dst)) = split_note_path(&rest[..j]) { + match out.iter_mut().find(|l| l.dst == dst) { + // A typed link is more informative than an untyped one. + Some(l) => l.kind = l.kind.take().or(kind), + None => out.push(Link { dst, kind }), + } + } + } + pos = start + j; + } + _ => pos = start, + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn body_links_read_the_id_and_ignore_the_description() { + let l = body_links( + "see [[note:abc-1][A title: with colon]] and [[note:<x.y@z>][d]] or [[note:q]] twice [[note:q][again]]", + ); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!(got, vec![("abc-1", None), ("<x.y@z>", None), ("q", None)]); + } + + #[test] + fn body_links_may_be_typed() { + let l = body_links("[[note::supports:a1][A]] [[note::contra:<b@c>]] [[note:d]] [[note:d2][x]]"); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!( + got, + vec![("a1", Some("supports")), ("<b@c>", Some("contra")), ("d", None), ("d2", None)] + ); + // The typed form wins when the same note is linked twice. + let l = body_links("[[note:a]] and [[note::cites:a]]"); + assert_eq!(l.len(), 1); + assert_eq!(l[0].kind.as_deref(), Some("cites")); + } + + #[test] + fn body_links_ignore_other_links() { + let body = "[[https://a.b][web]] [[file:x.org]] [[*heading]] [[#custom]] [[./rel][r]] \\ + [[two words]] [[n1][fuzzy]] [[note:]] [[note:two words]] [[note::bad]] \\ + [[notes:x]] [[note:a\nb]]"; + assert!(body_links(body).is_empty(), "{:?}", body_links(body)); + } + + #[test] + fn links_property_has_typed_and_plain_tokens() { + let l = parse_links_prop("a1 supports:b2 notes:c3 <d@e>"); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!( + got, + vec![("a1", None), ("b2", Some("supports")), ("c3", None), ("<d@e>", None)] + ); + } + + #[test] + fn org_notes_are_top_level_headings_only() { + let text = "intro\n* One :a:b:\n:PROPERTIES:\n:ID: 1\n:PARENT: 0\n:END:\nbody\n** sub\nmore\n* Two\ntext\n"; + let notes = org::parse("/x.org", text); + assert_eq!(notes.len(), 2); + assert_eq!(notes[0].id.as_deref(), Some("1")); + assert_eq!(notes[0].title, "One"); + assert_eq!(notes[0].tags, vec!["a", "b"]); + assert_eq!(notes[0].parent.as_deref(), Some("0")); + assert!(notes[0].body.contains("** sub")); + assert_eq!(notes[0].line, 2); + assert_eq!(&text[notes[0].start..notes[0].end].lines().next(), &Some("* One :a:b:")); + assert_eq!(notes[1].id, None); + } + + #[test] + fn txt_notes_are_split_at_form_feeds() { + let text = "ID: 1\nTitle: A\nTags: x y\n\nbody a\n\x0c\nID: 2\nTitle: B\n\nbody b\n\x0c\n\n"; + let notes = txt::parse("/x.txt", text); + assert_eq!(notes.len(), 2); + assert_eq!(notes[0].tags, vec!["x", "y"]); + assert_eq!(notes[0].body, "body a"); + assert_eq!(notes[1].id.as_deref(), Some("2")); + assert_eq!(notes[1].line, 7); + } + + #[test] + fn txt_note_without_header_takes_its_title_from_the_text() { + let notes = txt::parse("/x.txt", "just a thought\nmore\n"); + assert_eq!(notes.len(), 1); + assert_eq!(notes[0].title, "just a thought"); + assert_eq!(notes[0].id, None); + } +} diff --git a/src/adapter/org.rs b/src/adapter/org.rs new file mode 100644 index 0000000..a999814 --- /dev/null +++ b/src/adapter/org.rs @@ -0,0 +1,84 @@ +//! Org adapter: each top-level heading is a note. Metadata lives in +//! the heading (title, tags) and in its property drawer. + +use super::{Fields, build}; +use crate::note::Note; + +fn is_tag_string(s: &str) -> bool { + s.len() > 2 + && s.starts_with(':') + && s.ends_with(':') + && s.chars() + .all(|c| c.is_alphanumeric() || "_@#%:.".contains(c)) +} + +/// Split a heading line into its title and its tags. +fn parse_heading(line: &str) -> (String, Vec<String>) { + let h = line.trim_start_matches('*').trim(); + if let Some(idx) = h.rfind(char::is_whitespace) { + let last = &h[idx + 1..]; + if is_tag_string(last) { + let tags = last + .split(':') + .filter(|t| !t.is_empty()) + .map(|t| t.to_string()) + .collect(); + return (h[..idx].trim().to_string(), tags); + } + } + (h.to_string(), Vec::new()) +} + +fn is_planning(line: &str) -> bool { + let t = line.trim_start(); + t.starts_with("SCHEDULED:") || t.starts_with("DEADLINE:") || t.starts_with("CLOSED:") +} + +pub fn parse(path: &str, text: &str) -> Vec<Note> { + // Find the top-level headings: a single star at column 0. + let mut heads: Vec<(usize, usize)> = Vec::new(); + let mut off = 0; + for (ln, line) in text.split_inclusive('\n').enumerate() { + let t = line.trim_end(); + if line.starts_with("* ") || t == "*" { + heads.push((off, ln + 1)); + } + off += line.len(); + } + let mut notes = Vec::new(); + for (i, &(start, line_no)) in heads.iter().enumerate() { + let end = heads.get(i + 1).map(|h| h.0).unwrap_or(text.len()); + let chunk = &text[start..end]; + let mut lines = chunk.lines(); + let heading = lines.next().unwrap_or(""); + let (title, tags) = parse_heading(heading); + let mut fields = Fields { + title, + tags, + ..Default::default() + }; + let mut rest: Vec<&str> = lines.collect(); + // Skip planning lines, then read the property drawer, if any. + let mut k = 0; + while k < rest.len() && is_planning(rest[k]) { + k += 1; + } + let mut body_from = 0; + if k < rest.len() && rest[k].trim() == ":PROPERTIES:" { + let mut j = k + 1; + while j < rest.len() && rest[j].trim() != ":END:" { + let l = rest[j].trim(); + if let Some(r) = l.strip_prefix(':') { + if let Some((key, val)) = r.split_once(':') { + fields.set(key, val); + } + } + j += 1; + } + body_from = (j + 1).min(rest.len()); + } + let body = rest.split_off(body_from).join("\n"); + notes.push(build(path, text, start, end, line_no, fields, body)); + } + notes +} diff --git a/src/adapter/txt.rs b/src/adapter/txt.rs new file mode 100644 index 0000000..ddea799 --- /dev/null +++ b/src/adapter/txt.rs @@ -0,0 +1,85 @@ +//! Plain text adapter. Notes are separated by a form-feed line. A +//! note starts with `Key: value` lines, then a blank line, then the +//! body. + +use super::{Fields, build}; +use crate::note::Note; + +const KEYS: [&str; 9] = [ + "id", "title", "date", "kind", "tags", "parent", "links", "context", "summary", +]; + +/// Parse `Key: value` if the key is a known one. +fn header_line(line: &str) -> Option<(&str, &str)> { + let (k, v) = line.split_once(':')?; + let lk = k.trim().to_ascii_lowercase(); + if KEYS.contains(&lk.as_str()) && k.trim().len() == k.len() { + Some((k, v)) + } else { + None + } +} + +pub fn parse(path: &str, text: &str) -> Vec<Note> { + // Split into chunks at the form-feed lines. + let mut chunks: Vec<(usize, usize, usize)> = Vec::new(); // start, end, line + let mut off = 0; + let mut cur_start = 0; + let mut cur_line = 1; + for (ln, line) in text.split_inclusive('\n').enumerate() { + if line.trim_end_matches(['\r', '\n']) == "\x0c" { + chunks.push((cur_start, off, cur_line)); + cur_start = off + line.len(); + cur_line = ln + 2; + } + off += line.len(); + } + chunks.push((cur_start, text.len(), cur_line)); + + let mut notes = Vec::new(); + for (start, end, line_no) in chunks { + let chunk = &text[start..end]; + if chunk.trim().is_empty() { + continue; + } + let mut fields = Fields::default(); + let mut body_lines: Vec<&str> = Vec::new(); + let mut in_header = true; + let mut seen = 0; + for l in chunk.lines() { + if in_header { + if seen == 0 && l.trim().is_empty() { + continue; + } + if let Some((k, v)) = header_line(l) { + fields.set(k, v); + seen += 1; + continue; + } + in_header = false; + // The blank line that ends the header is not part of the body. + if seen > 0 && l.trim().is_empty() { + continue; + } + } + body_lines.push(l); + } + if fields.title.is_empty() { + fields.title = body_lines + .iter() + .find(|l| !l.trim().is_empty()) + .map(|l| l.trim().chars().take(80).collect()) + .unwrap_or_default(); + } + notes.push(build( + path, + text, + start, + end, + line_no, + fields, + body_lines.join("\n"), + )); + } + notes +} diff --git a/src/chunk.rs b/src/chunk.rs new file mode 100644 index 0000000..b17316b --- /dev/null +++ b/src/chunk.rs @@ -0,0 +1,314 @@ +//! Cut a note into chunks for the embeddings. +//! +//! A chunk is a paragraph or a few paragraphs. Blocks that must stay +//! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A +//! paragraph that is only a formula is merged into a neighbour, because +//! alone it would be embedded as "some mathematics" and look like every +//! other formula. Each chunk is prefixed with the title, so a bare +//! "this follows from the lemma" still knows what it is about. + +/// Paragraphs shorter than this are merged with a neighbour. +const MIN: usize = 200; +/// A chunk is not grown beyond this (a single paragraph may exceed it). +const MAX: usize = 1500; +/// A paragraph with fewer words than this, once the math is removed, +/// is a formula paragraph. +const MIN_WORDS: usize = 5; + +#[derive(Debug, Clone, PartialEq)] +pub struct Chunk { + pub text: String, + /// 0-based line of the body where the chunk starts. + pub line: usize, +} + +/// A paragraph of the body: its text and its first line. +struct Para { + text: String, + line: usize, +} + +/// Replace `[[note:ID][desc]]` by `desc` and org links by their text. +pub fn strip_links(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find("[[") { + out.push_str(&rest[..i]); + let tail = &rest[i + 2..]; + match tail.find("]]") { + Some(j) => { + let inner = &tail[..j]; + match inner.split_once("][") { + Some((_, desc)) => out.push_str(desc), + None => out.push_str(inner), + } + rest = &tail[j + 2..]; + } + None => { + out.push_str(&rest[i..]); + rest = ""; + } + } + } + out.push_str(rest); + out +} + +/// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed. +fn without_math(s: &str) -> String { + let mut out = String::new(); + let mut rest = s; + 'outer: while !rest.is_empty() { + for (open, close) in [ + ("$$", "$$"), + ("\\[", "\\]"), + ("\\(", "\\)"), + ("$", "$"), + ] { + if let Some(r) = rest.strip_prefix(open) { + match r.find(close) { + Some(j) => rest = &r[j + close.len()..], + None => rest = "", + } + out.push(' '); + continue 'outer; + } + } + if rest.starts_with("\\begin{") { + match rest.find("\\end{") { + Some(j) => { + let after = &rest[j..]; + rest = match after.find('}') { + Some(k) => &after[k + 1..], + None => "", + }; + out.push(' '); + continue 'outer; + } + None => break, + } + } + let c = rest.chars().next().unwrap(); + out.push(c); + rest = &rest[c.len_utf8()..]; + } + out +} + +/// Does the paragraph hold almost nothing but math? +fn is_formula(p: &str) -> bool { + let t = p.trim(); + if t.is_empty() { + return false; + } + let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{"); + if !has_math { + return false; + } + let left = without_math(t); + // CJK has no spaces: count its characters as words. + let words = left + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 }) + .sum::<usize>(); + words < MIN_WORDS +} + +/// Split into paragraphs, keeping blocks whole. +fn paragraphs(body: &str) -> Vec<Para> { + let mut out: Vec<Para> = Vec::new(); + let mut cur: Vec<&str> = Vec::new(); + let mut start = 0; + // Inside a block that ends with this line (lowercased, trimmed prefix). + let mut block_end: Option<String> = None; + let mut dollars = false; + let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec<Para>| { + if !cur.is_empty() { + out.push(Para { text: cur.join("\n"), line: start }); + cur.clear(); + } + }; + for (i, l) in body.lines().enumerate() { + let t = l.trim(); + let tl = t.to_lowercase(); + if let Some(end) = &block_end { + cur.push(l); + if tl.starts_with(end.as_str()) { + block_end = None; + } + continue; + } + if dollars { + cur.push(l); + if t.contains("$$") { + dollars = false; + } + continue; + } + if t.is_empty() { + flush(&mut cur, start, &mut out); + continue; + } + let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' '); + if heading { + flush(&mut cur, start, &mut out); + } + if cur.is_empty() { + start = i; + } + if let Some(rest) = tl.strip_prefix("#+begin_") { + let name = rest.split_whitespace().next().unwrap_or(""); + block_end = Some(format!("#+end_{}", name)); + } else if let Some(rest) = t.strip_prefix("\\begin{") { + let name = rest.split('}').next().unwrap_or(""); + block_end = Some(format!("\\end{{{}}}", name.to_lowercase())); + } else if t.matches("$$").count() % 2 == 1 { + dollars = true; + } + // A one-line block (\begin{x} .. \end{x}) closes at once. + if let Some(end) = &block_end { + if tl[1..].contains(end.as_str()) { + block_end = None; + } + } + cur.push(l); + } + flush(&mut cur, start, &mut out); + out +} + +/// The chunks of a note. A note without a body gets one chunk from its +/// title, tags and summary. +pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec<Chunk> { + let body = strip_links(body); + let paras = paragraphs(&body); + // Merge formula paragraphs into the previous paragraph (or the next one). + let mut merged: Vec<Para> = Vec::new(); + let mut carry: Option<Para> = None; + for p in paras { + let formula = is_formula(&p.text); + let p = match carry.take() { + Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line }, + None => p, + }; + if formula && merged.is_empty() { + // Opens the note: it joins the next paragraph. + carry = Some(p); + } else if formula { + let last = merged.last_mut().unwrap(); + last.text = format!("{}\n{}", last.text, p.text); + } else { + merged.push(p); + } + } + if let Some(c) = carry { + merged.push(c); + } + // Merge small paragraphs, up to MAX. + let mut out: Vec<Chunk> = Vec::new(); + let mut small = false; + for p in merged { + match out.last_mut() { + Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => { + last.text.push_str("\n\n"); + last.text.push_str(&p.text); + small = last.text.len() < MIN; + } + _ => { + small = p.text.len() < MIN; + out.push(Chunk { text: p.text, line: p.line }); + } + } + } + if out.is_empty() { + let mut t = String::new(); + if let Some(s) = summary { + t.push_str(s); + } + if !tags.is_empty() { + t.push_str("\n"); + t.push_str(&tags.join(" ")); + } + out.push(Chunk { text: t, line: 0 }); + } + for c in &mut out { + c.text = format!("{}\n\n{}", title, c.text.trim()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn long(s: &str) -> String { + format!("{} {}", s, "word ".repeat(60)) + } + + #[test] + fn formula_paragraph_joins_the_previous_one() { + let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 2, "{:?}", c); + assert!(c[0].text.contains("x^2")); + assert!(!c[1].text.contains("x^2")); + } + + #[test] + fn formula_at_the_start_joins_the_next() { + let body = format!("$$a=b$$\n\n{}", long("Prose")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose")); + } + + #[test] + fn prose_with_inline_math_is_not_a_formula() { + assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent.")); + assert!(is_formula("\\[ \\int_0^1 f \\]")); + assert!(is_formula("$a$ and $b$")); + } + + #[test] + fn source_blocks_stay_whole() { + let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two")); + let p = paragraphs(&body); + assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::<Vec<_>>()); + assert!(p[1].text.contains("(b)")); + } + + #[test] + fn environment_stays_whole() { + let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore"; + let p = paragraphs(body); + assert_eq!(p.len(), 3); + assert!(p[1].text.contains("\\end{align}")); + } + + #[test] + fn small_paragraphs_are_merged() { + let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third"); + assert_eq!(c.len(), 1); + assert!(c[0].text.starts_with("T\n\n")); + } + + #[test] + fn links_become_their_description() { + assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2"); + } + + #[test] + fn empty_body_uses_title_and_tags() { + let c = chunks("Title", &["x".into()], Some("sum"), ""); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("sum") && c[0].text.contains("x")); + } + + #[test] + fn line_numbers_point_at_the_paragraph() { + let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.iter().map(|x| x.line).collect::<Vec<_>>(), vec![0, 2, 4]); + } +} diff --git a/src/graph.rs b/src/graph.rs new file mode 100644 index 0000000..460bdec --- /dev/null +++ b/src/graph.rs @@ -0,0 +1,187 @@ +//! The manifest (what was indexed) and the graph built from it. +//! +//! The manifest is a small JSON file. The graph is rebuilt in memory +//! on every call; with thousands of notes this takes microseconds. + +use crate::note::Note; +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::path::Path; + +#[derive(Serialize, Deserialize)] +pub struct FileEntry { + pub hash: String, + pub notes: Vec<Note>, +} + +#[derive(Default, Serialize, Deserialize)] +pub struct Manifest { + pub files: BTreeMap<String, FileEntry>, +} + +impl Manifest { + pub fn load(path: &Path) -> Manifest { + std::fs::read_to_string(path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } + + /// Write through a temporary file so a crash cannot leave half a file. + pub fn save(&self, path: &Path) -> Result<()> { + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, serde_json::to_vec(self)?)?; + std::fs::rename(&tmp, path)?; + Ok(()) + } +} + +/// How a note is reached from the note we started at. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Rel { + Parent, + Child, + Link, + Backlink, +} + +impl Rel { + pub fn name(self) -> &'static str { + match self { + Rel::Parent => "parent", + Rel::Child => "child", + Rel::Link => "link", + Rel::Backlink => "backlink", + } + } +} + +pub struct Graph { + pub notes: Vec<Note>, + by_id: HashMap<String, usize>, + children: HashMap<String, Vec<usize>>, + backlinks: HashMap<String, Vec<usize>>, + pub duplicates: Vec<(String, Vec<usize>)>, +} + +impl Graph { + pub fn build(m: &Manifest) -> Graph { + let notes: Vec<Note> = m.files.values().flat_map(|f| f.notes.clone()).collect(); + let mut by_id: HashMap<String, usize> = HashMap::new(); + let mut seen: HashMap<String, Vec<usize>> = HashMap::new(); + for (i, n) in notes.iter().enumerate() { + if let Some(id) = &n.id { + seen.entry(id.clone()).or_default().push(i); + by_id.entry(id.clone()).or_insert(i); + } + } + let mut duplicates: Vec<(String, Vec<usize>)> = + seen.into_iter().filter(|(_, v)| v.len() > 1).collect(); + duplicates.sort(); + let mut children: HashMap<String, Vec<usize>> = HashMap::new(); + let mut backlinks: HashMap<String, Vec<usize>> = HashMap::new(); + for (i, n) in notes.iter().enumerate() { + if n.id.is_none() { + continue; + } + if let Some(p) = &n.parent { + children.entry(p.clone()).or_default().push(i); + } + for l in &n.links { + backlinks.entry(l.dst.clone()).or_default().push(i); + } + } + Graph { + notes, + by_id, + children, + backlinks, + duplicates, + } + } + + pub fn get(&self, id: &str) -> Option<&Note> { + self.by_id.get(id).map(|&i| &self.notes[i]) + } + + pub fn parent_of(&self, id: &str) -> Option<&Note> { + self.get(id) + .and_then(|n| n.parent.as_deref()) + .and_then(|p| self.get(p)) + } + + pub fn children_of(&self, id: &str) -> Vec<&Note> { + self.idx(self.children.get(id)) + } + + pub fn backlinks_of(&self, id: &str) -> Vec<&Note> { + self.idx(self.backlinks.get(id)) + } + + pub fn outlinks_of(&self, id: &str) -> Vec<&Note> { + match self.get(id) { + Some(n) => n.links.iter().filter_map(|l| self.get(&l.dst)).collect(), + None => Vec::new(), + } + } + + fn idx(&self, v: Option<&Vec<usize>>) -> Vec<&Note> { + match v { + Some(v) => v.iter().map(|&i| &self.notes[i]).collect(), + None => Vec::new(), + } + } + + /// Does the note have a link or a backlink or a parent or a child? + pub fn is_connected(&self, n: &Note) -> bool { + let id = match &n.id { + Some(id) => id, + None => return false, + }; + n.parent.is_some() + || !n.links.is_empty() + || self.children.contains_key(id) + || self.backlinks.contains_key(id) + } + + /// Breadth-first search over all relations, ignoring direction. + /// Returns (note, distance, relation at the first step). + pub fn reach(&self, id: &str, hops: usize) -> Vec<(&Note, usize, Rel)> { + let mut out = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + seen.insert(id); + let mut queue: VecDeque<(&str, usize, Option<Rel>)> = VecDeque::new(); + queue.push_back((id, 0, None)); + while let Some((cur, d, first)) = queue.pop_front() { + if d == hops { + continue; + } + let mut next: Vec<(&Note, Rel)> = Vec::new(); + if let Some(p) = self.parent_of(cur) { + next.push((p, Rel::Parent)); + } + for c in self.children_of(cur) { + next.push((c, Rel::Child)); + } + for l in self.outlinks_of(cur) { + next.push((l, Rel::Link)); + } + for b in self.backlinks_of(cur) { + next.push((b, Rel::Backlink)); + } + for (n, rel) in next { + let nid = match n.id.as_deref() { + Some(x) => x, + None => continue, + }; + if seen.insert(nid) { + let r = first.unwrap_or(rel); + out.push((n, d + 1, r)); + queue.push_back((nid, d + 1, Some(r))); + } + } + } + out + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9824351 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,588 @@ +//! elel: Emacs Lexical Embedding Link. A notes engine. +//! +//! This program only reads the note files. It never writes to them. +//! Everything it stores lives in the index directory and can be rebuilt. + +mod adapter; +mod chunk; +mod graph; +mod note; +mod query; +mod text; +mod vec; + +use anyhow::{Context, Result, bail}; +use clap::{Parser, Subcommand}; +use graph::{FileEntry, Graph, Manifest}; +use note::Note; +use query::Query; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use text::Text; +use vec::{Ollama, Store}; + +#[derive(Parser)] +#[command(version, about = "Search and link notes kept in plain files")] +struct Cli { + /// Directory that holds the note files. + #[arg(long, env = "ELEL_ROOT", global = true)] + root: Option<PathBuf>, + /// Directory for the derived index. + #[arg(long, env = "ELEL_INDEX", global = true)] + index: Option<PathBuf>, + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Scan the note files and update the index. + Index { + /// Re-read every file, even the unchanged ones. + #[arg(long)] + force: bool, + /// Do not compute embeddings (no Ollama needed). + #[arg(long)] + no_embed: bool, + }, + /// Search notes. The query may hold filters such as `tag:X`, + /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README); + /// the other words go to the full text index. An empty query lists + /// every note. + Search { + query: Option<String>, + #[arg(short, default_value_t = 50)] + n: usize, + /// Also rank by meaning: fuse the text search with the embeddings. + #[arg(long)] + hybrid: bool, + }, + /// Print how many notes each query matches, one line per query. + Count { queries: Vec<String> }, + /// Show the neighbourhood of a note. + Hood { + id: String, + #[arg(long, default_value_t = 1)] + hops: usize, + }, + /// Print a note with its body, read fresh from its file. + Get { id: String }, + /// Report notes without IDs, duplicate IDs and dangling links. + Check, + /// List the tags with the number of notes that carry each. + Tags { + /// Rank the tags by how close their notes are to this note. + #[arg(long)] + like: Option<String>, + }, +} + +struct Ctx { + root: PathBuf, + index: PathBuf, +} + +impl Ctx { + fn new(cli: &Cli) -> Result<Ctx> { + let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?); + let root = cli.root.clone().unwrap_or_else(|| home.join("notes")); + let index = cli + .index + .clone() + .unwrap_or_else(|| home.join(".local/share/elel")); + Ok(Ctx { root, index }) + } + + fn manifest_path(&self) -> PathBuf { + self.index.join("graph.json") + } + + fn tantivy_dir(&self) -> PathBuf { + self.index.join("tantivy") + } +} + +/// A note as shown to the front end. +#[derive(Serialize)] +struct View<'a> { + #[serde(flatten)] + note: &'a Note, + #[serde(skip_serializing_if = "Option::is_none")] + score: Option<f32>, + #[serde(skip_serializing_if = "Option::is_none")] + snippet: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + dist: Option<usize>, + #[serde(skip_serializing_if = "Option::is_none")] + rel: Option<&'static str>, + /// For hits by meaning: the line of the body where the best chunk starts. + #[serde(skip_serializing_if = "Option::is_none")] + chunk_line: Option<usize>, + /// Number of notes that link here. + backlinks: usize, +} + +fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> { + View { + note: n, + score: None, + snippet: None, + dist: None, + rel: None, + chunk_line: None, + backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0), + } +} + +fn print<T: Serialize>(v: &T) -> Result<()> { + println!("{}", serde_json::to_string(v)?); + Ok(()) +} + +fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> { + let mut entries: Vec<_> = std::fs::read_dir(dir) + .with_context(|| format!("cannot read {}", dir.display()))? + .filter_map(|e| e.ok()) + .collect(); + entries.sort_by_key(|e| e.file_name()); + for e in entries { + let p = e.path(); + let name = e.file_name().to_string_lossy().to_string(); + if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') { + continue; + } + if p.is_dir() { + collect_files(&p, out)?; + } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) { + out.push(p); + } + } + Ok(()) +} + +fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> { + std::fs::create_dir_all(&ctx.index)?; + let text = Text::open(&ctx.tantivy_dir())?; + let old = if text.fresh || force { + Manifest::default() + } else { + Manifest::load(&ctx.manifest_path()) + }; + let root = ctx + .root + .canonicalize() + .with_context(|| format!("no notes directory at {}", ctx.root.display()))?; + let mut files = Vec::new(); + collect_files(&root, &mut files)?; + + let mut new = Manifest::default(); + let mut changed: Vec<(String, Vec<Note>)> = Vec::new(); + for f in &files { + let key = f.to_string_lossy().to_string(); + let content = match std::fs::read_to_string(f) { + Ok(c) => c, + Err(e) => { + eprintln!("skipping {}: {}", key, e); + continue; + } + }; + let hash = blake3::hash(content.as_bytes()).to_hex().to_string(); + match old.files.get(&key) { + Some(e) if e.hash == hash => { + new.files.insert( + key, + FileEntry { + hash, + notes: e.notes.clone(), + }, + ); + } + _ => { + let notes = adapter::parse_file(f, &content).unwrap_or_default(); + changed.push((key.clone(), notes.clone())); + new.files.insert(key, FileEntry { hash, notes }); + } + } + } + let removed: Vec<String> = old + .files + .keys() + .filter(|k| !new.files.contains_key(*k)) + .cloned() + .collect(); + text.update(&removed, &changed)?; + new.save(&ctx.manifest_path())?; + let g = Graph::build(&new); + let mut report = serde_json::json!({ + "files": new.files.len(), + "changed": changed.len(), + "removed": removed.len(), + "notes": g.notes.len(), + }); + if !no_embed { + let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?; + report["embedded"] = r.embedded.into(); + report["pending"] = r.pending.into(); + if let Some(e) = r.error { + eprintln!("elel: embeddings stopped: {}", e); + report["embed_error"] = e.into(); + } + } + print(&report) +} + +fn load_graph(ctx: &Ctx) -> Result<Graph> { + let p = ctx.manifest_path(); + if !p.exists() { + bail!("no index yet; run `elel index` first"); + } + Ok(Graph::build(&Manifest::load(&p))) +} + +/// One answer of a query: the note, and for text queries the score +/// and the snippet. +struct Hit<'a> { + note: &'a Note, + score: Option<f32>, + snippet: Option<String>, + /// The best chunk, for hits by meaning. + row: Option<vec::Row>, +} + +/// The words of a text query that can be embedded: no `-x`, `OR`, or +/// field prefixes, and no quotes. +fn plain_words(words: &str) -> String { + words + .split_whitespace() + .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND") + .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"')) + .filter(|w| !w.is_empty()) + .collect::<Vec<_>>() + .join(" ") +} + +/// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words). +fn semantic( + ctx: &Ctx, + g: &Graph, + q: &Query, + m: &query::Matcher, + hybrid: bool, +) -> Result<Vec<vec::VecHit>> { + let store = Store::load(&ctx.index); + if store.is_empty() { + bail!("no embeddings yet; start Ollama and run `elel index`"); + } + let ollama = Ollama::from_env(); + if store.model != ollama.model { + bail!( + "the embeddings were made with {}, but the model is {}; run `elel index`", + store.model, + ollama.model + ); + } + let mut by_chunk = false; + let mut exclude: HashSet<String> = HashSet::new(); + let mut resolve = |s: &str| -> Result<Vec<f32>> { + if g.get(s).is_some() { + exclude.insert(s.to_string()); + store + .note_vec(s) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s)) + } else { + by_chunk = true; + ollama.embed_query(s) + } + }; + let mut pos = Vec::new(); + for s in &q.like { + pos.push(resolve(s)?); + } + let mut neg = Vec::new(); + for s in &q.unlike { + neg.push(resolve(s)?); + } + if pos.is_empty() { + if !neg.is_empty() { + bail!("-like: needs a like: to go with"); + } + by_chunk = true; + pos.push(ollama.embed_query(&plain_words(&q.words))?); + } + let qv = vec::rocchio(&pos, &neg).unwrap(); + let _ = hybrid; + let allow = |id: &str| match g.get(id) { + Some(n) => !exclude.contains(id) && m.pass(n), + None => false, + }; + Ok(store.rank(&qv, by_chunk, &allow)) +} + +fn run_query<'a>( + ctx: &Ctx, + g: &'a Graph, + query: &str, + n: usize, + hybrid: bool, +) -> Result<Vec<Hit<'a>>> { + let q = Query::parse(query)?; + let root = ctx + .root + .canonicalize() + .unwrap_or_else(|_| ctx.root.clone()) + .to_string_lossy() + .to_string(); + let m = q.matcher(g, &root); + let has_words = !q.words.trim().is_empty(); + let has_like = !q.like.is_empty() || !q.unlike.is_empty(); + let mut out = Vec::new(); + + // The ranking by meaning, if the query asks for it. + let mut by_meaning: Option<Vec<vec::VecHit>> = None; + if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) { + match semantic(ctx, g, &q, &m, hybrid) { + Ok(v) => by_meaning = Some(v), + Err(e) if has_like => return Err(e), + Err(e) => eprintln!("elel: {}; using the text search only", e), + } + } + + // The ranking by words. + let mut lexical: Vec<(&'a Note, f32, Option<String>)> = Vec::new(); + if has_words { + let text = Text::open(&ctx.tantivy_dir())?; + // Filters apply after the text search, so ask for plenty. + for f in text.search(&q.words, 100_000)? { + let note = match g.get(&f.id) { + Some(note) => note, + None => continue, + }; + if m.pass(note) { + lexical.push((note, f.score, f.snippet)); + } + } + } + + match by_meaning { + None if has_words => { + for (note, score, snippet) in lexical.into_iter().take(n) { + out.push(Hit { note, score: Some(score), snippet, row: None }); + } + } + None => { + let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect(); + // Newest first; notes without a date go last. + all.sort_by(|a, b| b.date.cmp(&a.date)); + for note in all.into_iter().take(n) { + out.push(Hit { note, score: None, snippet: None, row: None }); + } + } + Some(v) if !has_words => { + for h in v.into_iter().take(n) { + if let Some(note) = g.get(&h.id) { + out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row }); + } + } + } + Some(v) => { + let lists = vec![ + lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::<Vec<_>>(), + v.iter().map(|h| h.id.clone()).collect::<Vec<_>>(), + ]; + let mut rows: HashMap<String, vec::Row> = HashMap::new(); + for h in v { + if let Some(r) = h.row { + rows.insert(h.id, r); + } + } + let mut snippets: HashMap<String, Option<String>> = HashMap::new(); + for (note, _, s) in lexical { + if let Some(id) = ¬e.id { + snippets.insert(id.clone(), s); + } + } + for (id, score) in vec::rrf(&lists).into_iter().take(n) { + if let Some(note) = g.get(&id) { + out.push(Hit { + note, + score: Some(score), + snippet: snippets.remove(&id).flatten(), + row: rows.remove(&id), + }); + } + } + } + } + Ok(out) +} + +/// The first text of a chunk, for a hit by meaning. +fn chunk_snippet(h: &Hit, cache: &mut HashMap<String, Vec<Note>>) -> Option<String> { + let row = h.row.as_ref()?; + let path = &h.note.path; + if !cache.contains_key(path) { + let content = std::fs::read_to_string(path).ok()?; + let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default(); + cache.insert(path.clone(), v); + } + let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?; + let cs = vec::note_chunks(fresh); + let text = &cs.get(row.chunk)?.text; + let body = text.strip_prefix(&fresh.title).unwrap_or(text); + let flat = body.split_whitespace().collect::<Vec<_>>().join(" "); + Some(flat.chars().take(200).collect()) +} + +fn cmd_search(ctx: &Ctx, query: Option<String>, n: usize, hybrid: bool) -> Result<()> { + let g = load_graph(ctx)?; + let mut cache = HashMap::new(); + for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? { + let mut v = view(&g, h.note); + v.score = h.score; + v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache)); + v.chunk_line = h.row.as_ref().map(|r| r.line); + print(&v)?; + } + Ok(()) +} + +fn cmd_count(ctx: &Ctx, queries: Vec<String>) -> Result<()> { + let g = load_graph(ctx)?; + for q in queries { + let n = run_query(ctx, &g, &q, usize::MAX, false)?.len(); + print(&serde_json::json!({ "count": n }))?; + } + Ok(()) +} + +fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> { + let g = load_graph(ctx)?; + let center = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + print(&serde_json::json!({ "center": view(&g, center) }))?; + for (n, d, rel) in g.reach(id, hops.max(1)) { + let mut v = view(&g, n); + v.dist = Some(d); + v.rel = Some(rel.name()); + print(&v)?; + } + Ok(()) +} + +fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> { + let g = load_graph(ctx)?; + let n = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + // Read the file again, so the body is never stale. + let path = PathBuf::from(&n.path); + let content = std::fs::read_to_string(&path) + .with_context(|| format!("index out of date: cannot read {}", path.display()))?; + let fresh = adapter::parse_file(&path, &content) + .unwrap_or_default() + .into_iter() + .find(|x| x.id.as_deref() == Some(id)); + let note = fresh.as_ref().unwrap_or(n); + print(&serde_json::json!({ + "note": view(&g, note), + "body": note.body, + // The exact text of the note in its file, for display. + "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()), + "stale": fresh.is_none(), + "parent": g.parent_of(id).map(|x| view(&g, x)), + "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + })) +} + +fn cmd_check(ctx: &Ctx) -> Result<()> { + let g = load_graph(ctx)?; + let mut issues: Vec<serde_json::Value> = Vec::new(); + for n in &g.notes { + match &n.id { + None => issues.push(serde_json::json!({ + "issue": "missing-id", "path": n.path, "line": n.line, + "start": n.start, "title": n.title, + })), + Some(id) => { + if let Some(p) = &n.parent { + if g.get(p).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-parent", "id": id, "target": p, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for l in &n.links { + if g.get(&l.dst).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-link", "id": id, "target": l.dst, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + } + } + } + let mut by_path: BTreeMap<&str, usize> = BTreeMap::new(); + for (id, idxs) in &g.duplicates { + for &i in idxs { + let n = &g.notes[i]; + *by_path.entry(&n.path).or_default() += 1; + issues.push(serde_json::json!({ + "issue": "duplicate-id", "id": id, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for i in issues { + print(&i)?; + } + Ok(()) +} + +fn cmd_tags(ctx: &Ctx, like: Option<String>) -> Result<()> { + let g = load_graph(ctx)?; + if let Some(id) = like { + let store = Store::load(&ctx.index); + let v = store + .note_vec(&id) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?; + for (tag, count, score, has) in store.tag_scores(&g, &id, &v) { + print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?; + } + return Ok(()); + } + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for n in g.notes.iter().filter(|n| n.id.is_some()) { + for t in &n.tags { + *counts.entry(t).or_default() += 1; + } + } + let mut v: Vec<(&str, usize)> = counts.into_iter().collect(); + // Most used first, then alphabetical. + v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + for (tag, count) in v { + print(&serde_json::json!({ "tag": tag, "count": count }))?; + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let ctx = Ctx::new(&cli)?; + match cli.cmd { + Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed), + Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid), + Cmd::Count { queries } => cmd_count(&ctx, queries), + Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops), + Cmd::Get { id } => cmd_get(&ctx, &id), + Cmd::Check => cmd_check(&ctx), + Cmd::Tags { like } => cmd_tags(&ctx, like), + } +} diff --git a/src/note.rs b/src/note.rs new file mode 100644 index 0000000..ef38094 --- /dev/null +++ b/src/note.rs @@ -0,0 +1,36 @@ +//! The data model shared by all adapters and by the index. + +use serde::{Deserialize, Serialize}; + +/// A link from one note to another. Types are optional. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Link { + pub dst: String, + pub kind: Option<String>, +} + +/// One note. A file is a container of notes; the note is the unit. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Note { + /// `None` for a note that has no ID yet; `check` reports these. + pub id: Option<String>, + pub path: String, + /// Byte span of the note inside its file. + pub start: usize, + pub end: usize, + /// 1-based line number of the first line of the note. + pub line: usize, + /// Hash of the span, used to detect changed notes. + pub hash: String, + pub title: String, + pub date: Option<String>, + pub kind: Option<String>, + pub tags: Vec<String>, + pub parent: Option<String>, + pub links: Vec<Link>, + pub context: Option<String>, + pub summary: Option<String>, + /// The text to index. It is never written to the manifest. + #[serde(skip)] + pub body: String, +} diff --git a/src/query.rs b/src/query.rs new file mode 100644 index 0000000..2b53ae6 --- /dev/null +++ b/src/query.rs @@ -0,0 +1,335 @@ +//! The query language. +//! +//! A query is one string. Words such as `tag:X` are filters on the +//! metadata and the graph; everything else is passed on to the full +//! text index. See the README for the list. + +use crate::graph::Graph; +use crate::note::Note; +use anyhow::{Result, bail}; +use std::collections::HashSet; + +/// A pattern for tags and kinds: `*` (anything), `abc*` (prefix) or `abc`. +#[derive(Debug, PartialEq)] +pub enum Pat { + Any, + Prefix(String), + Exact(String), +} + +impl Pat { + fn new(v: &str) -> Pat { + match v.strip_suffix('*') { + Some("") => Pat::Any, + Some(p) => Pat::Prefix(p.to_string()), + None => Pat::Exact(v.to_string()), + } + } + + fn matches(&self, s: &str) -> bool { + match self { + Pat::Any => true, + Pat::Prefix(p) => s.starts_with(p.as_str()), + Pat::Exact(e) => s == e, + } + } +} + +/// A range of dates, compared as prefixes: `2025` is the whole year. +#[derive(Debug, PartialEq)] +pub struct DateRange { + pub from: Option<String>, + pub to: Option<String>, +} + +impl DateRange { + fn contains(&self, date: &str) -> bool { + let cut = |n: usize| date.get(..n).unwrap_or(date); + self.from.as_ref().is_none_or(|a| cut(a.len()) >= a.as_str()) + && self.to.as_ref().is_none_or(|b| cut(b.len()) <= b.as_str()) + } +} + +#[derive(Debug, Default, PartialEq)] +pub struct Query { + pub tags: Vec<Pat>, + pub no_tags: Vec<Pat>, + pub kinds: Vec<Pat>, + pub no_kinds: Vec<Pat>, + pub files: Vec<String>, + pub no_files: Vec<String>, + pub from: Vec<String>, + pub to: Vec<String>, + pub parent: Vec<String>, + pub thread: Vec<String>, + pub dates: Vec<DateRange>, + pub inbox: bool, + /// `like:X` and `-like:X`: note IDs or free text, for the embeddings. + pub like: Vec<String>, + pub unlike: Vec<String>, + /// What is left for the full text index. + pub words: String, +} + +/// Split on white space, but keep double-quoted stretches together. +/// The quotes stay in the tokens. +fn tokens(q: &str) -> Vec<String> { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quoted = false; + for c in q.chars() { + match c { + '"' => { + quoted = !quoted; + cur.push(c); + } + c if c.is_whitespace() && !quoted => { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } + c => cur.push(c), + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +fn unquote(v: &str) -> &str { + v.strip_prefix('"') + .map(|r| r.strip_suffix('"').unwrap_or(r)) + .unwrap_or(v) +} + +impl Query { + pub fn parse(q: &str) -> Result<Query> { + let mut out = Query::default(); + let mut words: Vec<String> = Vec::new(); + for tok in tokens(q) { + let (neg, rest) = match tok.strip_prefix('-') { + Some(r) if !r.is_empty() => (true, r), + _ => (false, tok.as_str()), + }; + let (key, value) = match rest.split_once(':') { + Some((k, v)) if !k.starts_with('"') => (k, v), + _ => { + words.push(tok); + continue; + } + }; + let known = matches!( + key, + "tag" | "kind" | "file" | "from" | "to" | "parent" | "thread" | "date" | "is" | "like" + ); + if !known { + // `title:foo` and friends belong to the text index. + words.push(tok); + continue; + } + let value = unquote(value); + if value.is_empty() { + bail!("empty value for {}:", key); + } + if neg && !matches!(key, "tag" | "kind" | "file" | "like") { + bail!("cannot negate {}:", key); + } + match key { + "tag" if neg => out.no_tags.push(Pat::new(value)), + "tag" => out.tags.push(Pat::new(value)), + "kind" if neg => out.no_kinds.push(Pat::new(value)), + "kind" => out.kinds.push(Pat::new(value)), + "file" if neg => out.no_files.push(value.to_string()), + "file" => out.files.push(value.to_string()), + "like" if neg => out.unlike.push(value.to_string()), + "like" => out.like.push(value.to_string()), + "from" => out.from.push(value.to_string()), + "to" => out.to.push(value.to_string()), + "parent" => out.parent.push(value.to_string()), + "thread" => out.thread.push(value.to_string()), + "date" => { + let range = match value.split_once("..") { + Some((a, b)) => DateRange { + from: (!a.is_empty()).then(|| a.to_string()), + to: (!b.is_empty()).then(|| b.to_string()), + }, + None => DateRange { + from: Some(value.to_string()), + to: Some(value.to_string()), + }, + }; + out.dates.push(range); + } + "is" => match value { + "inbox" => out.inbox = true, + _ => bail!("unknown is:{}", value), + }, + _ => unreachable!(), + } + } + out.words = words.join(" "); + Ok(out) + } + + /// Resolve the parts that need the graph, and return a matcher. + pub fn matcher<'a>(&'a self, g: &'a Graph, root: &'a str) -> Matcher<'a> { + let ids = |v: Vec<&Note>| -> HashSet<String> { + v.into_iter().filter_map(|n| n.id.clone()).collect() + }; + let mut sets: Vec<HashSet<String>> = Vec::new(); + for id in &self.from { + sets.push(ids(g.outlinks_of(id))); + } + for id in &self.to { + sets.push(ids(g.backlinks_of(id))); + } + for id in &self.parent { + sets.push(ids(g.children_of(id))); + } + for id in &self.thread { + sets.push(thread_of(g, id)); + } + Matcher { + q: self, + g, + root, + sets, + } + } +} + +/// All the notes in the thread of `id`: the root above it and every +/// descendant of that root. +fn thread_of(g: &Graph, id: &str) -> HashSet<String> { + let mut out = HashSet::new(); + if g.get(id).is_none() { + return out; + } + let mut root = id.to_string(); + let mut seen = HashSet::new(); + while seen.insert(root.clone()) { + match g.parent_of(&root).and_then(|p| p.id.clone()) { + Some(p) => root = p, + None => break, + } + } + let mut stack = vec![root]; + while let Some(cur) = stack.pop() { + if out.insert(cur.clone()) { + for c in g.children_of(&cur) { + if let Some(cid) = &c.id { + stack.push(cid.clone()); + } + } + } + } + out +} + +pub struct Matcher<'a> { + q: &'a Query, + g: &'a Graph, + root: &'a str, + sets: Vec<HashSet<String>>, +} + +impl Matcher<'_> { + pub fn pass(&self, note: &Note) -> bool { + let id = match ¬e.id { + Some(i) => i, + None => return false, + }; + let q = self.q; + let any_tag = |p: &Pat| note.tags.iter().any(|t| p.matches(t)); + let rel = note + .path + .strip_prefix(self.root) + .unwrap_or(¬e.path) + .trim_start_matches('/'); + q.tags.iter().all(|p| match p { + Pat::Any => !note.tags.is_empty(), + _ => any_tag(p), + }) && q.no_tags.iter().all(|p| match p { + Pat::Any => note.tags.is_empty(), + _ => !any_tag(p), + }) && q + .kinds + .iter() + .all(|p| note.kind.as_deref().is_some_and(|k| p.matches(k))) + && q.no_kinds.iter().all(|p| match p { + Pat::Any => note.kind.is_none(), + _ => !note.kind.as_deref().is_some_and(|k| p.matches(k)), + }) + && q.files.iter().all(|f| rel.contains(f.as_str())) + && q.no_files.iter().all(|f| !rel.contains(f.as_str())) + && self.sets.iter().all(|s| s.contains(id)) + && q.dates + .iter() + .all(|r| note.date.as_deref().is_some_and(|d| r.contains(d))) + && (!q.inbox || (!self.g.is_connected(note) && note.tags.is_empty())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filters_are_split_from_words() { + let q = Query::parse("tag:math -tag:old kind:idea \"exact phrase\" word title:foo").unwrap(); + assert_eq!(q.tags, vec![Pat::Exact("math".into())]); + assert_eq!(q.no_tags, vec![Pat::Exact("old".into())]); + assert_eq!(q.kinds, vec![Pat::Exact("idea".into())]); + assert_eq!(q.words, "\"exact phrase\" word title:foo"); + } + + #[test] + fn star_patterns() { + let q = Query::parse("tag:* -tag:alg* kind:*").unwrap(); + assert_eq!(q.tags, vec![Pat::Any]); + assert_eq!(q.no_tags, vec![Pat::Prefix("alg".into())]); + assert_eq!(q.kinds, vec![Pat::Any]); + assert_eq!(q.words, ""); + } + + #[test] + fn dates_and_quoted_values() { + let q = Query::parse("date:2025-03..2025-06 date:..2024 date:2020 file:\"my notes\"").unwrap(); + assert_eq!(q.dates.len(), 3); + assert!(q.dates[0].contains("2025-04-01T10:00:00")); + assert!(!q.dates[0].contains("2025-07-01")); + assert!(q.dates[0].contains("2025-06-30")); + assert!(q.dates[1].contains("2023-01-01")); + assert!(!q.dates[1].contains("2025-01-01")); + assert!(q.dates[2].contains("2020-05-14T23:21:08+08:00")); + assert_eq!(q.files, vec!["my notes"]); + } + + #[test] + fn errors() { + assert!(Query::parse("tag:").is_err()); + assert!(Query::parse("-from:x").is_err()); + assert!(Query::parse("is:weird").is_err()); + // A colon in an ordinary word is fine. + assert_eq!(Query::parse("a:b").unwrap().words, "a:b"); + // A lone dash is a word. + assert_eq!(Query::parse("- x").unwrap().words, "- x"); + } + + #[test] + fn like_filters() { + let q = Query::parse("like:abc-1 -like:\"moduli of curves\" tag:x word").unwrap(); + assert_eq!(q.like, vec!["abc-1"]); + assert_eq!(q.unlike, vec!["moduli of curves"]); + assert_eq!(q.words, "word"); + } + + #[test] + fn ids_with_angle_brackets() { + let q = Query::parse("thread:<a.b@c> to:abc-1 is:inbox").unwrap(); + assert_eq!(q.thread, vec!["<a.b@c>"]); + assert_eq!(q.to, vec!["abc-1"]); + assert!(q.inbox); + } +} diff --git a/src/text.rs b/src/text.rs new file mode 100644 index 0000000..466e065 --- /dev/null +++ b/src/text.rs @@ -0,0 +1,188 @@ +//! Full-text index on top of Tantivy. +//! +//! Two views of the text are indexed: the normal one, split into words, +//! and a bigram one, so that Chinese (which has no spaces) can be +//! matched by substring. + +use crate::note::Note; +use anyhow::Result; +use std::path::Path; +use tantivy::collector::TopDocs; +use tantivy::directory::MmapDirectory; +use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; +use tantivy::schema::{ + Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value, +}; +use tantivy::snippet::SnippetGenerator; +use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer}; +use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc}; + +/// Marks around highlighted words in snippets. These are private-use +/// characters, so they cannot appear in normal text. +pub const HL_START: char = '\u{e000}'; +pub const HL_END: char = '\u{e001}'; + +pub struct Text { + index: Index, + id: Field, + path: Field, + title: Field, + body: Field, + tags: Field, + grams: Field, + /// True if the index directory was created by this call. + pub fresh: bool, +} + +pub struct Found { + pub id: String, + pub score: f32, + pub snippet: Option<String>, +} + +fn has_cjk(s: &str) -> bool { + s.chars().any(|c| !c.is_ascii() && c.is_alphabetic()) +} + +impl Text { + pub fn open(dir: &Path) -> Result<Text> { + let fresh = !dir.join("meta.json").exists(); + std::fs::create_dir_all(dir)?; + let mut sb = Schema::builder(); + let id = sb.add_text_field("id", STRING | STORED); + let path = sb.add_text_field("path", STRING | STORED); + let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED); + let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED); + let tags = sb.add_text_field("tags", tantivy::schema::TEXT); + let grams = sb.add_text_field( + "grams", + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("bigram") + .set_index_option(IndexRecordOption::Basic), + ), + ); + let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?; + index.tokenizers().register( + "bigram", + TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?) + .filter(LowerCaser) + .build(), + ); + Ok(Text { + index, + id, + path, + title, + body, + tags, + grams, + fresh, + }) + } + + /// Remove the notes of `removed` files, and replace the notes of the + /// `changed` files. Only notes with an ID are indexed. + pub fn update(&self, removed: &[String], changed: &[(String, Vec<Note>)]) -> Result<()> { + let mut w: IndexWriter = self.index.writer(50_000_000)?; + for p in removed { + w.delete_term(Term::from_field_text(self.path, p)); + } + for (p, notes) in changed { + w.delete_term(Term::from_field_text(self.path, p)); + for n in notes { + let id = match &n.id { + Some(id) => id, + None => continue, + }; + let mut text = String::new(); + if let Some(s) = &n.summary { + text.push_str(s); + text.push('\n'); + } + text.push_str(&n.body); + let all = format!("{}\n{}", n.title, text); + w.add_document(doc!( + self.id => id.as_str(), + self.path => p.as_str(), + self.title => n.title.as_str(), + self.body => text, + self.tags => n.tags.join(" "), + self.grams => all, + ))?; + } + } + w.commit()?; + Ok(()) + } + + /// Build a query. Words go through the query parser (so phrases and + /// `title:foo` work); words with CJK characters become a conjunction + /// of bigrams. + fn build_query(&self, q: &str) -> Result<Box<dyn Query>> { + let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]); + parser.set_conjunction_by_default(); + if !has_cjk(q) { + return Ok(parser.parse_query(q)?); + } + let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new(); + for word in q.split_whitespace() { + if has_cjk(word) { + let chars: Vec<char> = word.to_lowercase().chars().collect(); + if chars.len() == 1 { + // A single character has no bigram; match any bigram with it. + let t = Term::from_field_text(self.grams, &chars[0].to_string()); + clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic)))); + } else { + for w in chars.windows(2) { + let g: String = w.iter().collect(); + let t = Term::from_field_text(self.grams, &g); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(t, IndexRecordOption::Basic)), + )); + } + } + } else { + clauses.push((Occur::Must, parser.parse_query(word)?)); + } + } + Ok(Box::new(BooleanQuery::new(clauses))) + } + + pub fn search(&self, q: &str, limit: usize) -> Result<Vec<Found>> { + let reader = self.index.reader()?; + let searcher = reader.searcher(); + let query = self.build_query(q)?; + let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; + let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok(); + let mut out = Vec::new(); + for (score, addr) in top { + let d: TantivyDocument = searcher.doc(addr)?; + let id = match d.get_first(self.id).and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => continue, + }; + let snippet = snipgen.as_ref().and_then(|g| { + let s = g.snippet_from_doc(&d); + let frag = s.fragment(); + if frag.is_empty() { + return None; + } + let mut out = String::new(); + let mut last = 0; + for r in s.highlighted() { + out.push_str(&frag[last..r.start]); + out.push(HL_START); + out.push_str(&frag[r.start..r.end]); + out.push(HL_END); + last = r.end; + } + out.push_str(&frag[last..]); + Some(out.replace('\n', " ")) + }); + out.push(Found { id, score, snippet }); + } + Ok(out) + } +} diff --git a/src/vec.rs b/src/vec.rs new file mode 100644 index 0000000..2c30ab5 --- /dev/null +++ b/src/vec.rs @@ -0,0 +1,651 @@ +//! The semantic side: embeddings from Ollama, a small vector store, and +//! the ranking on top of it. +//! +//! One vector per chunk of a note (see `chunk.rs`). The store keeps two +//! files next to each other: the vectors as f16 for exact scores, and one +//! sign bit per dimension for a fast first pass by Hamming distance when +//! there are very many chunks. Everything here can be rebuilt from the +//! note files, so it lives in the index directory. + +use crate::chunk; +use crate::graph::Graph; +use crate::note::Note; +use anyhow::{Context, Result, anyhow, bail}; +use half::f16; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub const DEFAULT_MODEL: &str = "qwen3-embedding:0.6b"; +const BATCH: usize = 16; +/// Above this many chunks, a Hamming pass narrows the candidates first. +const EXACT_LIMIT: usize = 2000; + +// ---------------------------------------------------------------- Ollama + +pub struct Ollama { + url: String, + pub model: String, +} + +/// Why a call to Ollama failed: it is not running, or it is something else. +#[derive(Debug)] +pub struct Unreachable(pub String); + +impl std::fmt::Display for Unreachable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for Unreachable {} + +impl Ollama { + pub fn from_env() -> Ollama { + Ollama { + url: std::env::var("ELEL_OLLAMA").unwrap_or_else(|_| "http://localhost:11434".into()), + model: std::env::var("ELEL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()), + } + } + + /// Embed texts, already formatted for the model. The vectors are + /// normalized to length 1. + pub fn embed(&self, inputs: &[String]) -> Result<Vec<Vec<f32>>> { + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_secs(3)) + .timeout(Duration::from_secs(300)) + .build(); + let resp = agent + .post(&format!("{}/api/embed", self.url.trim_end_matches('/'))) + .send_json(serde_json::json!({ + "model": self.model, + "input": inputs, + "truncate": true, + })); + let v: serde_json::Value = match resp { + Ok(r) => r.into_json()?, + Err(ureq::Error::Status(404, _)) => bail!( + "model {} is not available in Ollama; run `ollama pull {}`", + self.model, + self.model + ), + Err(ureq::Error::Status(code, r)) => { + let body = r.into_string().unwrap_or_default(); + bail!("Ollama answered {}: {}", code, body) + } + Err(e @ ureq::Error::Transport(_)) => { + return Err(anyhow::Error::new(Unreachable(format!( + "cannot reach Ollama at {} ({})", + self.url, e + )))); + } + }; + let arr = v["embeddings"] + .as_array() + .ok_or_else(|| anyhow!("Ollama gave no embeddings: {}", v))?; + let mut out = Vec::new(); + for e in arr { + let mut x: Vec<f32> = e + .as_array() + .ok_or_else(|| anyhow!("bad embedding"))? + .iter() + .map(|n| n.as_f64().unwrap_or(0.0) as f32) + .collect(); + normalize(&mut x); + out.push(x); + } + if out.len() != inputs.len() { + bail!("asked for {} embeddings, got {}", inputs.len(), out.len()); + } + Ok(out) + } + + /// A note chunk, as the model wants a document. + pub fn doc_text(&self, text: &str) -> String { + if self.model.starts_with("embeddinggemma") { + format!("title: none | text: {}", text) + } else if self.model.starts_with("nomic-embed") { + format!("search_document: {}", text) + } else { + text.to_string() + } + } + + /// A search text, as the model wants a query. + pub fn query_text(&self, text: &str) -> String { + if self.model.starts_with("qwen3-embedding") { + format!( + "Instruct: Given a search query, retrieve the notes that are about it\nQuery: {}", + text + ) + } else if self.model.starts_with("embeddinggemma") { + format!("task: search result | query: {}", text) + } else if self.model.starts_with("nomic-embed") { + format!("search_query: {}", text) + } else { + text.to_string() + } + } + + pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> { + Ok(self.embed(&[self.query_text(text)])?.remove(0)) + } +} + +// ------------------------------------------------------------ vector math + +pub fn normalize(v: &mut [f32]) { + let n = v.iter().map(|x| x * x).sum::<f32>().sqrt(); + if n > 0.0 { + for x in v.iter_mut() { + *x /= n; + } + } +} + +pub fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +/// The sign bits of a vector, packed eight to a byte. +pub fn to_bits(v: &[f32]) -> Vec<u8> { + let mut out = vec![0u8; v.len().div_ceil(8)]; + for (i, x) in v.iter().enumerate() { + if *x > 0.0 { + out[i / 8] |= 1 << (i % 8); + } + } + out +} + +pub fn hamming(a: &[u8], b: &[u8]) -> u32 { + a.iter().zip(b).map(|(x, y)| (x ^ y).count_ones()).sum() +} + +/// The normalized mean of some vectors. +pub fn mean(vs: &[&[f32]]) -> Option<Vec<f32>> { + let first = vs.first()?; + let mut m = vec![0.0f32; first.len()]; + for v in vs { + for (a, b) in m.iter_mut().zip(*v) { + *a += b; + } + } + normalize(&mut m); + Some(m) +} + +/// The merged vector of some liked and some disliked vectors: +/// `normalize(mean(pos) - 0.5 * mean(neg))`. +pub fn rocchio(pos: &[Vec<f32>], neg: &[Vec<f32>]) -> Option<Vec<f32>> { + let p: Vec<&[f32]> = pos.iter().map(|v| v.as_slice()).collect(); + let mut q = mean(&p)?; + let n: Vec<&[f32]> = neg.iter().map(|v| v.as_slice()).collect(); + if let Some(nm) = mean(&n) { + for (a, b) in q.iter_mut().zip(&nm) { + *a -= 0.5 * b; + } + normalize(&mut q); + } + Some(q) +} + +/// Reciprocal rank fusion of ranked lists of IDs: `sum 1/(60 + rank)`. +pub fn rrf(lists: &[Vec<String>]) -> Vec<(String, f32)> { + let mut score: HashMap<&str, f32> = HashMap::new(); + for l in lists { + for (r, id) in l.iter().enumerate() { + *score.entry(id).or_default() += 1.0 / (60.0 + r as f32 + 1.0); + } + } + let mut v: Vec<(String, f32)> = score.into_iter().map(|(k, s)| (k.to_string(), s)).collect(); + v.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap().then(a.0.cmp(&b.0))); + v +} + +// ------------------------------------------------------------------ store + +#[derive(Clone, Serialize, Deserialize)] +pub struct Row { + pub id: String, + /// `Note.hash` when the vector was made. + pub hash: String, + /// Index of the chunk in the note. + pub chunk: usize, + /// 0-based line in the body where the chunk starts. + pub line: usize, +} + +#[derive(Default, Serialize, Deserialize)] +struct Meta { + model: String, + dim: usize, + rows: Vec<Row>, +} + +#[derive(Default)] +pub struct Store { + pub model: String, + pub dim: usize, + pub rows: Vec<Row>, + vecs: Vec<f32>, + bits: Vec<u8>, +} + +/// A hit of the vector ranking. +pub struct VecHit { + pub id: String, + pub score: f32, + /// The best chunk, when ranked by chunk. + pub row: Option<Row>, +} + +pub fn dir(index: &Path) -> PathBuf { + index.join("vec") +} + +impl Store { + pub fn load(index: &Path) -> Store { + let d = dir(index); + let meta: Meta = std::fs::read_to_string(d.join("meta.json")) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let raw = std::fs::read(d.join("f16.bin")).unwrap_or_default(); + let want = meta.rows.len() * meta.dim * 2; + if meta.rows.is_empty() || raw.len() != want { + return Store::default(); + } + let vecs: Vec<f32> = raw + .chunks_exact(2) + .map(|b| f16::from_le_bytes([b[0], b[1]]).to_f32()) + .collect(); + let bits = std::fs::read(d.join("bits.bin")).unwrap_or_default(); + Store { + model: meta.model, + dim: meta.dim, + rows: meta.rows, + vecs, + bits, + } + } + + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub fn save(&self, index: &Path) -> Result<()> { + let d = dir(index); + std::fs::create_dir_all(&d)?; + let write = |name: &str, bytes: &[u8]| -> Result<()> { + let tmp = d.join(format!("{}.tmp", name)); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, d.join(name))?; + Ok(()) + }; + let mut f = Vec::with_capacity(self.vecs.len() * 2); + for x in &self.vecs { + f.extend_from_slice(&f16::from_f32(*x).to_le_bytes()); + } + // The bit file must be consistent with the (rounded) f16 file. + write("f16.bin", &f)?; + write("bits.bin", &self.bits)?; + let meta = Meta { + model: self.model.clone(), + dim: self.dim, + rows: self.rows.clone(), + }; + write("meta.json", &serde_json::to_vec(&meta)?) + } + + fn vec_of(&self, i: usize) -> &[f32] { + &self.vecs[i * self.dim..(i + 1) * self.dim] + } + + fn bits_of(&self, i: usize) -> &[u8] { + let w = self.dim.div_ceil(8); + &self.bits[i * w..(i + 1) * w] + } + + /// Keep the rows for which `keep` is true. + pub fn retain(&mut self, keep: impl Fn(&Row) -> bool) { + let w = self.dim.div_ceil(8); + let mut rows = Vec::new(); + let mut vecs = Vec::new(); + let mut bits = Vec::new(); + for (i, r) in self.rows.iter().enumerate() { + if keep(r) { + rows.push(r.clone()); + vecs.extend_from_slice(self.vec_of(i)); + bits.extend_from_slice(&self.bits[i * w..(i + 1) * w]); + } + } + self.rows = rows; + self.vecs = vecs; + self.bits = bits; + } + + pub fn push(&mut self, row: Row, v: &[f32]) { + if self.rows.is_empty() { + self.dim = v.len(); + } + self.bits.extend_from_slice(&to_bits(v)); + self.vecs.extend_from_slice(v); + self.rows.push(row); + } + + #[cfg(test)] + pub fn has(&self, id: &str, hash: &str) -> bool { + self.rows.iter().any(|r| r.id == id && r.hash == hash) + } + + /// The vector of a whole note: the mean of its chunks. + pub fn note_vec(&self, id: &str) -> Option<Vec<f32>> { + let vs: Vec<&[f32]> = self + .rows + .iter() + .enumerate() + .filter(|(_, r)| r.id == id) + .map(|(i, _)| self.vec_of(i)) + .collect(); + mean(&vs) + } + + /// All note vectors at once. + pub fn note_vecs(&self) -> HashMap<String, Vec<f32>> { + let mut groups: HashMap<&str, Vec<&[f32]>> = HashMap::new(); + for (i, r) in self.rows.iter().enumerate() { + groups.entry(&r.id).or_default().push(self.vec_of(i)); + } + groups + .into_iter() + .filter_map(|(id, vs)| mean(&vs).map(|m| (id.to_string(), m))) + .collect() + } + + /// Rank the notes for the query vector `q`. With `by_chunk` a note + /// scores as its best chunk (for text queries); otherwise as the mean + /// of its chunks (for "like this note"). `allow` filters by ID. + pub fn rank(&self, q: &[f32], by_chunk: bool, allow: &dyn Fn(&str) -> bool) -> Vec<VecHit> { + let mut out: Vec<VecHit> = Vec::new(); + if self.is_empty() || q.len() != self.dim { + return out; + } + if by_chunk { + let mut cand: Vec<usize> = (0..self.rows.len()) + .filter(|&i| allow(&self.rows[i].id)) + .collect(); + if cand.len() > EXACT_LIMIT { + let qb = to_bits(q); + cand.sort_by_key(|&i| hamming(&qb, self.bits_of(i))); + cand.truncate(EXACT_LIMIT); + } + let mut best: HashMap<&str, (f32, usize)> = HashMap::new(); + for i in cand { + let s = dot(q, self.vec_of(i)); + let e = best.entry(&self.rows[i].id).or_insert((f32::MIN, i)); + if s > e.0 { + *e = (s, i); + } + } + for (id, (s, i)) in best { + out.push(VecHit { + id: id.to_string(), + score: s, + row: Some(self.rows[i].clone()), + }); + } + } else { + for (id, v) in self.note_vecs() { + if allow(&id) { + out.push(VecHit { + score: dot(q, &v), + id, + row: None, + }); + } + } + } + out.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap().then(a.id.cmp(&b.id))); + out + } + + /// Rank the tags for a note vector. The anchor of a tag is the mean + /// of the vectors of the other notes that carry it; tags with fewer + /// than two such notes are skipped. + pub fn tag_scores(&self, g: &Graph, id: &str, v: &[f32]) -> Vec<(String, usize, f32, bool)> { + let vecs = self.note_vecs(); + let mut by_tag: HashMap<&str, Vec<&[f32]>> = HashMap::new(); + let mut count: HashMap<&str, usize> = HashMap::new(); + let mut has: HashSet<&str> = HashSet::new(); + for n in g.notes.iter().filter(|n| n.id.is_some()) { + let nid = n.id.as_deref().unwrap(); + for t in &n.tags { + *count.entry(t).or_default() += 1; + if nid == id { + has.insert(t); + } else if let Some(x) = vecs.get(nid) { + by_tag.entry(t).or_default().push(x); + } + } + } + let mut out = Vec::new(); + for (t, vs) in by_tag { + if vs.len() < 2 { + continue; + } + if let Some(a) = mean(&vs) { + out.push((t.to_string(), count[t], dot(v, &a), has.contains(t))); + } + } + out.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap().then(a.0.cmp(&b.0))); + out + } +} + +// ------------------------------------------------------------------ sync + +/// The text of the chunks of a note, formatted for the model. +pub fn note_chunks(n: &Note) -> Vec<chunk::Chunk> { + chunk::chunks(&n.title, &n.tags, n.summary.as_deref(), &n.body) +} + +pub struct SyncReport { + pub embedded: usize, + pub pending: usize, + pub error: Option<String>, +} + +/// Bring the store up to date with the graph. Notes without a vector are +/// embedded; vectors of notes that changed or vanished are dropped. +/// If Ollama fails, the notes done so far are kept and the rest stay +/// pending for the next run. +pub fn sync(index: &Path, g: &Graph, ollama: &Ollama) -> Result<SyncReport> { + let mut store = Store::load(index); + if store.model != ollama.model { + store = Store::default(); + } + let current: HashMap<&str, &str> = g + .notes + .iter() + .filter_map(|n| n.id.as_deref().map(|i| (i, n.hash.as_str()))) + .collect(); + store.retain(|r| current.get(r.id.as_str()) == Some(&r.hash.as_str())); + store.model = ollama.model.clone(); + let have: HashSet<String> = store.rows.iter().map(|r| r.id.clone()).collect(); + let mut seen: HashSet<&str> = HashSet::new(); + let todo: Vec<&Note> = g + .notes + .iter() + .filter(|n| match n.id.as_deref() { + Some(i) => !have.contains(i) && seen.insert(i), + None => false, + }) + .collect(); + let total = todo.len(); + let mut embedded = 0; + let mut error = None; + let mut parsed: HashMap<String, Vec<Note>> = HashMap::new(); + // Work in groups of notes, so a failure never leaves half a note. + let mut group: Vec<(&Note, Vec<chunk::Chunk>)> = Vec::new(); + let mut size = 0; + let flush = |group: &mut Vec<(&Note, Vec<chunk::Chunk>)>, + store: &mut Store| + -> Result<usize> { + if group.is_empty() { + return Ok(0); + } + let texts: Vec<String> = group + .iter() + .flat_map(|(_, cs)| cs.iter().map(|c| ollama.doc_text(&c.text))) + .collect(); + let mut vs = Vec::new(); + for b in texts.chunks(BATCH) { + let t0 = std::time::Instant::now(); + vs.extend(ollama.embed(b)?); + if std::env::var_os("ELEL_DEBUG").is_some() { + eprintln!( + "embed batch of {}: {} chars, max {}, {:.1}s", + b.len(), + b.iter().map(|x| x.len()).sum::<usize>(), + b.iter().map(|x| x.len()).max().unwrap_or(0), + t0.elapsed().as_secs_f32() + ); + } + } + let mut k = 0; + let mut notes = 0; + for (n, cs) in group.iter() { + for (ci, c) in cs.iter().enumerate() { + store.push( + Row { + id: n.id.clone().unwrap(), + hash: n.hash.clone(), + chunk: ci, + line: c.line, + }, + &vs[k], + ); + k += 1; + } + notes += 1; + } + group.clear(); + Ok(notes) + }; + 'work: for n in todo { + // The body is not in the manifest: read it from the file again. + let notes = match parsed.get(&n.path) { + Some(v) => v, + None => { + let content = match std::fs::read_to_string(&n.path) { + Ok(c) => c, + Err(_) => continue, + }; + let v = crate::adapter::parse_file(Path::new(&n.path), &content).unwrap_or_default(); + parsed.entry(n.path.clone()).or_insert(v) + } + }; + let fresh = match notes.iter().find(|x| x.id == n.id && x.hash == n.hash) { + Some(f) => f, + None => continue, + }; + let cs = note_chunks(fresh); + size += cs.len(); + group.push((n, cs)); + if size >= BATCH { + match flush(&mut group, &mut store) { + Ok(k) => embedded += k, + Err(e) => { + error = Some(e.to_string()); + break 'work; + } + } + size = 0; + } + } + if error.is_none() { + match flush(&mut group, &mut store) { + Ok(k) => embedded += k, + Err(e) => error = Some(e.to_string()), + } + } + store.save(index).context("cannot save the vectors")?; + Ok(SyncReport { + embedded, + pending: total - embedded, + error, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn unit(v: &[f32]) -> Vec<f32> { + let mut x = v.to_vec(); + normalize(&mut x); + x + } + + #[test] + fn bits_and_hamming() { + let a = to_bits(&[1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0]); + assert_eq!(a, vec![0b0000_1101, 0b0000_0001]); + let b = to_bits(&[1.0, 1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, -1.0]); + assert_eq!(hamming(&a, &b), 2); + } + + #[test] + fn f16_round_trip_keeps_the_order() { + let dir = std::env::temp_dir().join(format!("elel-vec-{}", std::process::id())); + let mut s = Store::default(); + s.model = "m".into(); + let row = |id: &str| Row { id: id.into(), hash: "h".into(), chunk: 0, line: 0 }; + s.push(row("a"), &unit(&[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.push(row("b"), &unit(&[1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.push(row("c"), &unit(&[0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.save(&dir).unwrap(); + let t = Store::load(&dir); + std::fs::remove_dir_all(&dir).ok(); + assert_eq!(t.rows.len(), 3); + let q = unit(&[1.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]); + let r = t.rank(&q, true, &|_| true); + assert_eq!(r.iter().map(|h| h.id.as_str()).collect::<Vec<_>>(), vec!["a", "b", "c"]); + let r = t.rank(&q, false, &|id| id != "a"); + assert_eq!(r[0].id, "b"); + } + + #[test] + fn rocchio_moves_away_from_the_negative() { + let pos = vec![unit(&[1.0, 1.0, 0.0])]; + let neg = vec![unit(&[0.0, 1.0, 0.0])]; + let q = rocchio(&pos, &neg).unwrap(); + assert!(q[0] > q[1], "{:?}", q); + let plain = rocchio(&pos, &[]).unwrap(); + assert!((plain[0] - plain[1]).abs() < 1e-6); + assert!(rocchio(&[], &neg).is_none()); + } + + #[test] + fn rrf_prefers_notes_high_in_both_lists() { + let l1 = vec!["a".to_string(), "b".into(), "c".into()]; + let l2 = vec!["b".to_string(), "c".into(), "d".into()]; + let r = rrf(&[l1, l2]); + assert_eq!(r[0].0, "b"); + assert_eq!(r.len(), 4); + } + + #[test] + fn retain_drops_stale_rows() { + let mut s = Store::default(); + let row = |id: &str, h: &str| Row { id: id.into(), hash: h.into(), chunk: 0, line: 0 }; + s.push(row("a", "1"), &unit(&[1.0, 0.0])); + s.push(row("b", "1"), &unit(&[0.0, 1.0])); + s.retain(|r| r.id == "a" && r.hash == "1"); + assert_eq!(s.rows.len(), 1); + assert!(s.has("a", "1") && !s.has("b", "1")); + assert_eq!(s.vec_of(0), &[1.0, 0.0]); + } +} |
