aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/adapter/mod.rs275
-rw-r--r--src/adapter/org.rs84
-rw-r--r--src/adapter/txt.rs85
-rw-r--r--src/chunk.rs314
-rw-r--r--src/graph.rs187
-rw-r--r--src/main.rs588
-rw-r--r--src/note.rs36
-rw-r--r--src/query.rs335
-rw-r--r--src/text.rs188
-rw-r--r--src/vec.rs651
10 files changed, 2743 insertions, 0 deletions
diff --git a/src/adapter/mod.rs b/src/adapter/mod.rs
new file mode 100644
index 0000000..8f56b9b
--- /dev/null
+++ b/src/adapter/mod.rs
@@ -0,0 +1,275 @@
+//! Format adapters. Each adapter turns the text of a file into notes.
+
+pub mod org;
+pub mod txt;
+
+use crate::note::{Link, Note};
+use std::path::Path;
+
+/// Parse the file at `path`, whose content is `text`. Returns `None`
+/// when no adapter handles the extension.
+pub fn parse_file(path: &Path, text: &str) -> Option<Vec<Note>> {
+ let p = path.to_string_lossy().to_string();
+ match path.extension().and_then(|e| e.to_str()) {
+ Some("org") => Some(org::parse(&p, text)),
+ Some("txt") => Some(txt::parse(&p, text)),
+ _ => None,
+ }
+}
+
+/// The fields every adapter collects before building a note.
+#[derive(Default)]
+pub struct Fields {
+ pub id: Option<String>,
+ pub title: String,
+ pub date: Option<String>,
+ pub kind: Option<String>,
+ pub tags: Vec<String>,
+ pub parent: Option<String>,
+ pub links: Option<String>,
+ pub context: Option<String>,
+ pub summary: Option<String>,
+}
+
+impl Fields {
+ /// Store the value of a metadata key. Returns false for unknown keys.
+ pub fn set(&mut self, key: &str, value: &str) -> bool {
+ let value = value.trim();
+ let some = |v: &str| {
+ if v.is_empty() {
+ None
+ } else {
+ Some(v.to_string())
+ }
+ };
+ match key.to_ascii_lowercase().as_str() {
+ "id" => self.id = some(value),
+ "title" => self.title = value.to_string(),
+ "date" => self.date = some(value),
+ "kind" => self.kind = some(value),
+ "tags" => {
+ for t in split_list(value) {
+ if !self.tags.contains(&t) {
+ self.tags.push(t);
+ }
+ }
+ }
+ "parent" => self.parent = some(value),
+ "links" => self.links = some(value),
+ "context" => self.context = some(value),
+ "summary" => self.summary = some(value),
+ _ => return false,
+ }
+ true
+ }
+}
+
+/// Assemble a note from its fields, its body, and its place in the file.
+#[allow(clippy::too_many_arguments)]
+pub fn build(
+ path: &str,
+ text: &str,
+ start: usize,
+ end: usize,
+ line: usize,
+ fields: Fields,
+ body: String,
+) -> Note {
+ let mut links = fields
+ .links
+ .as_deref()
+ .map(parse_links_prop)
+ .unwrap_or_default();
+ for l in body_links(&body) {
+ if !links.iter().any(|x| x.dst == l.dst) {
+ links.push(l);
+ }
+ }
+ Note {
+ id: fields.id,
+ path: path.to_string(),
+ start,
+ end,
+ line,
+ hash: blake3::hash(text[start..end].as_bytes()).to_hex().to_string(),
+ title: fields.title,
+ date: fields.date,
+ kind: fields.kind,
+ tags: fields.tags,
+ parent: fields.parent,
+ links,
+ context: fields.context,
+ summary: fields.summary,
+ body,
+ }
+}
+
+/// Split on whitespace, commas and colons.
+pub fn split_list(s: &str) -> Vec<String> {
+ s.split(|c: char| c.is_whitespace() || c == ',' || c == ':')
+ .filter(|t| !t.is_empty())
+ .map(|t| t.to_string())
+ .collect()
+}
+
+fn is_kind(s: &str) -> bool {
+ !s.is_empty()
+ && s
+ .chars()
+ .all(|c| c.is_ascii_alphabetic() || c == '-' || c == '_')
+}
+
+/// Parse the value of a LINKS property: tokens `ID` or `kind:ID`.
+pub fn parse_links_prop(s: &str) -> Vec<Link> {
+ let mut out = Vec::new();
+ for tok in s.split_whitespace() {
+ let link = match tok.split_once(':') {
+ Some(("notes" | "note", rest)) if !rest.is_empty() => Link {
+ dst: rest.to_string(),
+ kind: None,
+ },
+ Some((k, rest)) if is_kind(k) && !rest.is_empty() => Link {
+ dst: rest.to_string(),
+ kind: Some(k.to_string()),
+ },
+ _ => Link {
+ dst: tok.to_string(),
+ kind: None,
+ },
+ };
+ out.push(link);
+ }
+ out
+}
+
+/// Split the path of a `note:` link into its kind and its ID.
+///
+/// The path is `ID` for an untyped link and `:KIND:ID` for a typed one,
+/// so `[[note:ID]]` and `[[note::KIND:ID]]` are the two forms.
+fn split_note_path(p: &str) -> Option<(Option<String>, String)> {
+ if p.is_empty() || p.contains(char::is_whitespace) {
+ return None;
+ }
+ match p.strip_prefix(':') {
+ Some(rest) => {
+ let (k, id) = rest.split_once(':')?;
+ let ok = !k.is_empty()
+ && k.chars()
+ .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
+ (ok && !id.is_empty()).then(|| (Some(k.to_string()), id.to_string()))
+ }
+ None => Some((None, p.to_string())),
+ }
+}
+
+/// Find the `[[note:ID][description]]` links in a body, and their kinds.
+/// The description is for humans (usually the title of the note); only
+/// the ID is used to locate the note.
+pub fn body_links(body: &str) -> Vec<Link> {
+ let mut out: Vec<Link> = Vec::new();
+ let mut pos = 0;
+ while let Some(i) = body[pos..].find("[[note:") {
+ let start = pos + i + "[[note:".len();
+ let rest = &body[start..];
+ match rest.find(']') {
+ Some(j) if !rest[..j].contains('\n') => {
+ let closes = rest[j..].starts_with("]]") || rest[j..].starts_with("][");
+ if closes {
+ if let Some((kind, dst)) = split_note_path(&rest[..j]) {
+ match out.iter_mut().find(|l| l.dst == dst) {
+ // A typed link is more informative than an untyped one.
+ Some(l) => l.kind = l.kind.take().or(kind),
+ None => out.push(Link { dst, kind }),
+ }
+ }
+ }
+ pos = start + j;
+ }
+ _ => pos = start,
+ }
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn body_links_read_the_id_and_ignore_the_description() {
+ let l = body_links(
+ "see [[note:abc-1][A title: with colon]] and [[note:<x.y@z>][d]] or [[note:q]] twice [[note:q][again]]",
+ );
+ let got: Vec<(&str, Option<&str>)> =
+ l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect();
+ assert_eq!(got, vec![("abc-1", None), ("<x.y@z>", None), ("q", None)]);
+ }
+
+ #[test]
+ fn body_links_may_be_typed() {
+ let l = body_links("[[note::supports:a1][A]] [[note::contra:<b@c>]] [[note:d]] [[note:d2][x]]");
+ let got: Vec<(&str, Option<&str>)> =
+ l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect();
+ assert_eq!(
+ got,
+ vec![("a1", Some("supports")), ("<b@c>", Some("contra")), ("d", None), ("d2", None)]
+ );
+ // The typed form wins when the same note is linked twice.
+ let l = body_links("[[note:a]] and [[note::cites:a]]");
+ assert_eq!(l.len(), 1);
+ assert_eq!(l[0].kind.as_deref(), Some("cites"));
+ }
+
+ #[test]
+ fn body_links_ignore_other_links() {
+ let body = "[[https://a.b][web]] [[file:x.org]] [[*heading]] [[#custom]] [[./rel][r]] \\
+ [[two words]] [[n1][fuzzy]] [[note:]] [[note:two words]] [[note::bad]] \\
+ [[notes:x]] [[note:a\nb]]";
+ assert!(body_links(body).is_empty(), "{:?}", body_links(body));
+ }
+
+ #[test]
+ fn links_property_has_typed_and_plain_tokens() {
+ let l = parse_links_prop("a1 supports:b2 notes:c3 <d@e>");
+ let got: Vec<(&str, Option<&str>)> =
+ l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect();
+ assert_eq!(
+ got,
+ vec![("a1", None), ("b2", Some("supports")), ("c3", None), ("<d@e>", None)]
+ );
+ }
+
+ #[test]
+ fn org_notes_are_top_level_headings_only() {
+ let text = "intro\n* One :a:b:\n:PROPERTIES:\n:ID: 1\n:PARENT: 0\n:END:\nbody\n** sub\nmore\n* Two\ntext\n";
+ let notes = org::parse("/x.org", text);
+ assert_eq!(notes.len(), 2);
+ assert_eq!(notes[0].id.as_deref(), Some("1"));
+ assert_eq!(notes[0].title, "One");
+ assert_eq!(notes[0].tags, vec!["a", "b"]);
+ assert_eq!(notes[0].parent.as_deref(), Some("0"));
+ assert!(notes[0].body.contains("** sub"));
+ assert_eq!(notes[0].line, 2);
+ assert_eq!(&text[notes[0].start..notes[0].end].lines().next(), &Some("* One :a:b:"));
+ assert_eq!(notes[1].id, None);
+ }
+
+ #[test]
+ fn txt_notes_are_split_at_form_feeds() {
+ let text = "ID: 1\nTitle: A\nTags: x y\n\nbody a\n\x0c\nID: 2\nTitle: B\n\nbody b\n\x0c\n\n";
+ let notes = txt::parse("/x.txt", text);
+ assert_eq!(notes.len(), 2);
+ assert_eq!(notes[0].tags, vec!["x", "y"]);
+ assert_eq!(notes[0].body, "body a");
+ assert_eq!(notes[1].id.as_deref(), Some("2"));
+ assert_eq!(notes[1].line, 7);
+ }
+
+ #[test]
+ fn txt_note_without_header_takes_its_title_from_the_text() {
+ let notes = txt::parse("/x.txt", "just a thought\nmore\n");
+ assert_eq!(notes.len(), 1);
+ assert_eq!(notes[0].title, "just a thought");
+ assert_eq!(notes[0].id, None);
+ }
+}
diff --git a/src/adapter/org.rs b/src/adapter/org.rs
new file mode 100644
index 0000000..a999814
--- /dev/null
+++ b/src/adapter/org.rs
@@ -0,0 +1,84 @@
+//! Org adapter: each top-level heading is a note. Metadata lives in
+//! the heading (title, tags) and in its property drawer.
+
+use super::{Fields, build};
+use crate::note::Note;
+
+fn is_tag_string(s: &str) -> bool {
+ s.len() > 2
+ && s.starts_with(':')
+ && s.ends_with(':')
+ && s.chars()
+ .all(|c| c.is_alphanumeric() || "_@#%:.".contains(c))
+}
+
+/// Split a heading line into its title and its tags.
+fn parse_heading(line: &str) -> (String, Vec<String>) {
+ let h = line.trim_start_matches('*').trim();
+ if let Some(idx) = h.rfind(char::is_whitespace) {
+ let last = &h[idx + 1..];
+ if is_tag_string(last) {
+ let tags = last
+ .split(':')
+ .filter(|t| !t.is_empty())
+ .map(|t| t.to_string())
+ .collect();
+ return (h[..idx].trim().to_string(), tags);
+ }
+ }
+ (h.to_string(), Vec::new())
+}
+
+fn is_planning(line: &str) -> bool {
+ let t = line.trim_start();
+ t.starts_with("SCHEDULED:") || t.starts_with("DEADLINE:") || t.starts_with("CLOSED:")
+}
+
+pub fn parse(path: &str, text: &str) -> Vec<Note> {
+ // Find the top-level headings: a single star at column 0.
+ let mut heads: Vec<(usize, usize)> = Vec::new();
+ let mut off = 0;
+ for (ln, line) in text.split_inclusive('\n').enumerate() {
+ let t = line.trim_end();
+ if line.starts_with("* ") || t == "*" {
+ heads.push((off, ln + 1));
+ }
+ off += line.len();
+ }
+ let mut notes = Vec::new();
+ for (i, &(start, line_no)) in heads.iter().enumerate() {
+ let end = heads.get(i + 1).map(|h| h.0).unwrap_or(text.len());
+ let chunk = &text[start..end];
+ let mut lines = chunk.lines();
+ let heading = lines.next().unwrap_or("");
+ let (title, tags) = parse_heading(heading);
+ let mut fields = Fields {
+ title,
+ tags,
+ ..Default::default()
+ };
+ let mut rest: Vec<&str> = lines.collect();
+ // Skip planning lines, then read the property drawer, if any.
+ let mut k = 0;
+ while k < rest.len() && is_planning(rest[k]) {
+ k += 1;
+ }
+ let mut body_from = 0;
+ if k < rest.len() && rest[k].trim() == ":PROPERTIES:" {
+ let mut j = k + 1;
+ while j < rest.len() && rest[j].trim() != ":END:" {
+ let l = rest[j].trim();
+ if let Some(r) = l.strip_prefix(':') {
+ if let Some((key, val)) = r.split_once(':') {
+ fields.set(key, val);
+ }
+ }
+ j += 1;
+ }
+ body_from = (j + 1).min(rest.len());
+ }
+ let body = rest.split_off(body_from).join("\n");
+ notes.push(build(path, text, start, end, line_no, fields, body));
+ }
+ notes
+}
diff --git a/src/adapter/txt.rs b/src/adapter/txt.rs
new file mode 100644
index 0000000..ddea799
--- /dev/null
+++ b/src/adapter/txt.rs
@@ -0,0 +1,85 @@
+//! Plain text adapter. Notes are separated by a form-feed line. A
+//! note starts with `Key: value` lines, then a blank line, then the
+//! body.
+
+use super::{Fields, build};
+use crate::note::Note;
+
+const KEYS: [&str; 9] = [
+ "id", "title", "date", "kind", "tags", "parent", "links", "context", "summary",
+];
+
+/// Parse `Key: value` if the key is a known one.
+fn header_line(line: &str) -> Option<(&str, &str)> {
+ let (k, v) = line.split_once(':')?;
+ let lk = k.trim().to_ascii_lowercase();
+ if KEYS.contains(&lk.as_str()) && k.trim().len() == k.len() {
+ Some((k, v))
+ } else {
+ None
+ }
+}
+
+pub fn parse(path: &str, text: &str) -> Vec<Note> {
+ // Split into chunks at the form-feed lines.
+ let mut chunks: Vec<(usize, usize, usize)> = Vec::new(); // start, end, line
+ let mut off = 0;
+ let mut cur_start = 0;
+ let mut cur_line = 1;
+ for (ln, line) in text.split_inclusive('\n').enumerate() {
+ if line.trim_end_matches(['\r', '\n']) == "\x0c" {
+ chunks.push((cur_start, off, cur_line));
+ cur_start = off + line.len();
+ cur_line = ln + 2;
+ }
+ off += line.len();
+ }
+ chunks.push((cur_start, text.len(), cur_line));
+
+ let mut notes = Vec::new();
+ for (start, end, line_no) in chunks {
+ let chunk = &text[start..end];
+ if chunk.trim().is_empty() {
+ continue;
+ }
+ let mut fields = Fields::default();
+ let mut body_lines: Vec<&str> = Vec::new();
+ let mut in_header = true;
+ let mut seen = 0;
+ for l in chunk.lines() {
+ if in_header {
+ if seen == 0 && l.trim().is_empty() {
+ continue;
+ }
+ if let Some((k, v)) = header_line(l) {
+ fields.set(k, v);
+ seen += 1;
+ continue;
+ }
+ in_header = false;
+ // The blank line that ends the header is not part of the body.
+ if seen > 0 && l.trim().is_empty() {
+ continue;
+ }
+ }
+ body_lines.push(l);
+ }
+ if fields.title.is_empty() {
+ fields.title = body_lines
+ .iter()
+ .find(|l| !l.trim().is_empty())
+ .map(|l| l.trim().chars().take(80).collect())
+ .unwrap_or_default();
+ }
+ notes.push(build(
+ path,
+ text,
+ start,
+ end,
+ line_no,
+ fields,
+ body_lines.join("\n"),
+ ));
+ }
+ notes
+}
diff --git a/src/chunk.rs b/src/chunk.rs
new file mode 100644
index 0000000..b17316b
--- /dev/null
+++ b/src/chunk.rs
@@ -0,0 +1,314 @@
+//! Cut a note into chunks for the embeddings.
+//!
+//! A chunk is a paragraph or a few paragraphs. Blocks that must stay
+//! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A
+//! paragraph that is only a formula is merged into a neighbour, because
+//! alone it would be embedded as "some mathematics" and look like every
+//! other formula. Each chunk is prefixed with the title, so a bare
+//! "this follows from the lemma" still knows what it is about.
+
+/// Paragraphs shorter than this are merged with a neighbour.
+const MIN: usize = 200;
+/// A chunk is not grown beyond this (a single paragraph may exceed it).
+const MAX: usize = 1500;
+/// A paragraph with fewer words than this, once the math is removed,
+/// is a formula paragraph.
+const MIN_WORDS: usize = 5;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Chunk {
+ pub text: String,
+ /// 0-based line of the body where the chunk starts.
+ pub line: usize,
+}
+
+/// A paragraph of the body: its text and its first line.
+struct Para {
+ text: String,
+ line: usize,
+}
+
+/// Replace `[[note:ID][desc]]` by `desc` and org links by their text.
+pub fn strip_links(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(i) = rest.find("[[") {
+ out.push_str(&rest[..i]);
+ let tail = &rest[i + 2..];
+ match tail.find("]]") {
+ Some(j) => {
+ let inner = &tail[..j];
+ match inner.split_once("][") {
+ Some((_, desc)) => out.push_str(desc),
+ None => out.push_str(inner),
+ }
+ rest = &tail[j + 2..];
+ }
+ None => {
+ out.push_str(&rest[i..]);
+ rest = "";
+ }
+ }
+ }
+ out.push_str(rest);
+ out
+}
+
+/// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed.
+fn without_math(s: &str) -> String {
+ let mut out = String::new();
+ let mut rest = s;
+ 'outer: while !rest.is_empty() {
+ for (open, close) in [
+ ("$$", "$$"),
+ ("\\[", "\\]"),
+ ("\\(", "\\)"),
+ ("$", "$"),
+ ] {
+ if let Some(r) = rest.strip_prefix(open) {
+ match r.find(close) {
+ Some(j) => rest = &r[j + close.len()..],
+ None => rest = "",
+ }
+ out.push(' ');
+ continue 'outer;
+ }
+ }
+ if rest.starts_with("\\begin{") {
+ match rest.find("\\end{") {
+ Some(j) => {
+ let after = &rest[j..];
+ rest = match after.find('}') {
+ Some(k) => &after[k + 1..],
+ None => "",
+ };
+ out.push(' ');
+ continue 'outer;
+ }
+ None => break,
+ }
+ }
+ let c = rest.chars().next().unwrap();
+ out.push(c);
+ rest = &rest[c.len_utf8()..];
+ }
+ out
+}
+
+/// Does the paragraph hold almost nothing but math?
+fn is_formula(p: &str) -> bool {
+ let t = p.trim();
+ if t.is_empty() {
+ return false;
+ }
+ let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{");
+ if !has_math {
+ return false;
+ }
+ let left = without_math(t);
+ // CJK has no spaces: count its characters as words.
+ let words = left
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|w| !w.is_empty())
+ .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 })
+ .sum::<usize>();
+ words < MIN_WORDS
+}
+
+/// Split into paragraphs, keeping blocks whole.
+fn paragraphs(body: &str) -> Vec<Para> {
+ let mut out: Vec<Para> = Vec::new();
+ let mut cur: Vec<&str> = Vec::new();
+ let mut start = 0;
+ // Inside a block that ends with this line (lowercased, trimmed prefix).
+ let mut block_end: Option<String> = None;
+ let mut dollars = false;
+ let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec<Para>| {
+ if !cur.is_empty() {
+ out.push(Para { text: cur.join("\n"), line: start });
+ cur.clear();
+ }
+ };
+ for (i, l) in body.lines().enumerate() {
+ let t = l.trim();
+ let tl = t.to_lowercase();
+ if let Some(end) = &block_end {
+ cur.push(l);
+ if tl.starts_with(end.as_str()) {
+ block_end = None;
+ }
+ continue;
+ }
+ if dollars {
+ cur.push(l);
+ if t.contains("$$") {
+ dollars = false;
+ }
+ continue;
+ }
+ if t.is_empty() {
+ flush(&mut cur, start, &mut out);
+ continue;
+ }
+ let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' ');
+ if heading {
+ flush(&mut cur, start, &mut out);
+ }
+ if cur.is_empty() {
+ start = i;
+ }
+ if let Some(rest) = tl.strip_prefix("#+begin_") {
+ let name = rest.split_whitespace().next().unwrap_or("");
+ block_end = Some(format!("#+end_{}", name));
+ } else if let Some(rest) = t.strip_prefix("\\begin{") {
+ let name = rest.split('}').next().unwrap_or("");
+ block_end = Some(format!("\\end{{{}}}", name.to_lowercase()));
+ } else if t.matches("$$").count() % 2 == 1 {
+ dollars = true;
+ }
+ // A one-line block (\begin{x} .. \end{x}) closes at once.
+ if let Some(end) = &block_end {
+ if tl[1..].contains(end.as_str()) {
+ block_end = None;
+ }
+ }
+ cur.push(l);
+ }
+ flush(&mut cur, start, &mut out);
+ out
+}
+
+/// The chunks of a note. A note without a body gets one chunk from its
+/// title, tags and summary.
+pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec<Chunk> {
+ let body = strip_links(body);
+ let paras = paragraphs(&body);
+ // Merge formula paragraphs into the previous paragraph (or the next one).
+ let mut merged: Vec<Para> = Vec::new();
+ let mut carry: Option<Para> = None;
+ for p in paras {
+ let formula = is_formula(&p.text);
+ let p = match carry.take() {
+ Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line },
+ None => p,
+ };
+ if formula && merged.is_empty() {
+ // Opens the note: it joins the next paragraph.
+ carry = Some(p);
+ } else if formula {
+ let last = merged.last_mut().unwrap();
+ last.text = format!("{}\n{}", last.text, p.text);
+ } else {
+ merged.push(p);
+ }
+ }
+ if let Some(c) = carry {
+ merged.push(c);
+ }
+ // Merge small paragraphs, up to MAX.
+ let mut out: Vec<Chunk> = Vec::new();
+ let mut small = false;
+ for p in merged {
+ match out.last_mut() {
+ Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => {
+ last.text.push_str("\n\n");
+ last.text.push_str(&p.text);
+ small = last.text.len() < MIN;
+ }
+ _ => {
+ small = p.text.len() < MIN;
+ out.push(Chunk { text: p.text, line: p.line });
+ }
+ }
+ }
+ if out.is_empty() {
+ let mut t = String::new();
+ if let Some(s) = summary {
+ t.push_str(s);
+ }
+ if !tags.is_empty() {
+ t.push_str("\n");
+ t.push_str(&tags.join(" "));
+ }
+ out.push(Chunk { text: t, line: 0 });
+ }
+ for c in &mut out {
+ c.text = format!("{}\n\n{}", title, c.text.trim());
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn long(s: &str) -> String {
+ format!("{} {}", s, "word ".repeat(60))
+ }
+
+ #[test]
+ fn formula_paragraph_joins_the_previous_one() {
+ let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.len(), 2, "{:?}", c);
+ assert!(c[0].text.contains("x^2"));
+ assert!(!c[1].text.contains("x^2"));
+ }
+
+ #[test]
+ fn formula_at_the_start_joins_the_next() {
+ let body = format!("$$a=b$$\n\n{}", long("Prose"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose"));
+ }
+
+ #[test]
+ fn prose_with_inline_math_is_not_a_formula() {
+ assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent."));
+ assert!(is_formula("\\[ \\int_0^1 f \\]"));
+ assert!(is_formula("$a$ and $b$"));
+ }
+
+ #[test]
+ fn source_blocks_stay_whole() {
+ let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two"));
+ let p = paragraphs(&body);
+ assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::<Vec<_>>());
+ assert!(p[1].text.contains("(b)"));
+ }
+
+ #[test]
+ fn environment_stays_whole() {
+ let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore";
+ let p = paragraphs(body);
+ assert_eq!(p.len(), 3);
+ assert!(p[1].text.contains("\\end{align}"));
+ }
+
+ #[test]
+ fn small_paragraphs_are_merged() {
+ let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third");
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.starts_with("T\n\n"));
+ }
+
+ #[test]
+ fn links_become_their_description() {
+ assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2");
+ }
+
+ #[test]
+ fn empty_body_uses_title_and_tags() {
+ let c = chunks("Title", &["x".into()], Some("sum"), "");
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.contains("sum") && c[0].text.contains("x"));
+ }
+
+ #[test]
+ fn line_numbers_point_at_the_paragraph() {
+ let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.iter().map(|x| x.line).collect::<Vec<_>>(), vec![0, 2, 4]);
+ }
+}
diff --git a/src/graph.rs b/src/graph.rs
new file mode 100644
index 0000000..460bdec
--- /dev/null
+++ b/src/graph.rs
@@ -0,0 +1,187 @@
+//! The manifest (what was indexed) and the graph built from it.
+//!
+//! The manifest is a small JSON file. The graph is rebuilt in memory
+//! on every call; with thousands of notes this takes microseconds.
+
+use crate::note::Note;
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::path::Path;
+
+#[derive(Serialize, Deserialize)]
+pub struct FileEntry {
+ pub hash: String,
+ pub notes: Vec<Note>,
+}
+
+#[derive(Default, Serialize, Deserialize)]
+pub struct Manifest {
+ pub files: BTreeMap<String, FileEntry>,
+}
+
+impl Manifest {
+ pub fn load(path: &Path) -> Manifest {
+ std::fs::read_to_string(path)
+ .ok()
+ .and_then(|s| serde_json::from_str(&s).ok())
+ .unwrap_or_default()
+ }
+
+ /// Write through a temporary file so a crash cannot leave half a file.
+ pub fn save(&self, path: &Path) -> Result<()> {
+ let tmp = path.with_extension("json.tmp");
+ std::fs::write(&tmp, serde_json::to_vec(self)?)?;
+ std::fs::rename(&tmp, path)?;
+ Ok(())
+ }
+}
+
+/// How a note is reached from the note we started at.
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum Rel {
+ Parent,
+ Child,
+ Link,
+ Backlink,
+}
+
+impl Rel {
+ pub fn name(self) -> &'static str {
+ match self {
+ Rel::Parent => "parent",
+ Rel::Child => "child",
+ Rel::Link => "link",
+ Rel::Backlink => "backlink",
+ }
+ }
+}
+
+pub struct Graph {
+ pub notes: Vec<Note>,
+ by_id: HashMap<String, usize>,
+ children: HashMap<String, Vec<usize>>,
+ backlinks: HashMap<String, Vec<usize>>,
+ pub duplicates: Vec<(String, Vec<usize>)>,
+}
+
+impl Graph {
+ pub fn build(m: &Manifest) -> Graph {
+ let notes: Vec<Note> = m.files.values().flat_map(|f| f.notes.clone()).collect();
+ let mut by_id: HashMap<String, usize> = HashMap::new();
+ let mut seen: HashMap<String, Vec<usize>> = HashMap::new();
+ for (i, n) in notes.iter().enumerate() {
+ if let Some(id) = &n.id {
+ seen.entry(id.clone()).or_default().push(i);
+ by_id.entry(id.clone()).or_insert(i);
+ }
+ }
+ let mut duplicates: Vec<(String, Vec<usize>)> =
+ seen.into_iter().filter(|(_, v)| v.len() > 1).collect();
+ duplicates.sort();
+ let mut children: HashMap<String, Vec<usize>> = HashMap::new();
+ let mut backlinks: HashMap<String, Vec<usize>> = HashMap::new();
+ for (i, n) in notes.iter().enumerate() {
+ if n.id.is_none() {
+ continue;
+ }
+ if let Some(p) = &n.parent {
+ children.entry(p.clone()).or_default().push(i);
+ }
+ for l in &n.links {
+ backlinks.entry(l.dst.clone()).or_default().push(i);
+ }
+ }
+ Graph {
+ notes,
+ by_id,
+ children,
+ backlinks,
+ duplicates,
+ }
+ }
+
+ pub fn get(&self, id: &str) -> Option<&Note> {
+ self.by_id.get(id).map(|&i| &self.notes[i])
+ }
+
+ pub fn parent_of(&self, id: &str) -> Option<&Note> {
+ self.get(id)
+ .and_then(|n| n.parent.as_deref())
+ .and_then(|p| self.get(p))
+ }
+
+ pub fn children_of(&self, id: &str) -> Vec<&Note> {
+ self.idx(self.children.get(id))
+ }
+
+ pub fn backlinks_of(&self, id: &str) -> Vec<&Note> {
+ self.idx(self.backlinks.get(id))
+ }
+
+ pub fn outlinks_of(&self, id: &str) -> Vec<&Note> {
+ match self.get(id) {
+ Some(n) => n.links.iter().filter_map(|l| self.get(&l.dst)).collect(),
+ None => Vec::new(),
+ }
+ }
+
+ fn idx(&self, v: Option<&Vec<usize>>) -> Vec<&Note> {
+ match v {
+ Some(v) => v.iter().map(|&i| &self.notes[i]).collect(),
+ None => Vec::new(),
+ }
+ }
+
+ /// Does the note have a link or a backlink or a parent or a child?
+ pub fn is_connected(&self, n: &Note) -> bool {
+ let id = match &n.id {
+ Some(id) => id,
+ None => return false,
+ };
+ n.parent.is_some()
+ || !n.links.is_empty()
+ || self.children.contains_key(id)
+ || self.backlinks.contains_key(id)
+ }
+
+ /// Breadth-first search over all relations, ignoring direction.
+ /// Returns (note, distance, relation at the first step).
+ pub fn reach(&self, id: &str, hops: usize) -> Vec<(&Note, usize, Rel)> {
+ let mut out = Vec::new();
+ let mut seen: HashSet<&str> = HashSet::new();
+ seen.insert(id);
+ let mut queue: VecDeque<(&str, usize, Option<Rel>)> = VecDeque::new();
+ queue.push_back((id, 0, None));
+ while let Some((cur, d, first)) = queue.pop_front() {
+ if d == hops {
+ continue;
+ }
+ let mut next: Vec<(&Note, Rel)> = Vec::new();
+ if let Some(p) = self.parent_of(cur) {
+ next.push((p, Rel::Parent));
+ }
+ for c in self.children_of(cur) {
+ next.push((c, Rel::Child));
+ }
+ for l in self.outlinks_of(cur) {
+ next.push((l, Rel::Link));
+ }
+ for b in self.backlinks_of(cur) {
+ next.push((b, Rel::Backlink));
+ }
+ for (n, rel) in next {
+ let nid = match n.id.as_deref() {
+ Some(x) => x,
+ None => continue,
+ };
+ if seen.insert(nid) {
+ let r = first.unwrap_or(rel);
+ out.push((n, d + 1, r));
+ queue.push_back((nid, d + 1, Some(r)));
+ }
+ }
+ }
+ out
+ }
+}
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 0000000..9824351
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,588 @@
+//! elel: Emacs Lexical Embedding Link. A notes engine.
+//!
+//! This program only reads the note files. It never writes to them.
+//! Everything it stores lives in the index directory and can be rebuilt.
+
+mod adapter;
+mod chunk;
+mod graph;
+mod note;
+mod query;
+mod text;
+mod vec;
+
+use anyhow::{Context, Result, bail};
+use clap::{Parser, Subcommand};
+use graph::{FileEntry, Graph, Manifest};
+use note::Note;
+use query::Query;
+use serde::Serialize;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::path::{Path, PathBuf};
+use text::Text;
+use vec::{Ollama, Store};
+
+#[derive(Parser)]
+#[command(version, about = "Search and link notes kept in plain files")]
+struct Cli {
+ /// Directory that holds the note files.
+ #[arg(long, env = "ELEL_ROOT", global = true)]
+ root: Option<PathBuf>,
+ /// Directory for the derived index.
+ #[arg(long, env = "ELEL_INDEX", global = true)]
+ index: Option<PathBuf>,
+ #[command(subcommand)]
+ cmd: Cmd,
+}
+
+#[derive(Subcommand)]
+enum Cmd {
+ /// Scan the note files and update the index.
+ Index {
+ /// Re-read every file, even the unchanged ones.
+ #[arg(long)]
+ force: bool,
+ /// Do not compute embeddings (no Ollama needed).
+ #[arg(long)]
+ no_embed: bool,
+ },
+ /// Search notes. The query may hold filters such as `tag:X`,
+ /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README);
+ /// the other words go to the full text index. An empty query lists
+ /// every note.
+ Search {
+ query: Option<String>,
+ #[arg(short, default_value_t = 50)]
+ n: usize,
+ /// Also rank by meaning: fuse the text search with the embeddings.
+ #[arg(long)]
+ hybrid: bool,
+ },
+ /// Print how many notes each query matches, one line per query.
+ Count { queries: Vec<String> },
+ /// Show the neighbourhood of a note.
+ Hood {
+ id: String,
+ #[arg(long, default_value_t = 1)]
+ hops: usize,
+ },
+ /// Print a note with its body, read fresh from its file.
+ Get { id: String },
+ /// Report notes without IDs, duplicate IDs and dangling links.
+ Check,
+ /// List the tags with the number of notes that carry each.
+ Tags {
+ /// Rank the tags by how close their notes are to this note.
+ #[arg(long)]
+ like: Option<String>,
+ },
+}
+
+struct Ctx {
+ root: PathBuf,
+ index: PathBuf,
+}
+
+impl Ctx {
+ fn new(cli: &Cli) -> Result<Ctx> {
+ let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?);
+ let root = cli.root.clone().unwrap_or_else(|| home.join("notes"));
+ let index = cli
+ .index
+ .clone()
+ .unwrap_or_else(|| home.join(".local/share/elel"));
+ Ok(Ctx { root, index })
+ }
+
+ fn manifest_path(&self) -> PathBuf {
+ self.index.join("graph.json")
+ }
+
+ fn tantivy_dir(&self) -> PathBuf {
+ self.index.join("tantivy")
+ }
+}
+
+/// A note as shown to the front end.
+#[derive(Serialize)]
+struct View<'a> {
+ #[serde(flatten)]
+ note: &'a Note,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ score: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ snippet: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ dist: Option<usize>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ rel: Option<&'static str>,
+ /// For hits by meaning: the line of the body where the best chunk starts.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ chunk_line: Option<usize>,
+ /// Number of notes that link here.
+ backlinks: usize,
+}
+
+fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> {
+ View {
+ note: n,
+ score: None,
+ snippet: None,
+ dist: None,
+ rel: None,
+ chunk_line: None,
+ backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0),
+ }
+}
+
+fn print<T: Serialize>(v: &T) -> Result<()> {
+ println!("{}", serde_json::to_string(v)?);
+ Ok(())
+}
+
+fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> {
+ let mut entries: Vec<_> = std::fs::read_dir(dir)
+ .with_context(|| format!("cannot read {}", dir.display()))?
+ .filter_map(|e| e.ok())
+ .collect();
+ entries.sort_by_key(|e| e.file_name());
+ for e in entries {
+ let p = e.path();
+ let name = e.file_name().to_string_lossy().to_string();
+ if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') {
+ continue;
+ }
+ if p.is_dir() {
+ collect_files(&p, out)?;
+ } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) {
+ out.push(p);
+ }
+ }
+ Ok(())
+}
+
+fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> {
+ std::fs::create_dir_all(&ctx.index)?;
+ let text = Text::open(&ctx.tantivy_dir())?;
+ let old = if text.fresh || force {
+ Manifest::default()
+ } else {
+ Manifest::load(&ctx.manifest_path())
+ };
+ let root = ctx
+ .root
+ .canonicalize()
+ .with_context(|| format!("no notes directory at {}", ctx.root.display()))?;
+ let mut files = Vec::new();
+ collect_files(&root, &mut files)?;
+
+ let mut new = Manifest::default();
+ let mut changed: Vec<(String, Vec<Note>)> = Vec::new();
+ for f in &files {
+ let key = f.to_string_lossy().to_string();
+ let content = match std::fs::read_to_string(f) {
+ Ok(c) => c,
+ Err(e) => {
+ eprintln!("skipping {}: {}", key, e);
+ continue;
+ }
+ };
+ let hash = blake3::hash(content.as_bytes()).to_hex().to_string();
+ match old.files.get(&key) {
+ Some(e) if e.hash == hash => {
+ new.files.insert(
+ key,
+ FileEntry {
+ hash,
+ notes: e.notes.clone(),
+ },
+ );
+ }
+ _ => {
+ let notes = adapter::parse_file(f, &content).unwrap_or_default();
+ changed.push((key.clone(), notes.clone()));
+ new.files.insert(key, FileEntry { hash, notes });
+ }
+ }
+ }
+ let removed: Vec<String> = old
+ .files
+ .keys()
+ .filter(|k| !new.files.contains_key(*k))
+ .cloned()
+ .collect();
+ text.update(&removed, &changed)?;
+ new.save(&ctx.manifest_path())?;
+ let g = Graph::build(&new);
+ let mut report = serde_json::json!({
+ "files": new.files.len(),
+ "changed": changed.len(),
+ "removed": removed.len(),
+ "notes": g.notes.len(),
+ });
+ if !no_embed {
+ let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?;
+ report["embedded"] = r.embedded.into();
+ report["pending"] = r.pending.into();
+ if let Some(e) = r.error {
+ eprintln!("elel: embeddings stopped: {}", e);
+ report["embed_error"] = e.into();
+ }
+ }
+ print(&report)
+}
+
+fn load_graph(ctx: &Ctx) -> Result<Graph> {
+ let p = ctx.manifest_path();
+ if !p.exists() {
+ bail!("no index yet; run `elel index` first");
+ }
+ Ok(Graph::build(&Manifest::load(&p)))
+}
+
+/// One answer of a query: the note, and for text queries the score
+/// and the snippet.
+struct Hit<'a> {
+ note: &'a Note,
+ score: Option<f32>,
+ snippet: Option<String>,
+ /// The best chunk, for hits by meaning.
+ row: Option<vec::Row>,
+}
+
+/// The words of a text query that can be embedded: no `-x`, `OR`, or
+/// field prefixes, and no quotes.
+fn plain_words(words: &str) -> String {
+ words
+ .split_whitespace()
+ .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND")
+ .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"'))
+ .filter(|w| !w.is_empty())
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+/// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words).
+fn semantic(
+ ctx: &Ctx,
+ g: &Graph,
+ q: &Query,
+ m: &query::Matcher,
+ hybrid: bool,
+) -> Result<Vec<vec::VecHit>> {
+ let store = Store::load(&ctx.index);
+ if store.is_empty() {
+ bail!("no embeddings yet; start Ollama and run `elel index`");
+ }
+ let ollama = Ollama::from_env();
+ if store.model != ollama.model {
+ bail!(
+ "the embeddings were made with {}, but the model is {}; run `elel index`",
+ store.model,
+ ollama.model
+ );
+ }
+ let mut by_chunk = false;
+ let mut exclude: HashSet<String> = HashSet::new();
+ let mut resolve = |s: &str| -> Result<Vec<f32>> {
+ if g.get(s).is_some() {
+ exclude.insert(s.to_string());
+ store
+ .note_vec(s)
+ .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s))
+ } else {
+ by_chunk = true;
+ ollama.embed_query(s)
+ }
+ };
+ let mut pos = Vec::new();
+ for s in &q.like {
+ pos.push(resolve(s)?);
+ }
+ let mut neg = Vec::new();
+ for s in &q.unlike {
+ neg.push(resolve(s)?);
+ }
+ if pos.is_empty() {
+ if !neg.is_empty() {
+ bail!("-like: needs a like: to go with");
+ }
+ by_chunk = true;
+ pos.push(ollama.embed_query(&plain_words(&q.words))?);
+ }
+ let qv = vec::rocchio(&pos, &neg).unwrap();
+ let _ = hybrid;
+ let allow = |id: &str| match g.get(id) {
+ Some(n) => !exclude.contains(id) && m.pass(n),
+ None => false,
+ };
+ Ok(store.rank(&qv, by_chunk, &allow))
+}
+
+fn run_query<'a>(
+ ctx: &Ctx,
+ g: &'a Graph,
+ query: &str,
+ n: usize,
+ hybrid: bool,
+) -> Result<Vec<Hit<'a>>> {
+ let q = Query::parse(query)?;
+ let root = ctx
+ .root
+ .canonicalize()
+ .unwrap_or_else(|_| ctx.root.clone())
+ .to_string_lossy()
+ .to_string();
+ let m = q.matcher(g, &root);
+ let has_words = !q.words.trim().is_empty();
+ let has_like = !q.like.is_empty() || !q.unlike.is_empty();
+ let mut out = Vec::new();
+
+ // The ranking by meaning, if the query asks for it.
+ let mut by_meaning: Option<Vec<vec::VecHit>> = None;
+ if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) {
+ match semantic(ctx, g, &q, &m, hybrid) {
+ Ok(v) => by_meaning = Some(v),
+ Err(e) if has_like => return Err(e),
+ Err(e) => eprintln!("elel: {}; using the text search only", e),
+ }
+ }
+
+ // The ranking by words.
+ let mut lexical: Vec<(&'a Note, f32, Option<String>)> = Vec::new();
+ if has_words {
+ let text = Text::open(&ctx.tantivy_dir())?;
+ // Filters apply after the text search, so ask for plenty.
+ for f in text.search(&q.words, 100_000)? {
+ let note = match g.get(&f.id) {
+ Some(note) => note,
+ None => continue,
+ };
+ if m.pass(note) {
+ lexical.push((note, f.score, f.snippet));
+ }
+ }
+ }
+
+ match by_meaning {
+ None if has_words => {
+ for (note, score, snippet) in lexical.into_iter().take(n) {
+ out.push(Hit { note, score: Some(score), snippet, row: None });
+ }
+ }
+ None => {
+ let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect();
+ // Newest first; notes without a date go last.
+ all.sort_by(|a, b| b.date.cmp(&a.date));
+ for note in all.into_iter().take(n) {
+ out.push(Hit { note, score: None, snippet: None, row: None });
+ }
+ }
+ Some(v) if !has_words => {
+ for h in v.into_iter().take(n) {
+ if let Some(note) = g.get(&h.id) {
+ out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row });
+ }
+ }
+ }
+ Some(v) => {
+ let lists = vec![
+ lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::<Vec<_>>(),
+ v.iter().map(|h| h.id.clone()).collect::<Vec<_>>(),
+ ];
+ let mut rows: HashMap<String, vec::Row> = HashMap::new();
+ for h in v {
+ if let Some(r) = h.row {
+ rows.insert(h.id, r);
+ }
+ }
+ let mut snippets: HashMap<String, Option<String>> = HashMap::new();
+ for (note, _, s) in lexical {
+ if let Some(id) = &note.id {
+ snippets.insert(id.clone(), s);
+ }
+ }
+ for (id, score) in vec::rrf(&lists).into_iter().take(n) {
+ if let Some(note) = g.get(&id) {
+ out.push(Hit {
+ note,
+ score: Some(score),
+ snippet: snippets.remove(&id).flatten(),
+ row: rows.remove(&id),
+ });
+ }
+ }
+ }
+ }
+ Ok(out)
+}
+
+/// The first text of a chunk, for a hit by meaning.
+fn chunk_snippet(h: &Hit, cache: &mut HashMap<String, Vec<Note>>) -> Option<String> {
+ let row = h.row.as_ref()?;
+ let path = &h.note.path;
+ if !cache.contains_key(path) {
+ let content = std::fs::read_to_string(path).ok()?;
+ let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default();
+ cache.insert(path.clone(), v);
+ }
+ let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?;
+ let cs = vec::note_chunks(fresh);
+ let text = &cs.get(row.chunk)?.text;
+ let body = text.strip_prefix(&fresh.title).unwrap_or(text);
+ let flat = body.split_whitespace().collect::<Vec<_>>().join(" ");
+ Some(flat.chars().take(200).collect())
+}
+
+fn cmd_search(ctx: &Ctx, query: Option<String>, n: usize, hybrid: bool) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let mut cache = HashMap::new();
+ for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? {
+ let mut v = view(&g, h.note);
+ v.score = h.score;
+ v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache));
+ v.chunk_line = h.row.as_ref().map(|r| r.line);
+ print(&v)?;
+ }
+ Ok(())
+}
+
+fn cmd_count(ctx: &Ctx, queries: Vec<String>) -> Result<()> {
+ let g = load_graph(ctx)?;
+ for q in queries {
+ let n = run_query(ctx, &g, &q, usize::MAX, false)?.len();
+ print(&serde_json::json!({ "count": n }))?;
+ }
+ Ok(())
+}
+
+fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let center = match g.get(id) {
+ Some(n) => n,
+ None => bail!("no note with id {}", id),
+ };
+ print(&serde_json::json!({ "center": view(&g, center) }))?;
+ for (n, d, rel) in g.reach(id, hops.max(1)) {
+ let mut v = view(&g, n);
+ v.dist = Some(d);
+ v.rel = Some(rel.name());
+ print(&v)?;
+ }
+ Ok(())
+}
+
+fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let n = match g.get(id) {
+ Some(n) => n,
+ None => bail!("no note with id {}", id),
+ };
+ // Read the file again, so the body is never stale.
+ let path = PathBuf::from(&n.path);
+ let content = std::fs::read_to_string(&path)
+ .with_context(|| format!("index out of date: cannot read {}", path.display()))?;
+ let fresh = adapter::parse_file(&path, &content)
+ .unwrap_or_default()
+ .into_iter()
+ .find(|x| x.id.as_deref() == Some(id));
+ let note = fresh.as_ref().unwrap_or(n);
+ print(&serde_json::json!({
+ "note": view(&g, note),
+ "body": note.body,
+ // The exact text of the note in its file, for display.
+ "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()),
+ "stale": fresh.is_none(),
+ "parent": g.parent_of(id).map(|x| view(&g, x)),
+ "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(),
+ }))
+}
+
+fn cmd_check(ctx: &Ctx) -> Result<()> {
+ let g = load_graph(ctx)?;
+ let mut issues: Vec<serde_json::Value> = Vec::new();
+ for n in &g.notes {
+ match &n.id {
+ None => issues.push(serde_json::json!({
+ "issue": "missing-id", "path": n.path, "line": n.line,
+ "start": n.start, "title": n.title,
+ })),
+ Some(id) => {
+ if let Some(p) = &n.parent {
+ if g.get(p).is_none() {
+ issues.push(serde_json::json!({
+ "issue": "dangling-parent", "id": id, "target": p,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ for l in &n.links {
+ if g.get(&l.dst).is_none() {
+ issues.push(serde_json::json!({
+ "issue": "dangling-link", "id": id, "target": l.dst,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ }
+ }
+ }
+ let mut by_path: BTreeMap<&str, usize> = BTreeMap::new();
+ for (id, idxs) in &g.duplicates {
+ for &i in idxs {
+ let n = &g.notes[i];
+ *by_path.entry(&n.path).or_default() += 1;
+ issues.push(serde_json::json!({
+ "issue": "duplicate-id", "id": id,
+ "path": n.path, "line": n.line, "title": n.title,
+ }));
+ }
+ }
+ for i in issues {
+ print(&i)?;
+ }
+ Ok(())
+}
+
+fn cmd_tags(ctx: &Ctx, like: Option<String>) -> Result<()> {
+ let g = load_graph(ctx)?;
+ if let Some(id) = like {
+ let store = Store::load(&ctx.index);
+ let v = store
+ .note_vec(&id)
+ .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?;
+ for (tag, count, score, has) in store.tag_scores(&g, &id, &v) {
+ print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?;
+ }
+ return Ok(());
+ }
+ let mut counts: BTreeMap<&str, usize> = BTreeMap::new();
+ for n in g.notes.iter().filter(|n| n.id.is_some()) {
+ for t in &n.tags {
+ *counts.entry(t).or_default() += 1;
+ }
+ }
+ let mut v: Vec<(&str, usize)> = counts.into_iter().collect();
+ // Most used first, then alphabetical.
+ v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0)));
+ for (tag, count) in v {
+ print(&serde_json::json!({ "tag": tag, "count": count }))?;
+ }
+ Ok(())
+}
+
+fn main() -> Result<()> {
+ let cli = Cli::parse();
+ let ctx = Ctx::new(&cli)?;
+ match cli.cmd {
+ Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed),
+ Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid),
+ Cmd::Count { queries } => cmd_count(&ctx, queries),
+ Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops),
+ Cmd::Get { id } => cmd_get(&ctx, &id),
+ Cmd::Check => cmd_check(&ctx),
+ Cmd::Tags { like } => cmd_tags(&ctx, like),
+ }
+}
diff --git a/src/note.rs b/src/note.rs
new file mode 100644
index 0000000..ef38094
--- /dev/null
+++ b/src/note.rs
@@ -0,0 +1,36 @@
+//! The data model shared by all adapters and by the index.
+
+use serde::{Deserialize, Serialize};
+
+/// A link from one note to another. Types are optional.
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+pub struct Link {
+ pub dst: String,
+ pub kind: Option<String>,
+}
+
+/// One note. A file is a container of notes; the note is the unit.
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Note {
+ /// `None` for a note that has no ID yet; `check` reports these.
+ pub id: Option<String>,
+ pub path: String,
+ /// Byte span of the note inside its file.
+ pub start: usize,
+ pub end: usize,
+ /// 1-based line number of the first line of the note.
+ pub line: usize,
+ /// Hash of the span, used to detect changed notes.
+ pub hash: String,
+ pub title: String,
+ pub date: Option<String>,
+ pub kind: Option<String>,
+ pub tags: Vec<String>,
+ pub parent: Option<String>,
+ pub links: Vec<Link>,
+ pub context: Option<String>,
+ pub summary: Option<String>,
+ /// The text to index. It is never written to the manifest.
+ #[serde(skip)]
+ pub body: String,
+}
diff --git a/src/query.rs b/src/query.rs
new file mode 100644
index 0000000..2b53ae6
--- /dev/null
+++ b/src/query.rs
@@ -0,0 +1,335 @@
+//! The query language.
+//!
+//! A query is one string. Words such as `tag:X` are filters on the
+//! metadata and the graph; everything else is passed on to the full
+//! text index. See the README for the list.
+
+use crate::graph::Graph;
+use crate::note::Note;
+use anyhow::{Result, bail};
+use std::collections::HashSet;
+
+/// A pattern for tags and kinds: `*` (anything), `abc*` (prefix) or `abc`.
+#[derive(Debug, PartialEq)]
+pub enum Pat {
+ Any,
+ Prefix(String),
+ Exact(String),
+}
+
+impl Pat {
+ fn new(v: &str) -> Pat {
+ match v.strip_suffix('*') {
+ Some("") => Pat::Any,
+ Some(p) => Pat::Prefix(p.to_string()),
+ None => Pat::Exact(v.to_string()),
+ }
+ }
+
+ fn matches(&self, s: &str) -> bool {
+ match self {
+ Pat::Any => true,
+ Pat::Prefix(p) => s.starts_with(p.as_str()),
+ Pat::Exact(e) => s == e,
+ }
+ }
+}
+
+/// A range of dates, compared as prefixes: `2025` is the whole year.
+#[derive(Debug, PartialEq)]
+pub struct DateRange {
+ pub from: Option<String>,
+ pub to: Option<String>,
+}
+
+impl DateRange {
+ fn contains(&self, date: &str) -> bool {
+ let cut = |n: usize| date.get(..n).unwrap_or(date);
+ self.from.as_ref().is_none_or(|a| cut(a.len()) >= a.as_str())
+ && self.to.as_ref().is_none_or(|b| cut(b.len()) <= b.as_str())
+ }
+}
+
+#[derive(Debug, Default, PartialEq)]
+pub struct Query {
+ pub tags: Vec<Pat>,
+ pub no_tags: Vec<Pat>,
+ pub kinds: Vec<Pat>,
+ pub no_kinds: Vec<Pat>,
+ pub files: Vec<String>,
+ pub no_files: Vec<String>,
+ pub from: Vec<String>,
+ pub to: Vec<String>,
+ pub parent: Vec<String>,
+ pub thread: Vec<String>,
+ pub dates: Vec<DateRange>,
+ pub inbox: bool,
+ /// `like:X` and `-like:X`: note IDs or free text, for the embeddings.
+ pub like: Vec<String>,
+ pub unlike: Vec<String>,
+ /// What is left for the full text index.
+ pub words: String,
+}
+
+/// Split on white space, but keep double-quoted stretches together.
+/// The quotes stay in the tokens.
+fn tokens(q: &str) -> Vec<String> {
+ let mut out = Vec::new();
+ let mut cur = String::new();
+ let mut quoted = false;
+ for c in q.chars() {
+ match c {
+ '"' => {
+ quoted = !quoted;
+ cur.push(c);
+ }
+ c if c.is_whitespace() && !quoted => {
+ if !cur.is_empty() {
+ out.push(std::mem::take(&mut cur));
+ }
+ }
+ c => cur.push(c),
+ }
+ }
+ if !cur.is_empty() {
+ out.push(cur);
+ }
+ out
+}
+
+fn unquote(v: &str) -> &str {
+ v.strip_prefix('"')
+ .map(|r| r.strip_suffix('"').unwrap_or(r))
+ .unwrap_or(v)
+}
+
+impl Query {
+ pub fn parse(q: &str) -> Result<Query> {
+ let mut out = Query::default();
+ let mut words: Vec<String> = Vec::new();
+ for tok in tokens(q) {
+ let (neg, rest) = match tok.strip_prefix('-') {
+ Some(r) if !r.is_empty() => (true, r),
+ _ => (false, tok.as_str()),
+ };
+ let (key, value) = match rest.split_once(':') {
+ Some((k, v)) if !k.starts_with('"') => (k, v),
+ _ => {
+ words.push(tok);
+ continue;
+ }
+ };
+ let known = matches!(
+ key,
+ "tag" | "kind" | "file" | "from" | "to" | "parent" | "thread" | "date" | "is" | "like"
+ );
+ if !known {
+ // `title:foo` and friends belong to the text index.
+ words.push(tok);
+ continue;
+ }
+ let value = unquote(value);
+ if value.is_empty() {
+ bail!("empty value for {}:", key);
+ }
+ if neg && !matches!(key, "tag" | "kind" | "file" | "like") {
+ bail!("cannot negate {}:", key);
+ }
+ match key {
+ "tag" if neg => out.no_tags.push(Pat::new(value)),
+ "tag" => out.tags.push(Pat::new(value)),
+ "kind" if neg => out.no_kinds.push(Pat::new(value)),
+ "kind" => out.kinds.push(Pat::new(value)),
+ "file" if neg => out.no_files.push(value.to_string()),
+ "file" => out.files.push(value.to_string()),
+ "like" if neg => out.unlike.push(value.to_string()),
+ "like" => out.like.push(value.to_string()),
+ "from" => out.from.push(value.to_string()),
+ "to" => out.to.push(value.to_string()),
+ "parent" => out.parent.push(value.to_string()),
+ "thread" => out.thread.push(value.to_string()),
+ "date" => {
+ let range = match value.split_once("..") {
+ Some((a, b)) => DateRange {
+ from: (!a.is_empty()).then(|| a.to_string()),
+ to: (!b.is_empty()).then(|| b.to_string()),
+ },
+ None => DateRange {
+ from: Some(value.to_string()),
+ to: Some(value.to_string()),
+ },
+ };
+ out.dates.push(range);
+ }
+ "is" => match value {
+ "inbox" => out.inbox = true,
+ _ => bail!("unknown is:{}", value),
+ },
+ _ => unreachable!(),
+ }
+ }
+ out.words = words.join(" ");
+ Ok(out)
+ }
+
+ /// Resolve the parts that need the graph, and return a matcher.
+ pub fn matcher<'a>(&'a self, g: &'a Graph, root: &'a str) -> Matcher<'a> {
+ let ids = |v: Vec<&Note>| -> HashSet<String> {
+ v.into_iter().filter_map(|n| n.id.clone()).collect()
+ };
+ let mut sets: Vec<HashSet<String>> = Vec::new();
+ for id in &self.from {
+ sets.push(ids(g.outlinks_of(id)));
+ }
+ for id in &self.to {
+ sets.push(ids(g.backlinks_of(id)));
+ }
+ for id in &self.parent {
+ sets.push(ids(g.children_of(id)));
+ }
+ for id in &self.thread {
+ sets.push(thread_of(g, id));
+ }
+ Matcher {
+ q: self,
+ g,
+ root,
+ sets,
+ }
+ }
+}
+
+/// All the notes in the thread of `id`: the root above it and every
+/// descendant of that root.
+fn thread_of(g: &Graph, id: &str) -> HashSet<String> {
+ let mut out = HashSet::new();
+ if g.get(id).is_none() {
+ return out;
+ }
+ let mut root = id.to_string();
+ let mut seen = HashSet::new();
+ while seen.insert(root.clone()) {
+ match g.parent_of(&root).and_then(|p| p.id.clone()) {
+ Some(p) => root = p,
+ None => break,
+ }
+ }
+ let mut stack = vec![root];
+ while let Some(cur) = stack.pop() {
+ if out.insert(cur.clone()) {
+ for c in g.children_of(&cur) {
+ if let Some(cid) = &c.id {
+ stack.push(cid.clone());
+ }
+ }
+ }
+ }
+ out
+}
+
+pub struct Matcher<'a> {
+ q: &'a Query,
+ g: &'a Graph,
+ root: &'a str,
+ sets: Vec<HashSet<String>>,
+}
+
+impl Matcher<'_> {
+ pub fn pass(&self, note: &Note) -> bool {
+ let id = match &note.id {
+ Some(i) => i,
+ None => return false,
+ };
+ let q = self.q;
+ let any_tag = |p: &Pat| note.tags.iter().any(|t| p.matches(t));
+ let rel = note
+ .path
+ .strip_prefix(self.root)
+ .unwrap_or(&note.path)
+ .trim_start_matches('/');
+ q.tags.iter().all(|p| match p {
+ Pat::Any => !note.tags.is_empty(),
+ _ => any_tag(p),
+ }) && q.no_tags.iter().all(|p| match p {
+ Pat::Any => note.tags.is_empty(),
+ _ => !any_tag(p),
+ }) && q
+ .kinds
+ .iter()
+ .all(|p| note.kind.as_deref().is_some_and(|k| p.matches(k)))
+ && q.no_kinds.iter().all(|p| match p {
+ Pat::Any => note.kind.is_none(),
+ _ => !note.kind.as_deref().is_some_and(|k| p.matches(k)),
+ })
+ && q.files.iter().all(|f| rel.contains(f.as_str()))
+ && q.no_files.iter().all(|f| !rel.contains(f.as_str()))
+ && self.sets.iter().all(|s| s.contains(id))
+ && q.dates
+ .iter()
+ .all(|r| note.date.as_deref().is_some_and(|d| r.contains(d)))
+ && (!q.inbox || (!self.g.is_connected(note) && note.tags.is_empty()))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn filters_are_split_from_words() {
+ let q = Query::parse("tag:math -tag:old kind:idea \"exact phrase\" word title:foo").unwrap();
+ assert_eq!(q.tags, vec![Pat::Exact("math".into())]);
+ assert_eq!(q.no_tags, vec![Pat::Exact("old".into())]);
+ assert_eq!(q.kinds, vec![Pat::Exact("idea".into())]);
+ assert_eq!(q.words, "\"exact phrase\" word title:foo");
+ }
+
+ #[test]
+ fn star_patterns() {
+ let q = Query::parse("tag:* -tag:alg* kind:*").unwrap();
+ assert_eq!(q.tags, vec![Pat::Any]);
+ assert_eq!(q.no_tags, vec![Pat::Prefix("alg".into())]);
+ assert_eq!(q.kinds, vec![Pat::Any]);
+ assert_eq!(q.words, "");
+ }
+
+ #[test]
+ fn dates_and_quoted_values() {
+ let q = Query::parse("date:2025-03..2025-06 date:..2024 date:2020 file:\"my notes\"").unwrap();
+ assert_eq!(q.dates.len(), 3);
+ assert!(q.dates[0].contains("2025-04-01T10:00:00"));
+ assert!(!q.dates[0].contains("2025-07-01"));
+ assert!(q.dates[0].contains("2025-06-30"));
+ assert!(q.dates[1].contains("2023-01-01"));
+ assert!(!q.dates[1].contains("2025-01-01"));
+ assert!(q.dates[2].contains("2020-05-14T23:21:08+08:00"));
+ assert_eq!(q.files, vec!["my notes"]);
+ }
+
+ #[test]
+ fn errors() {
+ assert!(Query::parse("tag:").is_err());
+ assert!(Query::parse("-from:x").is_err());
+ assert!(Query::parse("is:weird").is_err());
+ // A colon in an ordinary word is fine.
+ assert_eq!(Query::parse("a:b").unwrap().words, "a:b");
+ // A lone dash is a word.
+ assert_eq!(Query::parse("- x").unwrap().words, "- x");
+ }
+
+ #[test]
+ fn like_filters() {
+ let q = Query::parse("like:abc-1 -like:\"moduli of curves\" tag:x word").unwrap();
+ assert_eq!(q.like, vec!["abc-1"]);
+ assert_eq!(q.unlike, vec!["moduli of curves"]);
+ assert_eq!(q.words, "word");
+ }
+
+ #[test]
+ fn ids_with_angle_brackets() {
+ let q = Query::parse("thread:<a.b@c> to:abc-1 is:inbox").unwrap();
+ assert_eq!(q.thread, vec!["<a.b@c>"]);
+ assert_eq!(q.to, vec!["abc-1"]);
+ assert!(q.inbox);
+ }
+}
diff --git a/src/text.rs b/src/text.rs
new file mode 100644
index 0000000..466e065
--- /dev/null
+++ b/src/text.rs
@@ -0,0 +1,188 @@
+//! Full-text index on top of Tantivy.
+//!
+//! Two views of the text are indexed: the normal one, split into words,
+//! and a bigram one, so that Chinese (which has no spaces) can be
+//! matched by substring.
+
+use crate::note::Note;
+use anyhow::Result;
+use std::path::Path;
+use tantivy::collector::TopDocs;
+use tantivy::directory::MmapDirectory;
+use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+use tantivy::schema::{
+ Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value,
+};
+use tantivy::snippet::SnippetGenerator;
+use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer};
+use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc};
+
+/// Marks around highlighted words in snippets. These are private-use
+/// characters, so they cannot appear in normal text.
+pub const HL_START: char = '\u{e000}';
+pub const HL_END: char = '\u{e001}';
+
+pub struct Text {
+ index: Index,
+ id: Field,
+ path: Field,
+ title: Field,
+ body: Field,
+ tags: Field,
+ grams: Field,
+ /// True if the index directory was created by this call.
+ pub fresh: bool,
+}
+
+pub struct Found {
+ pub id: String,
+ pub score: f32,
+ pub snippet: Option<String>,
+}
+
+fn has_cjk(s: &str) -> bool {
+ s.chars().any(|c| !c.is_ascii() && c.is_alphabetic())
+}
+
+impl Text {
+ pub fn open(dir: &Path) -> Result<Text> {
+ let fresh = !dir.join("meta.json").exists();
+ std::fs::create_dir_all(dir)?;
+ let mut sb = Schema::builder();
+ let id = sb.add_text_field("id", STRING | STORED);
+ let path = sb.add_text_field("path", STRING | STORED);
+ let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED);
+ let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED);
+ let tags = sb.add_text_field("tags", tantivy::schema::TEXT);
+ let grams = sb.add_text_field(
+ "grams",
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer("bigram")
+ .set_index_option(IndexRecordOption::Basic),
+ ),
+ );
+ let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?;
+ index.tokenizers().register(
+ "bigram",
+ TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?)
+ .filter(LowerCaser)
+ .build(),
+ );
+ Ok(Text {
+ index,
+ id,
+ path,
+ title,
+ body,
+ tags,
+ grams,
+ fresh,
+ })
+ }
+
+ /// Remove the notes of `removed` files, and replace the notes of the
+ /// `changed` files. Only notes with an ID are indexed.
+ pub fn update(&self, removed: &[String], changed: &[(String, Vec<Note>)]) -> Result<()> {
+ let mut w: IndexWriter = self.index.writer(50_000_000)?;
+ for p in removed {
+ w.delete_term(Term::from_field_text(self.path, p));
+ }
+ for (p, notes) in changed {
+ w.delete_term(Term::from_field_text(self.path, p));
+ for n in notes {
+ let id = match &n.id {
+ Some(id) => id,
+ None => continue,
+ };
+ let mut text = String::new();
+ if let Some(s) = &n.summary {
+ text.push_str(s);
+ text.push('\n');
+ }
+ text.push_str(&n.body);
+ let all = format!("{}\n{}", n.title, text);
+ w.add_document(doc!(
+ self.id => id.as_str(),
+ self.path => p.as_str(),
+ self.title => n.title.as_str(),
+ self.body => text,
+ self.tags => n.tags.join(" "),
+ self.grams => all,
+ ))?;
+ }
+ }
+ w.commit()?;
+ Ok(())
+ }
+
+ /// Build a query. Words go through the query parser (so phrases and
+ /// `title:foo` work); words with CJK characters become a conjunction
+ /// of bigrams.
+ fn build_query(&self, q: &str) -> Result<Box<dyn Query>> {
+ let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]);
+ parser.set_conjunction_by_default();
+ if !has_cjk(q) {
+ return Ok(parser.parse_query(q)?);
+ }
+ let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
+ for word in q.split_whitespace() {
+ if has_cjk(word) {
+ let chars: Vec<char> = word.to_lowercase().chars().collect();
+ if chars.len() == 1 {
+ // A single character has no bigram; match any bigram with it.
+ let t = Term::from_field_text(self.grams, &chars[0].to_string());
+ clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic))));
+ } else {
+ for w in chars.windows(2) {
+ let g: String = w.iter().collect();
+ let t = Term::from_field_text(self.grams, &g);
+ clauses.push((
+ Occur::Must,
+ Box::new(TermQuery::new(t, IndexRecordOption::Basic)),
+ ));
+ }
+ }
+ } else {
+ clauses.push((Occur::Must, parser.parse_query(word)?));
+ }
+ }
+ Ok(Box::new(BooleanQuery::new(clauses)))
+ }
+
+ pub fn search(&self, q: &str, limit: usize) -> Result<Vec<Found>> {
+ let reader = self.index.reader()?;
+ let searcher = reader.searcher();
+ let query = self.build_query(q)?;
+ let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?;
+ let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok();
+ let mut out = Vec::new();
+ for (score, addr) in top {
+ let d: TantivyDocument = searcher.doc(addr)?;
+ let id = match d.get_first(self.id).and_then(|v| v.as_str()) {
+ Some(s) => s.to_string(),
+ None => continue,
+ };
+ let snippet = snipgen.as_ref().and_then(|g| {
+ let s = g.snippet_from_doc(&d);
+ let frag = s.fragment();
+ if frag.is_empty() {
+ return None;
+ }
+ let mut out = String::new();
+ let mut last = 0;
+ for r in s.highlighted() {
+ out.push_str(&frag[last..r.start]);
+ out.push(HL_START);
+ out.push_str(&frag[r.start..r.end]);
+ out.push(HL_END);
+ last = r.end;
+ }
+ out.push_str(&frag[last..]);
+ Some(out.replace('\n', " "))
+ });
+ out.push(Found { id, score, snippet });
+ }
+ Ok(out)
+ }
+}
diff --git a/src/vec.rs b/src/vec.rs
new file mode 100644
index 0000000..2c30ab5
--- /dev/null
+++ b/src/vec.rs
@@ -0,0 +1,651 @@
+//! The semantic side: embeddings from Ollama, a small vector store, and
+//! the ranking on top of it.
+//!
+//! One vector per chunk of a note (see `chunk.rs`). The store keeps two
+//! files next to each other: the vectors as f16 for exact scores, and one
+//! sign bit per dimension for a fast first pass by Hamming distance when
+//! there are very many chunks. Everything here can be rebuilt from the
+//! note files, so it lives in the index directory.
+
+use crate::chunk;
+use crate::graph::Graph;
+use crate::note::Note;
+use anyhow::{Context, Result, anyhow, bail};
+use half::f16;
+use serde::{Deserialize, Serialize};
+use std::collections::{HashMap, HashSet};
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+pub const DEFAULT_MODEL: &str = "qwen3-embedding:0.6b";
+const BATCH: usize = 16;
+/// Above this many chunks, a Hamming pass narrows the candidates first.
+const EXACT_LIMIT: usize = 2000;
+
+// ---------------------------------------------------------------- Ollama
+
+pub struct Ollama {
+ url: String,
+ pub model: String,
+}
+
+/// Why a call to Ollama failed: it is not running, or it is something else.
+#[derive(Debug)]
+pub struct Unreachable(pub String);
+
+impl std::fmt::Display for Unreachable {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}", self.0)
+ }
+}
+
+impl std::error::Error for Unreachable {}
+
+impl Ollama {
+ pub fn from_env() -> Ollama {
+ Ollama {
+ url: std::env::var("ELEL_OLLAMA").unwrap_or_else(|_| "http://localhost:11434".into()),
+ model: std::env::var("ELEL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()),
+ }
+ }
+
+ /// Embed texts, already formatted for the model. The vectors are
+ /// normalized to length 1.
+ pub fn embed(&self, inputs: &[String]) -> Result<Vec<Vec<f32>>> {
+ let agent = ureq::AgentBuilder::new()
+ .timeout_connect(Duration::from_secs(3))
+ .timeout(Duration::from_secs(300))
+ .build();
+ let resp = agent
+ .post(&format!("{}/api/embed", self.url.trim_end_matches('/')))
+ .send_json(serde_json::json!({
+ "model": self.model,
+ "input": inputs,
+ "truncate": true,
+ }));
+ let v: serde_json::Value = match resp {
+ Ok(r) => r.into_json()?,
+ Err(ureq::Error::Status(404, _)) => bail!(
+ "model {} is not available in Ollama; run `ollama pull {}`",
+ self.model,
+ self.model
+ ),
+ Err(ureq::Error::Status(code, r)) => {
+ let body = r.into_string().unwrap_or_default();
+ bail!("Ollama answered {}: {}", code, body)
+ }
+ Err(e @ ureq::Error::Transport(_)) => {
+ return Err(anyhow::Error::new(Unreachable(format!(
+ "cannot reach Ollama at {} ({})",
+ self.url, e
+ ))));
+ }
+ };
+ let arr = v["embeddings"]
+ .as_array()
+ .ok_or_else(|| anyhow!("Ollama gave no embeddings: {}", v))?;
+ let mut out = Vec::new();
+ for e in arr {
+ let mut x: Vec<f32> = e
+ .as_array()
+ .ok_or_else(|| anyhow!("bad embedding"))?
+ .iter()
+ .map(|n| n.as_f64().unwrap_or(0.0) as f32)
+ .collect();
+ normalize(&mut x);
+ out.push(x);
+ }
+ if out.len() != inputs.len() {
+ bail!("asked for {} embeddings, got {}", inputs.len(), out.len());
+ }
+ Ok(out)
+ }
+
+ /// A note chunk, as the model wants a document.
+ pub fn doc_text(&self, text: &str) -> String {
+ if self.model.starts_with("embeddinggemma") {
+ format!("title: none | text: {}", text)
+ } else if self.model.starts_with("nomic-embed") {
+ format!("search_document: {}", text)
+ } else {
+ text.to_string()
+ }
+ }
+
+ /// A search text, as the model wants a query.
+ pub fn query_text(&self, text: &str) -> String {
+ if self.model.starts_with("qwen3-embedding") {
+ format!(
+ "Instruct: Given a search query, retrieve the notes that are about it\nQuery: {}",
+ text
+ )
+ } else if self.model.starts_with("embeddinggemma") {
+ format!("task: search result | query: {}", text)
+ } else if self.model.starts_with("nomic-embed") {
+ format!("search_query: {}", text)
+ } else {
+ text.to_string()
+ }
+ }
+
+ pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> {
+ Ok(self.embed(&[self.query_text(text)])?.remove(0))
+ }
+}
+
+// ------------------------------------------------------------ vector math
+
+pub fn normalize(v: &mut [f32]) {
+ let n = v.iter().map(|x| x * x).sum::<f32>().sqrt();
+ if n > 0.0 {
+ for x in v.iter_mut() {
+ *x /= n;
+ }
+ }
+}
+
+pub fn dot(a: &[f32], b: &[f32]) -> f32 {
+ a.iter().zip(b).map(|(x, y)| x * y).sum()
+}
+
+/// The sign bits of a vector, packed eight to a byte.
+pub fn to_bits(v: &[f32]) -> Vec<u8> {
+ let mut out = vec![0u8; v.len().div_ceil(8)];
+ for (i, x) in v.iter().enumerate() {
+ if *x > 0.0 {
+ out[i / 8] |= 1 << (i % 8);
+ }
+ }
+ out
+}
+
+pub fn hamming(a: &[u8], b: &[u8]) -> u32 {
+ a.iter().zip(b).map(|(x, y)| (x ^ y).count_ones()).sum()
+}
+
+/// The normalized mean of some vectors.
+pub fn mean(vs: &[&[f32]]) -> Option<Vec<f32>> {
+ let first = vs.first()?;
+ let mut m = vec![0.0f32; first.len()];
+ for v in vs {
+ for (a, b) in m.iter_mut().zip(*v) {
+ *a += b;
+ }
+ }
+ normalize(&mut m);
+ Some(m)
+}
+
+/// The merged vector of some liked and some disliked vectors:
+/// `normalize(mean(pos) - 0.5 * mean(neg))`.
+pub fn rocchio(pos: &[Vec<f32>], neg: &[Vec<f32>]) -> Option<Vec<f32>> {
+ let p: Vec<&[f32]> = pos.iter().map(|v| v.as_slice()).collect();
+ let mut q = mean(&p)?;
+ let n: Vec<&[f32]> = neg.iter().map(|v| v.as_slice()).collect();
+ if let Some(nm) = mean(&n) {
+ for (a, b) in q.iter_mut().zip(&nm) {
+ *a -= 0.5 * b;
+ }
+ normalize(&mut q);
+ }
+ Some(q)
+}
+
+/// Reciprocal rank fusion of ranked lists of IDs: `sum 1/(60 + rank)`.
+pub fn rrf(lists: &[Vec<String>]) -> Vec<(String, f32)> {
+ let mut score: HashMap<&str, f32> = HashMap::new();
+ for l in lists {
+ for (r, id) in l.iter().enumerate() {
+ *score.entry(id).or_default() += 1.0 / (60.0 + r as f32 + 1.0);
+ }
+ }
+ let mut v: Vec<(String, f32)> = score.into_iter().map(|(k, s)| (k.to_string(), s)).collect();
+ v.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap().then(a.0.cmp(&b.0)));
+ v
+}
+
+// ------------------------------------------------------------------ store
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct Row {
+ pub id: String,
+ /// `Note.hash` when the vector was made.
+ pub hash: String,
+ /// Index of the chunk in the note.
+ pub chunk: usize,
+ /// 0-based line in the body where the chunk starts.
+ pub line: usize,
+}
+
+#[derive(Default, Serialize, Deserialize)]
+struct Meta {
+ model: String,
+ dim: usize,
+ rows: Vec<Row>,
+}
+
+#[derive(Default)]
+pub struct Store {
+ pub model: String,
+ pub dim: usize,
+ pub rows: Vec<Row>,
+ vecs: Vec<f32>,
+ bits: Vec<u8>,
+}
+
+/// A hit of the vector ranking.
+pub struct VecHit {
+ pub id: String,
+ pub score: f32,
+ /// The best chunk, when ranked by chunk.
+ pub row: Option<Row>,
+}
+
+pub fn dir(index: &Path) -> PathBuf {
+ index.join("vec")
+}
+
+impl Store {
+ pub fn load(index: &Path) -> Store {
+ let d = dir(index);
+ let meta: Meta = std::fs::read_to_string(d.join("meta.json"))
+ .ok()
+ .and_then(|s| serde_json::from_str(&s).ok())
+ .unwrap_or_default();
+ let raw = std::fs::read(d.join("f16.bin")).unwrap_or_default();
+ let want = meta.rows.len() * meta.dim * 2;
+ if meta.rows.is_empty() || raw.len() != want {
+ return Store::default();
+ }
+ let vecs: Vec<f32> = raw
+ .chunks_exact(2)
+ .map(|b| f16::from_le_bytes([b[0], b[1]]).to_f32())
+ .collect();
+ let bits = std::fs::read(d.join("bits.bin")).unwrap_or_default();
+ Store {
+ model: meta.model,
+ dim: meta.dim,
+ rows: meta.rows,
+ vecs,
+ bits,
+ }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.rows.is_empty()
+ }
+
+ pub fn save(&self, index: &Path) -> Result<()> {
+ let d = dir(index);
+ std::fs::create_dir_all(&d)?;
+ let write = |name: &str, bytes: &[u8]| -> Result<()> {
+ let tmp = d.join(format!("{}.tmp", name));
+ std::fs::write(&tmp, bytes)?;
+ std::fs::rename(&tmp, d.join(name))?;
+ Ok(())
+ };
+ let mut f = Vec::with_capacity(self.vecs.len() * 2);
+ for x in &self.vecs {
+ f.extend_from_slice(&f16::from_f32(*x).to_le_bytes());
+ }
+ // The bit file must be consistent with the (rounded) f16 file.
+ write("f16.bin", &f)?;
+ write("bits.bin", &self.bits)?;
+ let meta = Meta {
+ model: self.model.clone(),
+ dim: self.dim,
+ rows: self.rows.clone(),
+ };
+ write("meta.json", &serde_json::to_vec(&meta)?)
+ }
+
+ fn vec_of(&self, i: usize) -> &[f32] {
+ &self.vecs[i * self.dim..(i + 1) * self.dim]
+ }
+
+ fn bits_of(&self, i: usize) -> &[u8] {
+ let w = self.dim.div_ceil(8);
+ &self.bits[i * w..(i + 1) * w]
+ }
+
+ /// Keep the rows for which `keep` is true.
+ pub fn retain(&mut self, keep: impl Fn(&Row) -> bool) {
+ let w = self.dim.div_ceil(8);
+ let mut rows = Vec::new();
+ let mut vecs = Vec::new();
+ let mut bits = Vec::new();
+ for (i, r) in self.rows.iter().enumerate() {
+ if keep(r) {
+ rows.push(r.clone());
+ vecs.extend_from_slice(self.vec_of(i));
+ bits.extend_from_slice(&self.bits[i * w..(i + 1) * w]);
+ }
+ }
+ self.rows = rows;
+ self.vecs = vecs;
+ self.bits = bits;
+ }
+
+ pub fn push(&mut self, row: Row, v: &[f32]) {
+ if self.rows.is_empty() {
+ self.dim = v.len();
+ }
+ self.bits.extend_from_slice(&to_bits(v));
+ self.vecs.extend_from_slice(v);
+ self.rows.push(row);
+ }
+
+ #[cfg(test)]
+ pub fn has(&self, id: &str, hash: &str) -> bool {
+ self.rows.iter().any(|r| r.id == id && r.hash == hash)
+ }
+
+ /// The vector of a whole note: the mean of its chunks.
+ pub fn note_vec(&self, id: &str) -> Option<Vec<f32>> {
+ let vs: Vec<&[f32]> = self
+ .rows
+ .iter()
+ .enumerate()
+ .filter(|(_, r)| r.id == id)
+ .map(|(i, _)| self.vec_of(i))
+ .collect();
+ mean(&vs)
+ }
+
+ /// All note vectors at once.
+ pub fn note_vecs(&self) -> HashMap<String, Vec<f32>> {
+ let mut groups: HashMap<&str, Vec<&[f32]>> = HashMap::new();
+ for (i, r) in self.rows.iter().enumerate() {
+ groups.entry(&r.id).or_default().push(self.vec_of(i));
+ }
+ groups
+ .into_iter()
+ .filter_map(|(id, vs)| mean(&vs).map(|m| (id.to_string(), m)))
+ .collect()
+ }
+
+ /// Rank the notes for the query vector `q`. With `by_chunk` a note
+ /// scores as its best chunk (for text queries); otherwise as the mean
+ /// of its chunks (for "like this note"). `allow` filters by ID.
+ pub fn rank(&self, q: &[f32], by_chunk: bool, allow: &dyn Fn(&str) -> bool) -> Vec<VecHit> {
+ let mut out: Vec<VecHit> = Vec::new();
+ if self.is_empty() || q.len() != self.dim {
+ return out;
+ }
+ if by_chunk {
+ let mut cand: Vec<usize> = (0..self.rows.len())
+ .filter(|&i| allow(&self.rows[i].id))
+ .collect();
+ if cand.len() > EXACT_LIMIT {
+ let qb = to_bits(q);
+ cand.sort_by_key(|&i| hamming(&qb, self.bits_of(i)));
+ cand.truncate(EXACT_LIMIT);
+ }
+ let mut best: HashMap<&str, (f32, usize)> = HashMap::new();
+ for i in cand {
+ let s = dot(q, self.vec_of(i));
+ let e = best.entry(&self.rows[i].id).or_insert((f32::MIN, i));
+ if s > e.0 {
+ *e = (s, i);
+ }
+ }
+ for (id, (s, i)) in best {
+ out.push(VecHit {
+ id: id.to_string(),
+ score: s,
+ row: Some(self.rows[i].clone()),
+ });
+ }
+ } else {
+ for (id, v) in self.note_vecs() {
+ if allow(&id) {
+ out.push(VecHit {
+ score: dot(q, &v),
+ id,
+ row: None,
+ });
+ }
+ }
+ }
+ out.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap().then(a.id.cmp(&b.id)));
+ out
+ }
+
+ /// Rank the tags for a note vector. The anchor of a tag is the mean
+ /// of the vectors of the other notes that carry it; tags with fewer
+ /// than two such notes are skipped.
+ pub fn tag_scores(&self, g: &Graph, id: &str, v: &[f32]) -> Vec<(String, usize, f32, bool)> {
+ let vecs = self.note_vecs();
+ let mut by_tag: HashMap<&str, Vec<&[f32]>> = HashMap::new();
+ let mut count: HashMap<&str, usize> = HashMap::new();
+ let mut has: HashSet<&str> = HashSet::new();
+ for n in g.notes.iter().filter(|n| n.id.is_some()) {
+ let nid = n.id.as_deref().unwrap();
+ for t in &n.tags {
+ *count.entry(t).or_default() += 1;
+ if nid == id {
+ has.insert(t);
+ } else if let Some(x) = vecs.get(nid) {
+ by_tag.entry(t).or_default().push(x);
+ }
+ }
+ }
+ let mut out = Vec::new();
+ for (t, vs) in by_tag {
+ if vs.len() < 2 {
+ continue;
+ }
+ if let Some(a) = mean(&vs) {
+ out.push((t.to_string(), count[t], dot(v, &a), has.contains(t)));
+ }
+ }
+ out.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap().then(a.0.cmp(&b.0)));
+ out
+ }
+}
+
+// ------------------------------------------------------------------ sync
+
+/// The text of the chunks of a note, formatted for the model.
+pub fn note_chunks(n: &Note) -> Vec<chunk::Chunk> {
+ chunk::chunks(&n.title, &n.tags, n.summary.as_deref(), &n.body)
+}
+
+pub struct SyncReport {
+ pub embedded: usize,
+ pub pending: usize,
+ pub error: Option<String>,
+}
+
+/// Bring the store up to date with the graph. Notes without a vector are
+/// embedded; vectors of notes that changed or vanished are dropped.
+/// If Ollama fails, the notes done so far are kept and the rest stay
+/// pending for the next run.
+pub fn sync(index: &Path, g: &Graph, ollama: &Ollama) -> Result<SyncReport> {
+ let mut store = Store::load(index);
+ if store.model != ollama.model {
+ store = Store::default();
+ }
+ let current: HashMap<&str, &str> = g
+ .notes
+ .iter()
+ .filter_map(|n| n.id.as_deref().map(|i| (i, n.hash.as_str())))
+ .collect();
+ store.retain(|r| current.get(r.id.as_str()) == Some(&r.hash.as_str()));
+ store.model = ollama.model.clone();
+ let have: HashSet<String> = store.rows.iter().map(|r| r.id.clone()).collect();
+ let mut seen: HashSet<&str> = HashSet::new();
+ let todo: Vec<&Note> = g
+ .notes
+ .iter()
+ .filter(|n| match n.id.as_deref() {
+ Some(i) => !have.contains(i) && seen.insert(i),
+ None => false,
+ })
+ .collect();
+ let total = todo.len();
+ let mut embedded = 0;
+ let mut error = None;
+ let mut parsed: HashMap<String, Vec<Note>> = HashMap::new();
+ // Work in groups of notes, so a failure never leaves half a note.
+ let mut group: Vec<(&Note, Vec<chunk::Chunk>)> = Vec::new();
+ let mut size = 0;
+ let flush = |group: &mut Vec<(&Note, Vec<chunk::Chunk>)>,
+ store: &mut Store|
+ -> Result<usize> {
+ if group.is_empty() {
+ return Ok(0);
+ }
+ let texts: Vec<String> = group
+ .iter()
+ .flat_map(|(_, cs)| cs.iter().map(|c| ollama.doc_text(&c.text)))
+ .collect();
+ let mut vs = Vec::new();
+ for b in texts.chunks(BATCH) {
+ let t0 = std::time::Instant::now();
+ vs.extend(ollama.embed(b)?);
+ if std::env::var_os("ELEL_DEBUG").is_some() {
+ eprintln!(
+ "embed batch of {}: {} chars, max {}, {:.1}s",
+ b.len(),
+ b.iter().map(|x| x.len()).sum::<usize>(),
+ b.iter().map(|x| x.len()).max().unwrap_or(0),
+ t0.elapsed().as_secs_f32()
+ );
+ }
+ }
+ let mut k = 0;
+ let mut notes = 0;
+ for (n, cs) in group.iter() {
+ for (ci, c) in cs.iter().enumerate() {
+ store.push(
+ Row {
+ id: n.id.clone().unwrap(),
+ hash: n.hash.clone(),
+ chunk: ci,
+ line: c.line,
+ },
+ &vs[k],
+ );
+ k += 1;
+ }
+ notes += 1;
+ }
+ group.clear();
+ Ok(notes)
+ };
+ 'work: for n in todo {
+ // The body is not in the manifest: read it from the file again.
+ let notes = match parsed.get(&n.path) {
+ Some(v) => v,
+ None => {
+ let content = match std::fs::read_to_string(&n.path) {
+ Ok(c) => c,
+ Err(_) => continue,
+ };
+ let v = crate::adapter::parse_file(Path::new(&n.path), &content).unwrap_or_default();
+ parsed.entry(n.path.clone()).or_insert(v)
+ }
+ };
+ let fresh = match notes.iter().find(|x| x.id == n.id && x.hash == n.hash) {
+ Some(f) => f,
+ None => continue,
+ };
+ let cs = note_chunks(fresh);
+ size += cs.len();
+ group.push((n, cs));
+ if size >= BATCH {
+ match flush(&mut group, &mut store) {
+ Ok(k) => embedded += k,
+ Err(e) => {
+ error = Some(e.to_string());
+ break 'work;
+ }
+ }
+ size = 0;
+ }
+ }
+ if error.is_none() {
+ match flush(&mut group, &mut store) {
+ Ok(k) => embedded += k,
+ Err(e) => error = Some(e.to_string()),
+ }
+ }
+ store.save(index).context("cannot save the vectors")?;
+ Ok(SyncReport {
+ embedded,
+ pending: total - embedded,
+ error,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn unit(v: &[f32]) -> Vec<f32> {
+ let mut x = v.to_vec();
+ normalize(&mut x);
+ x
+ }
+
+ #[test]
+ fn bits_and_hamming() {
+ let a = to_bits(&[1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0]);
+ assert_eq!(a, vec![0b0000_1101, 0b0000_0001]);
+ let b = to_bits(&[1.0, 1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, -1.0]);
+ assert_eq!(hamming(&a, &b), 2);
+ }
+
+ #[test]
+ fn f16_round_trip_keeps_the_order() {
+ let dir = std::env::temp_dir().join(format!("elel-vec-{}", std::process::id()));
+ let mut s = Store::default();
+ s.model = "m".into();
+ let row = |id: &str| Row { id: id.into(), hash: "h".into(), chunk: 0, line: 0 };
+ s.push(row("a"), &unit(&[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]));
+ s.push(row("b"), &unit(&[1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]));
+ s.push(row("c"), &unit(&[0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]));
+ s.save(&dir).unwrap();
+ let t = Store::load(&dir);
+ std::fs::remove_dir_all(&dir).ok();
+ assert_eq!(t.rows.len(), 3);
+ let q = unit(&[1.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]);
+ let r = t.rank(&q, true, &|_| true);
+ assert_eq!(r.iter().map(|h| h.id.as_str()).collect::<Vec<_>>(), vec!["a", "b", "c"]);
+ let r = t.rank(&q, false, &|id| id != "a");
+ assert_eq!(r[0].id, "b");
+ }
+
+ #[test]
+ fn rocchio_moves_away_from_the_negative() {
+ let pos = vec![unit(&[1.0, 1.0, 0.0])];
+ let neg = vec![unit(&[0.0, 1.0, 0.0])];
+ let q = rocchio(&pos, &neg).unwrap();
+ assert!(q[0] > q[1], "{:?}", q);
+ let plain = rocchio(&pos, &[]).unwrap();
+ assert!((plain[0] - plain[1]).abs() < 1e-6);
+ assert!(rocchio(&[], &neg).is_none());
+ }
+
+ #[test]
+ fn rrf_prefers_notes_high_in_both_lists() {
+ let l1 = vec!["a".to_string(), "b".into(), "c".into()];
+ let l2 = vec!["b".to_string(), "c".into(), "d".into()];
+ let r = rrf(&[l1, l2]);
+ assert_eq!(r[0].0, "b");
+ assert_eq!(r.len(), 4);
+ }
+
+ #[test]
+ fn retain_drops_stale_rows() {
+ let mut s = Store::default();
+ let row = |id: &str, h: &str| Row { id: id.into(), hash: h.into(), chunk: 0, line: 0 };
+ s.push(row("a", "1"), &unit(&[1.0, 0.0]));
+ s.push(row("b", "1"), &unit(&[0.0, 1.0]));
+ s.retain(|r| r.id == "a" && r.hash == "1");
+ assert_eq!(s.rows.len(), 1);
+ assert!(s.has("a", "1") && !s.has("b", "1"));
+ assert_eq!(s.vec_of(0), &[1.0, 0.0]);
+ }
+}