From 5b9a806af13d38163156302097e405a7b145b6cc Mon Sep 17 00:00:00 2001 From: JSDurand Date: Thu, 19 Nov 2026 09:29:47 +0800 Subject: elel: notes engine (Rust) and Gnus-like Emacs frontend Tantivy full text index, link graph, inline query filters, Ollama embeddings with paragraph chunks, and an Elisp summary/article UI. Co-Authored-By: Claude Sonnet 5.5 --- src/chunk.rs | 314 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 src/chunk.rs (limited to 'src/chunk.rs') diff --git a/src/chunk.rs b/src/chunk.rs new file mode 100644 index 0000000..b17316b --- /dev/null +++ b/src/chunk.rs @@ -0,0 +1,314 @@ +//! Cut a note into chunks for the embeddings. +//! +//! A chunk is a paragraph or a few paragraphs. Blocks that must stay +//! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A +//! paragraph that is only a formula is merged into a neighbour, because +//! alone it would be embedded as "some mathematics" and look like every +//! other formula. Each chunk is prefixed with the title, so a bare +//! "this follows from the lemma" still knows what it is about. + +/// Paragraphs shorter than this are merged with a neighbour. +const MIN: usize = 200; +/// A chunk is not grown beyond this (a single paragraph may exceed it). +const MAX: usize = 1500; +/// A paragraph with fewer words than this, once the math is removed, +/// is a formula paragraph. +const MIN_WORDS: usize = 5; + +#[derive(Debug, Clone, PartialEq)] +pub struct Chunk { + pub text: String, + /// 0-based line of the body where the chunk starts. + pub line: usize, +} + +/// A paragraph of the body: its text and its first line. +struct Para { + text: String, + line: usize, +} + +/// Replace `[[note:ID][desc]]` by `desc` and org links by their text. +pub fn strip_links(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find("[[") { + out.push_str(&rest[..i]); + let tail = &rest[i + 2..]; + match tail.find("]]") { + Some(j) => { + let inner = &tail[..j]; + match inner.split_once("][") { + Some((_, desc)) => out.push_str(desc), + None => out.push_str(inner), + } + rest = &tail[j + 2..]; + } + None => { + out.push_str(&rest[i..]); + rest = ""; + } + } + } + out.push_str(rest); + out +} + +/// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed. +fn without_math(s: &str) -> String { + let mut out = String::new(); + let mut rest = s; + 'outer: while !rest.is_empty() { + for (open, close) in [ + ("$$", "$$"), + ("\\[", "\\]"), + ("\\(", "\\)"), + ("$", "$"), + ] { + if let Some(r) = rest.strip_prefix(open) { + match r.find(close) { + Some(j) => rest = &r[j + close.len()..], + None => rest = "", + } + out.push(' '); + continue 'outer; + } + } + if rest.starts_with("\\begin{") { + match rest.find("\\end{") { + Some(j) => { + let after = &rest[j..]; + rest = match after.find('}') { + Some(k) => &after[k + 1..], + None => "", + }; + out.push(' '); + continue 'outer; + } + None => break, + } + } + let c = rest.chars().next().unwrap(); + out.push(c); + rest = &rest[c.len_utf8()..]; + } + out +} + +/// Does the paragraph hold almost nothing but math? +fn is_formula(p: &str) -> bool { + let t = p.trim(); + if t.is_empty() { + return false; + } + let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{"); + if !has_math { + return false; + } + let left = without_math(t); + // CJK has no spaces: count its characters as words. + let words = left + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 }) + .sum::(); + words < MIN_WORDS +} + +/// Split into paragraphs, keeping blocks whole. +fn paragraphs(body: &str) -> Vec { + let mut out: Vec = Vec::new(); + let mut cur: Vec<&str> = Vec::new(); + let mut start = 0; + // Inside a block that ends with this line (lowercased, trimmed prefix). + let mut block_end: Option = None; + let mut dollars = false; + let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec| { + if !cur.is_empty() { + out.push(Para { text: cur.join("\n"), line: start }); + cur.clear(); + } + }; + for (i, l) in body.lines().enumerate() { + let t = l.trim(); + let tl = t.to_lowercase(); + if let Some(end) = &block_end { + cur.push(l); + if tl.starts_with(end.as_str()) { + block_end = None; + } + continue; + } + if dollars { + cur.push(l); + if t.contains("$$") { + dollars = false; + } + continue; + } + if t.is_empty() { + flush(&mut cur, start, &mut out); + continue; + } + let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' '); + if heading { + flush(&mut cur, start, &mut out); + } + if cur.is_empty() { + start = i; + } + if let Some(rest) = tl.strip_prefix("#+begin_") { + let name = rest.split_whitespace().next().unwrap_or(""); + block_end = Some(format!("#+end_{}", name)); + } else if let Some(rest) = t.strip_prefix("\\begin{") { + let name = rest.split('}').next().unwrap_or(""); + block_end = Some(format!("\\end{{{}}}", name.to_lowercase())); + } else if t.matches("$$").count() % 2 == 1 { + dollars = true; + } + // A one-line block (\begin{x} .. \end{x}) closes at once. + if let Some(end) = &block_end { + if tl[1..].contains(end.as_str()) { + block_end = None; + } + } + cur.push(l); + } + flush(&mut cur, start, &mut out); + out +} + +/// The chunks of a note. A note without a body gets one chunk from its +/// title, tags and summary. +pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec { + let body = strip_links(body); + let paras = paragraphs(&body); + // Merge formula paragraphs into the previous paragraph (or the next one). + let mut merged: Vec = Vec::new(); + let mut carry: Option = None; + for p in paras { + let formula = is_formula(&p.text); + let p = match carry.take() { + Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line }, + None => p, + }; + if formula && merged.is_empty() { + // Opens the note: it joins the next paragraph. + carry = Some(p); + } else if formula { + let last = merged.last_mut().unwrap(); + last.text = format!("{}\n{}", last.text, p.text); + } else { + merged.push(p); + } + } + if let Some(c) = carry { + merged.push(c); + } + // Merge small paragraphs, up to MAX. + let mut out: Vec = Vec::new(); + let mut small = false; + for p in merged { + match out.last_mut() { + Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => { + last.text.push_str("\n\n"); + last.text.push_str(&p.text); + small = last.text.len() < MIN; + } + _ => { + small = p.text.len() < MIN; + out.push(Chunk { text: p.text, line: p.line }); + } + } + } + if out.is_empty() { + let mut t = String::new(); + if let Some(s) = summary { + t.push_str(s); + } + if !tags.is_empty() { + t.push_str("\n"); + t.push_str(&tags.join(" ")); + } + out.push(Chunk { text: t, line: 0 }); + } + for c in &mut out { + c.text = format!("{}\n\n{}", title, c.text.trim()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn long(s: &str) -> String { + format!("{} {}", s, "word ".repeat(60)) + } + + #[test] + fn formula_paragraph_joins_the_previous_one() { + let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 2, "{:?}", c); + assert!(c[0].text.contains("x^2")); + assert!(!c[1].text.contains("x^2")); + } + + #[test] + fn formula_at_the_start_joins_the_next() { + let body = format!("$$a=b$$\n\n{}", long("Prose")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose")); + } + + #[test] + fn prose_with_inline_math_is_not_a_formula() { + assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent.")); + assert!(is_formula("\\[ \\int_0^1 f \\]")); + assert!(is_formula("$a$ and $b$")); + } + + #[test] + fn source_blocks_stay_whole() { + let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two")); + let p = paragraphs(&body); + assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::>()); + assert!(p[1].text.contains("(b)")); + } + + #[test] + fn environment_stays_whole() { + let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore"; + let p = paragraphs(body); + assert_eq!(p.len(), 3); + assert!(p[1].text.contains("\\end{align}")); + } + + #[test] + fn small_paragraphs_are_merged() { + let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third"); + assert_eq!(c.len(), 1); + assert!(c[0].text.starts_with("T\n\n")); + } + + #[test] + fn links_become_their_description() { + assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2"); + } + + #[test] + fn empty_body_uses_title_and_tags() { + let c = chunks("Title", &["x".into()], Some("sum"), ""); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("sum") && c[0].text.contains("x")); + } + + #[test] + fn line_numbers_point_at_the_paragraph() { + let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.iter().map(|x| x.line).collect::>(), vec![0, 2, 4]); + } +} -- cgit v1.2.3-18-g5258