aboutsummaryrefslogtreecommitdiff
path: root/src/chunk.rs
diff options
context:
space:
mode:
authorJSDurand <mmemmew@gmail.com>2026-11-19 09:29:47 +0800
committerJSDurand <mmemmew@gmail.com>2026-11-19 09:29:47 +0800
commit5b9a806af13d38163156302097e405a7b145b6cc (patch)
treec079f3c4e027dc730892630f667b0e6401736043 /src/chunk.rs
elel: notes engine (Rust) and Gnus-like Emacs frontendHEADmaster
Tantivy full text index, link graph, inline query filters, Ollama embeddings with paragraph chunks, and an Elisp summary/article UI. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Diffstat (limited to 'src/chunk.rs')
-rw-r--r--src/chunk.rs314
1 files changed, 314 insertions, 0 deletions
diff --git a/src/chunk.rs b/src/chunk.rs
new file mode 100644
index 0000000..b17316b
--- /dev/null
+++ b/src/chunk.rs
@@ -0,0 +1,314 @@
+//! Cut a note into chunks for the embeddings.
+//!
+//! A chunk is a paragraph or a few paragraphs. Blocks that must stay
+//! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A
+//! paragraph that is only a formula is merged into a neighbour, because
+//! alone it would be embedded as "some mathematics" and look like every
+//! other formula. Each chunk is prefixed with the title, so a bare
+//! "this follows from the lemma" still knows what it is about.
+
+/// Paragraphs shorter than this are merged with a neighbour.
+const MIN: usize = 200;
+/// A chunk is not grown beyond this (a single paragraph may exceed it).
+const MAX: usize = 1500;
+/// A paragraph with fewer words than this, once the math is removed,
+/// is a formula paragraph.
+const MIN_WORDS: usize = 5;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Chunk {
+ pub text: String,
+ /// 0-based line of the body where the chunk starts.
+ pub line: usize,
+}
+
+/// A paragraph of the body: its text and its first line.
+struct Para {
+ text: String,
+ line: usize,
+}
+
+/// Replace `[[note:ID][desc]]` by `desc` and org links by their text.
+pub fn strip_links(s: &str) -> String {
+ let mut out = String::with_capacity(s.len());
+ let mut rest = s;
+ while let Some(i) = rest.find("[[") {
+ out.push_str(&rest[..i]);
+ let tail = &rest[i + 2..];
+ match tail.find("]]") {
+ Some(j) => {
+ let inner = &tail[..j];
+ match inner.split_once("][") {
+ Some((_, desc)) => out.push_str(desc),
+ None => out.push_str(inner),
+ }
+ rest = &tail[j + 2..];
+ }
+ None => {
+ out.push_str(&rest[i..]);
+ rest = "";
+ }
+ }
+ }
+ out.push_str(rest);
+ out
+}
+
+/// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed.
+fn without_math(s: &str) -> String {
+ let mut out = String::new();
+ let mut rest = s;
+ 'outer: while !rest.is_empty() {
+ for (open, close) in [
+ ("$$", "$$"),
+ ("\\[", "\\]"),
+ ("\\(", "\\)"),
+ ("$", "$"),
+ ] {
+ if let Some(r) = rest.strip_prefix(open) {
+ match r.find(close) {
+ Some(j) => rest = &r[j + close.len()..],
+ None => rest = "",
+ }
+ out.push(' ');
+ continue 'outer;
+ }
+ }
+ if rest.starts_with("\\begin{") {
+ match rest.find("\\end{") {
+ Some(j) => {
+ let after = &rest[j..];
+ rest = match after.find('}') {
+ Some(k) => &after[k + 1..],
+ None => "",
+ };
+ out.push(' ');
+ continue 'outer;
+ }
+ None => break,
+ }
+ }
+ let c = rest.chars().next().unwrap();
+ out.push(c);
+ rest = &rest[c.len_utf8()..];
+ }
+ out
+}
+
+/// Does the paragraph hold almost nothing but math?
+fn is_formula(p: &str) -> bool {
+ let t = p.trim();
+ if t.is_empty() {
+ return false;
+ }
+ let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{");
+ if !has_math {
+ return false;
+ }
+ let left = without_math(t);
+ // CJK has no spaces: count its characters as words.
+ let words = left
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|w| !w.is_empty())
+ .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 })
+ .sum::<usize>();
+ words < MIN_WORDS
+}
+
+/// Split into paragraphs, keeping blocks whole.
+fn paragraphs(body: &str) -> Vec<Para> {
+ let mut out: Vec<Para> = Vec::new();
+ let mut cur: Vec<&str> = Vec::new();
+ let mut start = 0;
+ // Inside a block that ends with this line (lowercased, trimmed prefix).
+ let mut block_end: Option<String> = None;
+ let mut dollars = false;
+ let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec<Para>| {
+ if !cur.is_empty() {
+ out.push(Para { text: cur.join("\n"), line: start });
+ cur.clear();
+ }
+ };
+ for (i, l) in body.lines().enumerate() {
+ let t = l.trim();
+ let tl = t.to_lowercase();
+ if let Some(end) = &block_end {
+ cur.push(l);
+ if tl.starts_with(end.as_str()) {
+ block_end = None;
+ }
+ continue;
+ }
+ if dollars {
+ cur.push(l);
+ if t.contains("$$") {
+ dollars = false;
+ }
+ continue;
+ }
+ if t.is_empty() {
+ flush(&mut cur, start, &mut out);
+ continue;
+ }
+ let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' ');
+ if heading {
+ flush(&mut cur, start, &mut out);
+ }
+ if cur.is_empty() {
+ start = i;
+ }
+ if let Some(rest) = tl.strip_prefix("#+begin_") {
+ let name = rest.split_whitespace().next().unwrap_or("");
+ block_end = Some(format!("#+end_{}", name));
+ } else if let Some(rest) = t.strip_prefix("\\begin{") {
+ let name = rest.split('}').next().unwrap_or("");
+ block_end = Some(format!("\\end{{{}}}", name.to_lowercase()));
+ } else if t.matches("$$").count() % 2 == 1 {
+ dollars = true;
+ }
+ // A one-line block (\begin{x} .. \end{x}) closes at once.
+ if let Some(end) = &block_end {
+ if tl[1..].contains(end.as_str()) {
+ block_end = None;
+ }
+ }
+ cur.push(l);
+ }
+ flush(&mut cur, start, &mut out);
+ out
+}
+
+/// The chunks of a note. A note without a body gets one chunk from its
+/// title, tags and summary.
+pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec<Chunk> {
+ let body = strip_links(body);
+ let paras = paragraphs(&body);
+ // Merge formula paragraphs into the previous paragraph (or the next one).
+ let mut merged: Vec<Para> = Vec::new();
+ let mut carry: Option<Para> = None;
+ for p in paras {
+ let formula = is_formula(&p.text);
+ let p = match carry.take() {
+ Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line },
+ None => p,
+ };
+ if formula && merged.is_empty() {
+ // Opens the note: it joins the next paragraph.
+ carry = Some(p);
+ } else if formula {
+ let last = merged.last_mut().unwrap();
+ last.text = format!("{}\n{}", last.text, p.text);
+ } else {
+ merged.push(p);
+ }
+ }
+ if let Some(c) = carry {
+ merged.push(c);
+ }
+ // Merge small paragraphs, up to MAX.
+ let mut out: Vec<Chunk> = Vec::new();
+ let mut small = false;
+ for p in merged {
+ match out.last_mut() {
+ Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => {
+ last.text.push_str("\n\n");
+ last.text.push_str(&p.text);
+ small = last.text.len() < MIN;
+ }
+ _ => {
+ small = p.text.len() < MIN;
+ out.push(Chunk { text: p.text, line: p.line });
+ }
+ }
+ }
+ if out.is_empty() {
+ let mut t = String::new();
+ if let Some(s) = summary {
+ t.push_str(s);
+ }
+ if !tags.is_empty() {
+ t.push_str("\n");
+ t.push_str(&tags.join(" "));
+ }
+ out.push(Chunk { text: t, line: 0 });
+ }
+ for c in &mut out {
+ c.text = format!("{}\n\n{}", title, c.text.trim());
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn long(s: &str) -> String {
+ format!("{} {}", s, "word ".repeat(60))
+ }
+
+ #[test]
+ fn formula_paragraph_joins_the_previous_one() {
+ let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.len(), 2, "{:?}", c);
+ assert!(c[0].text.contains("x^2"));
+ assert!(!c[1].text.contains("x^2"));
+ }
+
+ #[test]
+ fn formula_at_the_start_joins_the_next() {
+ let body = format!("$$a=b$$\n\n{}", long("Prose"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose"));
+ }
+
+ #[test]
+ fn prose_with_inline_math_is_not_a_formula() {
+ assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent."));
+ assert!(is_formula("\\[ \\int_0^1 f \\]"));
+ assert!(is_formula("$a$ and $b$"));
+ }
+
+ #[test]
+ fn source_blocks_stay_whole() {
+ let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two"));
+ let p = paragraphs(&body);
+ assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::<Vec<_>>());
+ assert!(p[1].text.contains("(b)"));
+ }
+
+ #[test]
+ fn environment_stays_whole() {
+ let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore";
+ let p = paragraphs(body);
+ assert_eq!(p.len(), 3);
+ assert!(p[1].text.contains("\\end{align}"));
+ }
+
+ #[test]
+ fn small_paragraphs_are_merged() {
+ let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third");
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.starts_with("T\n\n"));
+ }
+
+ #[test]
+ fn links_become_their_description() {
+ assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2");
+ }
+
+ #[test]
+ fn empty_body_uses_title_and_tags() {
+ let c = chunks("Title", &["x".into()], Some("sum"), "");
+ assert_eq!(c.len(), 1);
+ assert!(c[0].text.contains("sum") && c[0].text.contains("x"));
+ }
+
+ #[test]
+ fn line_numbers_point_at_the_paragraph() {
+ let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C"));
+ let c = chunks("T", &[], None, &body);
+ assert_eq!(c.iter().map(|x| x.line).collect::<Vec<_>>(), vec![0, 2, 4]);
+ }
+}