//! Cut a note into chunks for the embeddings. //! //! A chunk is a paragraph or a few paragraphs. Blocks that must stay //! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A //! paragraph that is only a formula is merged into a neighbour, because //! alone it would be embedded as "some mathematics" and look like every //! other formula. Each chunk is prefixed with the title, so a bare //! "this follows from the lemma" still knows what it is about. /// Paragraphs shorter than this are merged with a neighbour. const MIN: usize = 200; /// A chunk is not grown beyond this (a single paragraph may exceed it). const MAX: usize = 1500; /// A paragraph with fewer words than this, once the math is removed, /// is a formula paragraph. const MIN_WORDS: usize = 5; #[derive(Debug, Clone, PartialEq)] pub struct Chunk { pub text: String, /// 0-based line of the body where the chunk starts. pub line: usize, } /// A paragraph of the body: its text and its first line. struct Para { text: String, line: usize, } /// Replace `[[note:ID][desc]]` by `desc` and org links by their text. pub fn strip_links(s: &str) -> String { let mut out = String::with_capacity(s.len()); let mut rest = s; while let Some(i) = rest.find("[[") { out.push_str(&rest[..i]); let tail = &rest[i + 2..]; match tail.find("]]") { Some(j) => { let inner = &tail[..j]; match inner.split_once("][") { Some((_, desc)) => out.push_str(desc), None => out.push_str(inner), } rest = &tail[j + 2..]; } None => { out.push_str(&rest[i..]); rest = ""; } } } out.push_str(rest); out } /// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed. fn without_math(s: &str) -> String { let mut out = String::new(); let mut rest = s; 'outer: while !rest.is_empty() { for (open, close) in [ ("$$", "$$"), ("\\[", "\\]"), ("\\(", "\\)"), ("$", "$"), ] { if let Some(r) = rest.strip_prefix(open) { match r.find(close) { Some(j) => rest = &r[j + close.len()..], None => rest = "", } out.push(' '); continue 'outer; } } if rest.starts_with("\\begin{") { match rest.find("\\end{") { Some(j) => { let after = &rest[j..]; rest = match after.find('}') { Some(k) => &after[k + 1..], None => "", }; out.push(' '); continue 'outer; } None => break, } } let c = rest.chars().next().unwrap(); out.push(c); rest = &rest[c.len_utf8()..]; } out } /// Does the paragraph hold almost nothing but math? fn is_formula(p: &str) -> bool { let t = p.trim(); if t.is_empty() { return false; } let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{"); if !has_math { return false; } let left = without_math(t); // CJK has no spaces: count its characters as words. let words = left .split(|c: char| !c.is_alphanumeric()) .filter(|w| !w.is_empty()) .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 }) .sum::(); words < MIN_WORDS } /// Split into paragraphs, keeping blocks whole. fn paragraphs(body: &str) -> Vec { let mut out: Vec = Vec::new(); let mut cur: Vec<&str> = Vec::new(); let mut start = 0; // Inside a block that ends with this line (lowercased, trimmed prefix). let mut block_end: Option = None; let mut dollars = false; let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec| { if !cur.is_empty() { out.push(Para { text: cur.join("\n"), line: start }); cur.clear(); } }; for (i, l) in body.lines().enumerate() { let t = l.trim(); let tl = t.to_lowercase(); if let Some(end) = &block_end { cur.push(l); if tl.starts_with(end.as_str()) { block_end = None; } continue; } if dollars { cur.push(l); if t.contains("$$") { dollars = false; } continue; } if t.is_empty() { flush(&mut cur, start, &mut out); continue; } let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' '); if heading { flush(&mut cur, start, &mut out); } if cur.is_empty() { start = i; } if let Some(rest) = tl.strip_prefix("#+begin_") { let name = rest.split_whitespace().next().unwrap_or(""); block_end = Some(format!("#+end_{}", name)); } else if let Some(rest) = t.strip_prefix("\\begin{") { let name = rest.split('}').next().unwrap_or(""); block_end = Some(format!("\\end{{{}}}", name.to_lowercase())); } else if t.matches("$$").count() % 2 == 1 { dollars = true; } // A one-line block (\begin{x} .. \end{x}) closes at once. if let Some(end) = &block_end { if tl[1..].contains(end.as_str()) { block_end = None; } } cur.push(l); } flush(&mut cur, start, &mut out); out } /// The chunks of a note. A note without a body gets one chunk from its /// title, tags and summary. pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec { let body = strip_links(body); let paras = paragraphs(&body); // Merge formula paragraphs into the previous paragraph (or the next one). let mut merged: Vec = Vec::new(); let mut carry: Option = None; for p in paras { let formula = is_formula(&p.text); let p = match carry.take() { Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line }, None => p, }; if formula && merged.is_empty() { // Opens the note: it joins the next paragraph. carry = Some(p); } else if formula { let last = merged.last_mut().unwrap(); last.text = format!("{}\n{}", last.text, p.text); } else { merged.push(p); } } if let Some(c) = carry { merged.push(c); } // Merge small paragraphs, up to MAX. let mut out: Vec = Vec::new(); let mut small = false; for p in merged { match out.last_mut() { Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => { last.text.push_str("\n\n"); last.text.push_str(&p.text); small = last.text.len() < MIN; } _ => { small = p.text.len() < MIN; out.push(Chunk { text: p.text, line: p.line }); } } } if out.is_empty() { let mut t = String::new(); if let Some(s) = summary { t.push_str(s); } if !tags.is_empty() { t.push_str("\n"); t.push_str(&tags.join(" ")); } out.push(Chunk { text: t, line: 0 }); } for c in &mut out { c.text = format!("{}\n\n{}", title, c.text.trim()); } out } #[cfg(test)] mod tests { use super::*; fn long(s: &str) -> String { format!("{} {}", s, "word ".repeat(60)) } #[test] fn formula_paragraph_joins_the_previous_one() { let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second")); let c = chunks("T", &[], None, &body); assert_eq!(c.len(), 2, "{:?}", c); assert!(c[0].text.contains("x^2")); assert!(!c[1].text.contains("x^2")); } #[test] fn formula_at_the_start_joins_the_next() { let body = format!("$$a=b$$\n\n{}", long("Prose")); let c = chunks("T", &[], None, &body); assert_eq!(c.len(), 1); assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose")); } #[test] fn prose_with_inline_math_is_not_a_formula() { assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent.")); assert!(is_formula("\\[ \\int_0^1 f \\]")); assert!(is_formula("$a$ and $b$")); } #[test] fn source_blocks_stay_whole() { let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two")); let p = paragraphs(&body); assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::>()); assert!(p[1].text.contains("(b)")); } #[test] fn environment_stays_whole() { let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore"; let p = paragraphs(body); assert_eq!(p.len(), 3); assert!(p[1].text.contains("\\end{align}")); } #[test] fn small_paragraphs_are_merged() { let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third"); assert_eq!(c.len(), 1); assert!(c[0].text.starts_with("T\n\n")); } #[test] fn links_become_their_description() { assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2"); } #[test] fn empty_body_uses_title_and_tags() { let c = chunks("Title", &["x".into()], Some("sum"), ""); assert_eq!(c.len(), 1); assert!(c[0].text.contains("sum") && c[0].text.contains("x")); } #[test] fn line_numbers_point_at_the_paragraph() { let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C")); let c = chunks("T", &[], None, &body); assert_eq!(c.iter().map(|x| x.line).collect::>(), vec![0, 2, 4]); } }