From 5b9a806af13d38163156302097e405a7b145b6cc Mon Sep 17 00:00:00 2001 From: JSDurand Date: Thu, 19 Nov 2026 09:29:47 +0800 Subject: elel: notes engine (Rust) and Gnus-like Emacs frontend Tantivy full text index, link graph, inline query filters, Ollama embeddings with paragraph chunks, and an Elisp summary/article UI. Co-Authored-By: Claude Sonnet 5.5 --- src/text.rs | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 src/text.rs (limited to 'src/text.rs') diff --git a/src/text.rs b/src/text.rs new file mode 100644 index 0000000..466e065 --- /dev/null +++ b/src/text.rs @@ -0,0 +1,188 @@ +//! Full-text index on top of Tantivy. +//! +//! Two views of the text are indexed: the normal one, split into words, +//! and a bigram one, so that Chinese (which has no spaces) can be +//! matched by substring. + +use crate::note::Note; +use anyhow::Result; +use std::path::Path; +use tantivy::collector::TopDocs; +use tantivy::directory::MmapDirectory; +use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; +use tantivy::schema::{ + Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value, +}; +use tantivy::snippet::SnippetGenerator; +use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer}; +use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc}; + +/// Marks around highlighted words in snippets. These are private-use +/// characters, so they cannot appear in normal text. +pub const HL_START: char = '\u{e000}'; +pub const HL_END: char = '\u{e001}'; + +pub struct Text { + index: Index, + id: Field, + path: Field, + title: Field, + body: Field, + tags: Field, + grams: Field, + /// True if the index directory was created by this call. + pub fresh: bool, +} + +pub struct Found { + pub id: String, + pub score: f32, + pub snippet: Option, +} + +fn has_cjk(s: &str) -> bool { + s.chars().any(|c| !c.is_ascii() && c.is_alphabetic()) +} + +impl Text { + pub fn open(dir: &Path) -> Result { + let fresh = !dir.join("meta.json").exists(); + std::fs::create_dir_all(dir)?; + let mut sb = Schema::builder(); + let id = sb.add_text_field("id", STRING | STORED); + let path = sb.add_text_field("path", STRING | STORED); + let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED); + let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED); + let tags = sb.add_text_field("tags", tantivy::schema::TEXT); + let grams = sb.add_text_field( + "grams", + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("bigram") + .set_index_option(IndexRecordOption::Basic), + ), + ); + let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?; + index.tokenizers().register( + "bigram", + TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?) + .filter(LowerCaser) + .build(), + ); + Ok(Text { + index, + id, + path, + title, + body, + tags, + grams, + fresh, + }) + } + + /// Remove the notes of `removed` files, and replace the notes of the + /// `changed` files. Only notes with an ID are indexed. + pub fn update(&self, removed: &[String], changed: &[(String, Vec)]) -> Result<()> { + let mut w: IndexWriter = self.index.writer(50_000_000)?; + for p in removed { + w.delete_term(Term::from_field_text(self.path, p)); + } + for (p, notes) in changed { + w.delete_term(Term::from_field_text(self.path, p)); + for n in notes { + let id = match &n.id { + Some(id) => id, + None => continue, + }; + let mut text = String::new(); + if let Some(s) = &n.summary { + text.push_str(s); + text.push('\n'); + } + text.push_str(&n.body); + let all = format!("{}\n{}", n.title, text); + w.add_document(doc!( + self.id => id.as_str(), + self.path => p.as_str(), + self.title => n.title.as_str(), + self.body => text, + self.tags => n.tags.join(" "), + self.grams => all, + ))?; + } + } + w.commit()?; + Ok(()) + } + + /// Build a query. Words go through the query parser (so phrases and + /// `title:foo` work); words with CJK characters become a conjunction + /// of bigrams. + fn build_query(&self, q: &str) -> Result> { + let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]); + parser.set_conjunction_by_default(); + if !has_cjk(q) { + return Ok(parser.parse_query(q)?); + } + let mut clauses: Vec<(Occur, Box)> = Vec::new(); + for word in q.split_whitespace() { + if has_cjk(word) { + let chars: Vec = word.to_lowercase().chars().collect(); + if chars.len() == 1 { + // A single character has no bigram; match any bigram with it. + let t = Term::from_field_text(self.grams, &chars[0].to_string()); + clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic)))); + } else { + for w in chars.windows(2) { + let g: String = w.iter().collect(); + let t = Term::from_field_text(self.grams, &g); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(t, IndexRecordOption::Basic)), + )); + } + } + } else { + clauses.push((Occur::Must, parser.parse_query(word)?)); + } + } + Ok(Box::new(BooleanQuery::new(clauses))) + } + + pub fn search(&self, q: &str, limit: usize) -> Result> { + let reader = self.index.reader()?; + let searcher = reader.searcher(); + let query = self.build_query(q)?; + let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; + let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok(); + let mut out = Vec::new(); + for (score, addr) in top { + let d: TantivyDocument = searcher.doc(addr)?; + let id = match d.get_first(self.id).and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => continue, + }; + let snippet = snipgen.as_ref().and_then(|g| { + let s = g.snippet_from_doc(&d); + let frag = s.fragment(); + if frag.is_empty() { + return None; + } + let mut out = String::new(); + let mut last = 0; + for r in s.highlighted() { + out.push_str(&frag[last..r.start]); + out.push(HL_START); + out.push_str(&frag[r.start..r.end]); + out.push(HL_END); + last = r.end; + } + out.push_str(&frag[last..]); + Some(out.replace('\n', " ")) + }); + out.push(Found { id, score, snippet }); + } + Ok(out) + } +} -- cgit v1.2.3-18-g5258