//! Full-text index on top of Tantivy. //! //! Two views of the text are indexed: the normal one, split into words, //! and a bigram one, so that Chinese (which has no spaces) can be //! matched by substring. use crate::note::Note; use anyhow::Result; use std::path::Path; use tantivy::collector::TopDocs; use tantivy::directory::MmapDirectory; use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; use tantivy::schema::{ Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value, }; use tantivy::snippet::SnippetGenerator; use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer}; use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc}; /// Marks around highlighted words in snippets. These are private-use /// characters, so they cannot appear in normal text. pub const HL_START: char = '\u{e000}'; pub const HL_END: char = '\u{e001}'; pub struct Text { index: Index, id: Field, path: Field, title: Field, body: Field, tags: Field, grams: Field, /// True if the index directory was created by this call. pub fresh: bool, } pub struct Found { pub id: String, pub score: f32, pub snippet: Option, } fn has_cjk(s: &str) -> bool { s.chars().any(|c| !c.is_ascii() && c.is_alphabetic()) } impl Text { pub fn open(dir: &Path) -> Result { let fresh = !dir.join("meta.json").exists(); std::fs::create_dir_all(dir)?; let mut sb = Schema::builder(); let id = sb.add_text_field("id", STRING | STORED); let path = sb.add_text_field("path", STRING | STORED); let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED); let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED); let tags = sb.add_text_field("tags", tantivy::schema::TEXT); let grams = sb.add_text_field( "grams", TextOptions::default().set_indexing_options( TextFieldIndexing::default() .set_tokenizer("bigram") .set_index_option(IndexRecordOption::Basic), ), ); let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?; index.tokenizers().register( "bigram", TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?) .filter(LowerCaser) .build(), ); Ok(Text { index, id, path, title, body, tags, grams, fresh, }) } /// Remove the notes of `removed` files, and replace the notes of the /// `changed` files. Only notes with an ID are indexed. pub fn update(&self, removed: &[String], changed: &[(String, Vec)]) -> Result<()> { let mut w: IndexWriter = self.index.writer(50_000_000)?; for p in removed { w.delete_term(Term::from_field_text(self.path, p)); } for (p, notes) in changed { w.delete_term(Term::from_field_text(self.path, p)); for n in notes { let id = match &n.id { Some(id) => id, None => continue, }; let mut text = String::new(); if let Some(s) = &n.summary { text.push_str(s); text.push('\n'); } text.push_str(&n.body); let all = format!("{}\n{}", n.title, text); w.add_document(doc!( self.id => id.as_str(), self.path => p.as_str(), self.title => n.title.as_str(), self.body => text, self.tags => n.tags.join(" "), self.grams => all, ))?; } } w.commit()?; Ok(()) } /// Build a query. Words go through the query parser (so phrases and /// `title:foo` work); words with CJK characters become a conjunction /// of bigrams. fn build_query(&self, q: &str) -> Result> { let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]); parser.set_conjunction_by_default(); if !has_cjk(q) { return Ok(parser.parse_query(q)?); } let mut clauses: Vec<(Occur, Box)> = Vec::new(); for word in q.split_whitespace() { if has_cjk(word) { let chars: Vec = word.to_lowercase().chars().collect(); if chars.len() == 1 { // A single character has no bigram; match any bigram with it. let t = Term::from_field_text(self.grams, &chars[0].to_string()); clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic)))); } else { for w in chars.windows(2) { let g: String = w.iter().collect(); let t = Term::from_field_text(self.grams, &g); clauses.push(( Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic)), )); } } } else { clauses.push((Occur::Must, parser.parse_query(word)?)); } } Ok(Box::new(BooleanQuery::new(clauses))) } pub fn search(&self, q: &str, limit: usize) -> Result> { let reader = self.index.reader()?; let searcher = reader.searcher(); let query = self.build_query(q)?; let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok(); let mut out = Vec::new(); for (score, addr) in top { let d: TantivyDocument = searcher.doc(addr)?; let id = match d.get_first(self.id).and_then(|v| v.as_str()) { Some(s) => s.to_string(), None => continue, }; let snippet = snipgen.as_ref().and_then(|g| { let s = g.snippet_from_doc(&d); let frag = s.fragment(); if frag.is_empty() { return None; } let mut out = String::new(); let mut last = 0; for r in s.highlighted() { out.push_str(&frag[last..r.start]); out.push(HL_START); out.push_str(&frag[r.start..r.end]); out.push(HL_END); last = r.end; } out.push_str(&frag[last..]); Some(out.replace('\n', " ")) }); out.push(Found { id, score, snippet }); } Ok(out) } }