aboutsummaryrefslogtreecommitdiff
path: root/src/text.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/text.rs')
-rw-r--r--src/text.rs188
1 files changed, 188 insertions, 0 deletions
diff --git a/src/text.rs b/src/text.rs
new file mode 100644
index 0000000..466e065
--- /dev/null
+++ b/src/text.rs
@@ -0,0 +1,188 @@
+//! Full-text index on top of Tantivy.
+//!
+//! Two views of the text are indexed: the normal one, split into words,
+//! and a bigram one, so that Chinese (which has no spaces) can be
+//! matched by substring.
+
+use crate::note::Note;
+use anyhow::Result;
+use std::path::Path;
+use tantivy::collector::TopDocs;
+use tantivy::directory::MmapDirectory;
+use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
+use tantivy::schema::{
+ Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value,
+};
+use tantivy::snippet::SnippetGenerator;
+use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer};
+use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc};
+
+/// Marks around highlighted words in snippets. These are private-use
+/// characters, so they cannot appear in normal text.
+pub const HL_START: char = '\u{e000}';
+pub const HL_END: char = '\u{e001}';
+
+pub struct Text {
+ index: Index,
+ id: Field,
+ path: Field,
+ title: Field,
+ body: Field,
+ tags: Field,
+ grams: Field,
+ /// True if the index directory was created by this call.
+ pub fresh: bool,
+}
+
+pub struct Found {
+ pub id: String,
+ pub score: f32,
+ pub snippet: Option<String>,
+}
+
+fn has_cjk(s: &str) -> bool {
+ s.chars().any(|c| !c.is_ascii() && c.is_alphabetic())
+}
+
+impl Text {
+ pub fn open(dir: &Path) -> Result<Text> {
+ let fresh = !dir.join("meta.json").exists();
+ std::fs::create_dir_all(dir)?;
+ let mut sb = Schema::builder();
+ let id = sb.add_text_field("id", STRING | STORED);
+ let path = sb.add_text_field("path", STRING | STORED);
+ let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED);
+ let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED);
+ let tags = sb.add_text_field("tags", tantivy::schema::TEXT);
+ let grams = sb.add_text_field(
+ "grams",
+ TextOptions::default().set_indexing_options(
+ TextFieldIndexing::default()
+ .set_tokenizer("bigram")
+ .set_index_option(IndexRecordOption::Basic),
+ ),
+ );
+ let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?;
+ index.tokenizers().register(
+ "bigram",
+ TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?)
+ .filter(LowerCaser)
+ .build(),
+ );
+ Ok(Text {
+ index,
+ id,
+ path,
+ title,
+ body,
+ tags,
+ grams,
+ fresh,
+ })
+ }
+
+ /// Remove the notes of `removed` files, and replace the notes of the
+ /// `changed` files. Only notes with an ID are indexed.
+ pub fn update(&self, removed: &[String], changed: &[(String, Vec<Note>)]) -> Result<()> {
+ let mut w: IndexWriter = self.index.writer(50_000_000)?;
+ for p in removed {
+ w.delete_term(Term::from_field_text(self.path, p));
+ }
+ for (p, notes) in changed {
+ w.delete_term(Term::from_field_text(self.path, p));
+ for n in notes {
+ let id = match &n.id {
+ Some(id) => id,
+ None => continue,
+ };
+ let mut text = String::new();
+ if let Some(s) = &n.summary {
+ text.push_str(s);
+ text.push('\n');
+ }
+ text.push_str(&n.body);
+ let all = format!("{}\n{}", n.title, text);
+ w.add_document(doc!(
+ self.id => id.as_str(),
+ self.path => p.as_str(),
+ self.title => n.title.as_str(),
+ self.body => text,
+ self.tags => n.tags.join(" "),
+ self.grams => all,
+ ))?;
+ }
+ }
+ w.commit()?;
+ Ok(())
+ }
+
+ /// Build a query. Words go through the query parser (so phrases and
+ /// `title:foo` work); words with CJK characters become a conjunction
+ /// of bigrams.
+ fn build_query(&self, q: &str) -> Result<Box<dyn Query>> {
+ let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]);
+ parser.set_conjunction_by_default();
+ if !has_cjk(q) {
+ return Ok(parser.parse_query(q)?);
+ }
+ let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
+ for word in q.split_whitespace() {
+ if has_cjk(word) {
+ let chars: Vec<char> = word.to_lowercase().chars().collect();
+ if chars.len() == 1 {
+ // A single character has no bigram; match any bigram with it.
+ let t = Term::from_field_text(self.grams, &chars[0].to_string());
+ clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic))));
+ } else {
+ for w in chars.windows(2) {
+ let g: String = w.iter().collect();
+ let t = Term::from_field_text(self.grams, &g);
+ clauses.push((
+ Occur::Must,
+ Box::new(TermQuery::new(t, IndexRecordOption::Basic)),
+ ));
+ }
+ }
+ } else {
+ clauses.push((Occur::Must, parser.parse_query(word)?));
+ }
+ }
+ Ok(Box::new(BooleanQuery::new(clauses)))
+ }
+
+ pub fn search(&self, q: &str, limit: usize) -> Result<Vec<Found>> {
+ let reader = self.index.reader()?;
+ let searcher = reader.searcher();
+ let query = self.build_query(q)?;
+ let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?;
+ let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok();
+ let mut out = Vec::new();
+ for (score, addr) in top {
+ let d: TantivyDocument = searcher.doc(addr)?;
+ let id = match d.get_first(self.id).and_then(|v| v.as_str()) {
+ Some(s) => s.to_string(),
+ None => continue,
+ };
+ let snippet = snipgen.as_ref().and_then(|g| {
+ let s = g.snippet_from_doc(&d);
+ let frag = s.fragment();
+ if frag.is_empty() {
+ return None;
+ }
+ let mut out = String::new();
+ let mut last = 0;
+ for r in s.highlighted() {
+ out.push_str(&frag[last..r.start]);
+ out.push(HL_START);
+ out.push_str(&frag[r.start..r.end]);
+ out.push(HL_END);
+ last = r.end;
+ }
+ out.push_str(&frag[last..]);
+ Some(out.replace('\n', " "))
+ });
+ out.push(Found { id, score, snippet });
+ }
+ Ok(out)
+ }
+}