aboutsummaryrefslogtreecommitdiff
path: root/src/text.rs
blob: 466e065544ab9e8b0b183548b3d607cbd99478d5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
//! Full-text index on top of Tantivy.
//!
//! Two views of the text are indexed: the normal one, split into words,
//! and a bigram one, so that Chinese (which has no spaces) can be
//! matched by substring.

use crate::note::Note;
use anyhow::Result;
use std::path::Path;
use tantivy::collector::TopDocs;
use tantivy::directory::MmapDirectory;
use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery};
use tantivy::schema::{
    Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value,
};
use tantivy::snippet::SnippetGenerator;
use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer};
use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc};

/// Marks around highlighted words in snippets.  These are private-use
/// characters, so they cannot appear in normal text.
pub const HL_START: char = '\u{e000}';
pub const HL_END: char = '\u{e001}';

pub struct Text {
    index: Index,
    id: Field,
    path: Field,
    title: Field,
    body: Field,
    tags: Field,
    grams: Field,
    /// True if the index directory was created by this call.
    pub fresh: bool,
}

pub struct Found {
    pub id: String,
    pub score: f32,
    pub snippet: Option<String>,
}

fn has_cjk(s: &str) -> bool {
    s.chars().any(|c| !c.is_ascii() && c.is_alphabetic())
}

impl Text {
    pub fn open(dir: &Path) -> Result<Text> {
        let fresh = !dir.join("meta.json").exists();
        std::fs::create_dir_all(dir)?;
        let mut sb = Schema::builder();
        let id = sb.add_text_field("id", STRING | STORED);
        let path = sb.add_text_field("path", STRING | STORED);
        let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED);
        let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED);
        let tags = sb.add_text_field("tags", tantivy::schema::TEXT);
        let grams = sb.add_text_field(
            "grams",
            TextOptions::default().set_indexing_options(
                TextFieldIndexing::default()
                    .set_tokenizer("bigram")
                    .set_index_option(IndexRecordOption::Basic),
            ),
        );
        let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?;
        index.tokenizers().register(
            "bigram",
            TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?)
                .filter(LowerCaser)
                .build(),
        );
        Ok(Text {
            index,
            id,
            path,
            title,
            body,
            tags,
            grams,
            fresh,
        })
    }

    /// Remove the notes of `removed` files, and replace the notes of the
    /// `changed` files.  Only notes with an ID are indexed.
    pub fn update(&self, removed: &[String], changed: &[(String, Vec<Note>)]) -> Result<()> {
        let mut w: IndexWriter = self.index.writer(50_000_000)?;
        for p in removed {
            w.delete_term(Term::from_field_text(self.path, p));
        }
        for (p, notes) in changed {
            w.delete_term(Term::from_field_text(self.path, p));
            for n in notes {
                let id = match &n.id {
                    Some(id) => id,
                    None => continue,
                };
                let mut text = String::new();
                if let Some(s) = &n.summary {
                    text.push_str(s);
                    text.push('\n');
                }
                text.push_str(&n.body);
                let all = format!("{}\n{}", n.title, text);
                w.add_document(doc!(
                    self.id => id.as_str(),
                    self.path => p.as_str(),
                    self.title => n.title.as_str(),
                    self.body => text,
                    self.tags => n.tags.join(" "),
                    self.grams => all,
                ))?;
            }
        }
        w.commit()?;
        Ok(())
    }

    /// Build a query.  Words go through the query parser (so phrases and
    /// `title:foo` work); words with CJK characters become a conjunction
    /// of bigrams.
    fn build_query(&self, q: &str) -> Result<Box<dyn Query>> {
        let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]);
        parser.set_conjunction_by_default();
        if !has_cjk(q) {
            return Ok(parser.parse_query(q)?);
        }
        let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new();
        for word in q.split_whitespace() {
            if has_cjk(word) {
                let chars: Vec<char> = word.to_lowercase().chars().collect();
                if chars.len() == 1 {
                    // A single character has no bigram; match any bigram with it.
                    let t = Term::from_field_text(self.grams, &chars[0].to_string());
                    clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic))));
                } else {
                    for w in chars.windows(2) {
                        let g: String = w.iter().collect();
                        let t = Term::from_field_text(self.grams, &g);
                        clauses.push((
                            Occur::Must,
                            Box::new(TermQuery::new(t, IndexRecordOption::Basic)),
                        ));
                    }
                }
            } else {
                clauses.push((Occur::Must, parser.parse_query(word)?));
            }
        }
        Ok(Box::new(BooleanQuery::new(clauses)))
    }

    pub fn search(&self, q: &str, limit: usize) -> Result<Vec<Found>> {
        let reader = self.index.reader()?;
        let searcher = reader.searcher();
        let query = self.build_query(q)?;
        let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?;
        let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok();
        let mut out = Vec::new();
        for (score, addr) in top {
            let d: TantivyDocument = searcher.doc(addr)?;
            let id = match d.get_first(self.id).and_then(|v| v.as_str()) {
                Some(s) => s.to_string(),
                None => continue,
            };
            let snippet = snipgen.as_ref().and_then(|g| {
                let s = g.snippet_from_doc(&d);
                let frag = s.fragment();
                if frag.is_empty() {
                    return None;
                }
                let mut out = String::new();
                let mut last = 0;
                for r in s.highlighted() {
                    out.push_str(&frag[last..r.start]);
                    out.push(HL_START);
                    out.push_str(&frag[r.start..r.end]);
                    out.push(HL_END);
                    last = r.end;
                }
                out.push_str(&frag[last..]);
                Some(out.replace('\n', " "))
            });
            out.push(Found { id, score, snippet });
        }
        Ok(out)
    }
}