diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Cargo.lock | 1965 | ||||
| -rw-r--r-- | Cargo.toml | 14 | ||||
| -rw-r--r-- | README.org | 170 | ||||
| -rw-r--r-- | elel-import.el | 508 | ||||
| -rw-r--r-- | elel.el | 1816 | ||||
| -rw-r--r-- | src/adapter/mod.rs | 275 | ||||
| -rw-r--r-- | src/adapter/org.rs | 84 | ||||
| -rw-r--r-- | src/adapter/txt.rs | 85 | ||||
| -rw-r--r-- | src/chunk.rs | 314 | ||||
| -rw-r--r-- | src/graph.rs | 187 | ||||
| -rw-r--r-- | src/main.rs | 588 | ||||
| -rw-r--r-- | src/note.rs | 36 | ||||
| -rw-r--r-- | src/query.rs | 335 | ||||
| -rw-r--r-- | src/text.rs | 188 | ||||
| -rw-r--r-- | src/vec.rs | 651 |
16 files changed, 7218 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a11a018 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target +*.elc diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..86320d4 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1965 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "bon" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "bumpalo" +version = "3.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a0cf66fd1eb2fb7b822cb78370d82303750809bc6ba855f3f58e517d3238db" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac230478587d0269d8d6998b82ae69ea51509d55df66d9e92edd1f986904cd4a" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "895c9d8d2ac2dd8b8a29f3724a14e72aaaa6debcf8c38c82f97305c6a172767a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "clap_lex" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0921b3876b3a1df8a1c9f912e7f94342458022c2e84288015893d83c4d62a79" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.10", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "datasketches" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c0bd7d22989969a619bae09147b992a6aa7f31dd4f9d9af6345f533744781f" + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "elel" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake3", + "clap", + "half", + "serde", + "serde_json", + "tantivy", + "ureq", +] + +[[package]] +name = "erased-serde" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "frostem" +version = "1.20260821.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2488a10218a17366ce6bd38b6d081b4387a9548d350029db9feddc9bb44d6d" + +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-macro", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "lz4_flex" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56455fff36934a9500d69062aec1c77037179b6f0669e4d5d6b72b84722d6a3" + +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0ca1155a9ecc25aa12e8313a119f32edf0255fea2f44839c4ac0a12b2ac157" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "murmurhash32" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8afc6df942f4c022d70c5725e18df1a705773870e5b7f96fde94ca0334ce77a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "ordered-float" +version = "5.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f59006ce1e204c30956638535b74d317bca99477b9356dfaec54c37f6f600ab" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.10", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-tuple-vec-map" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a04d0ebe0de77d7d445bb729a895dcb0a288854b267ca85f030ce51cdc578c82" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "sketches-ddsketch" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e53e2f8d4e9c692f23a90208fad6a64862f413e286d533f1f649f160e2d5caee" +dependencies = [ + "serde", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "tantivy" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1ebacf6bc124b741ffbfb088f9835cb55e1e2ff57e182e9219e3efa1377b97" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64 0.23.1", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "datasketches", + "downcast-rs", + "fastdivide", + "fnv", + "frostem", + "fs4", + "htmlescape", + "itertools", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rustc-hash", + "serde", + "serde-tuple-vec-map", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror", + "time", + "typetag", + "unwrap-infallible", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c803ac72b3356f48fe4a399a03e058addd5c94e793e0bb7c6bcc224b238fcd7" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95101fe3dd4a76156bdeff8d14ad41c547f2c1bce7ee0d23fdb6f95599f312ec" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools", + "rand", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d062ce97f2a26f2a7b2ed5d926a62241f762e4395ac110eb5c18f3b4a13b71" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef6729a14fc5c5a662ee549a176640743577ef35c70ae7e86f0ca07fa83d161" +dependencies = [ + "fnv", + "nom", + "ordered-float", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6b3567417750e8aeb6e7c9c5ae8701376c07c06d51991050825832c2f8c431" +dependencies = [ + "futures-util", + "itertools", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4213e34acf53de44ab4778aafec3c6800a781be6a7c1906579acdc23c114d94" +dependencies = [ + "murmurhash32", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac258c2c6390673f2685813afeeafcb8c4e0ee7de8dd3fc46838dcc37263f98" +dependencies = [ + "serde", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "unwrap-infallible" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e497bb1f828cc9fb236722c2eaa100dcf201563f38f4da6252357a59037adf31" + +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "url", + "webpki-roots 0.26.11", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "uuid" +version = "1.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c14b15dffa2f4bd712103e857140aa04a4f2ff6e955c77d7eb8e53ee02a3b64" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.10", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.12", +] + +[[package]] +name = "webpki-roots" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7265c9301c16a8711c992c2d8da90ee77ef54b1ed51482d6dfeb00dc756b1297" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ccff8a4805aba1eaa4cd4afcd49f7557dd2cf5fd79ffc37f8959466effc5aa" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5020ab8f27bfc0024ad0a07aed6bf022a16208f479686fbed4fdd0c85eb0b3" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9ea9cc2576c2fcc104e9cec04f46e3ede564f4fbc206831e4ef7e859a4140d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.3+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f729a06d0257428fd9d997ac8db7e1407bd0262ab016b085b36bc52957c0052d" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..875140e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "elel" +version = "0.1.0" +edition = "2024" + +[dependencies] +anyhow = "1.0.104" +blake3 = "1.8.7" +clap = { version = "4.6.8", features = ["derive", "env"] } +half = "2.7.1" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.154" +tantivy = "0.27.0" +ureq = { version = "2", features = ["json"] } diff --git a/README.org b/README.org new file mode 100644 index 0000000..ae298ed --- /dev/null +++ b/README.org @@ -0,0 +1,170 @@ +#+TITLE: elel + +Emacs Lexical Embedding Link: a notes system whose essence is +searching. Notes are found by their words (index), their meaning +(semantics, second pass) and their relations (structure). + +* Layout +- =src/= :: the engine, a Rust CLI that only /reads/ note files. +- =elel.el= :: the Emacs front end, with a Gnus-like interface. + +* Files and notes +A file is a container of one or more notes. Each note carries its own +ID. Formats: +- =.org= :: each top-level heading is a note; metadata in the property + drawer (=:ID: :DATE: :KIND: :PARENT: :LINKS: :CONTEXT: :SUMMARY:=), + tags in the heading. +- =.txt= :: notes are separated by a form-feed line; each starts with + =Key: value= lines (=ID Title Date Kind Tags Parent Links Context + Summary=), a blank line, then the body. +A link in a body is written =[[note:ID][TITLE]]= in every format, and +=[[note::KIND:ID][TITLE]]= when it has a type (=KIND= is letters, digits, +=-= or =_=). The title is for the reader; the ID is how the engine finds +the note. An ID has no whitespace; a uuid-style ID is fine as it is, +e.g. =[[note:<a.b@c.local>][T]]=. Typed links can also go in the =LINKS= +property, as =supports:ID=. In org, =note= is a registered link type, so +org follows (=C-c C-o=), completes (=C-c C-l=, which offers the title as +the description) and exports these links itself. + +* Searching +One query string. Filters are words of the form =key:value=; they are +always ANDed. The remaining words go to the full text index (ANDed; +="a phrase"=, =-word=, =a OR b= and =title:foo= work; Chinese is +matched by character pairs). +| Filter | Meaning | +|-------------------------+-------------------------------------------------| +| =tag:X= =-tag:X= | has / lacks the tag X | +| =tag:*= =-tag:*= | has some tag / has no tag | +| =tag:alg*= | has a tag that starts with =alg= | +| =kind:X= =kind:*= | the KIND is X / there is a KIND (=-kind:= too) | +| =file:X= =-file:X= | the path (relative to the root) contains X | +| =from:ID= | the notes that ID links to | +| =to:ID= | the notes that link to ID | +| =parent:ID= | the children of ID | +| =thread:ID= | the whole thread of ID | +| =date:2025= | the DATE starts with 2025 | +| =date:2025-03..2025-06= | a range; =date:..2024= and =date:2025..= work | +| =is:inbox= | no links, parent, children or tags | +| =like:X= =-like:X= | by meaning: X is a note ID or a text (see Embeddings) | +Quote a value with spaces: =file:"my notes"=. The empty query lists +every note. Without words, results are newest first; with words, by +score. + +* Engine +#+begin_src sh +cargo build --release +elel index [--no-embed] # scan ELEL_ROOT (default ~/notes), update the index and the embeddings +elel search [-n N] [--hybrid] -- QUERY +elel count -- QUERY... # the number of notes for each query +elel hood ID [--hops N] +elel get ID # the note, with its exact text in "raw" +elel check # missing/duplicate IDs, dangling links +elel tags [--like ID] # with --like: the tags that fit the note, best first +#+end_src +The index lives in ELEL_INDEX (default =~/.local/share/elel=) and can be +deleted at any time. + +* Emacs +#+begin_src emacs-lisp +(add-to-list 'load-path "~/.emacs.d/claude/elel") +(autoload 'elel "elel" nil t) +(setq elel-root "~/notes") +#+end_src +The interface follows Gnus. =M-x elel= opens the views buffer (the +group buffer); =RET= opens a view in the summary buffer, with the +article below it. =elel-views= is a list of =(NAME QUERY . PLIST)=; +=:limit N= caps a view and =:hidden t= keeps it out of sight until =L=. +Views for a tag or a file are just more entries, e.g. +=("Math" "tag:math")=. + +*Views buffer*: =RET= open, =L= list hidden views, =G= or =/= search, +=u= update the index, =g= redraw, =N= new note, =C= check. + +*Summary buffer*: the notes are threaded by PARENT, drawn as in Gnus +(see =elel-thread-*=). =RET= shows the note, =SPC= / =DEL= scroll it, +=n= / =p= next / previous note, =g= show the note again (as in Gnus; to +refresh the list, search again with =G=), =/= narrow the query, =s= +snippets, ==== make the summary fill the frame (=C-u ==== restores both), +=A= select the article window, =#= / =-= mark liked / disliked, =R= related, =S= suggestions, =a= add the whole thread of the note, =^= parent, =c= children, +=o= outgoing links, =b= backlinks, =H= neighbourhood, =e= edit, =r= / =N= +child / new note (finish with =C-c C-c=), =q= leave. + +*Article buffer*: the note as it is in its file, in the major mode that +=elel-major-mode-alist= picks (org-mode for =.org=), with the minor mode +=elel-article-mode= on top. =SPC= / =j= scroll (at the end, go to the next +note), =DEL= / =k= back, =n= / =p= next / previous note of the summary, +=TAB= / =S-TAB= and =RET= move to and follow links, =l= back, =^= =c= =o= +=b= =H= =a= as in the summary, =h= go to the summary, =q= close the +article, =e= edit, =r= child note, =g= reread. You may close the +summary window; =n= and =p= still work. =elel-summary-window-height= +and =elel-article-keep-summary= control the windows. + +=M-x elel-assign-ids= adds missing IDs. + +In the note files (=elel-link-mode=, on automatically under +=elel-root=): =C-c n l= inserts a link by picking a title (an active +region becomes the description), =C-c C-o= follows the link at point +(org's own behaviour elsewhere), =C-c n o= opens the note around +point. + +* Importing old notes +=elel-import.el= (Emacs Lisp; it only reads the sources and never +overwrites). Without an output directory, or with =dry-run=, it only +reports. +#+begin_src sh +# notes kept as Gnus drafts (Gcc nnmaildir+notes:GROUP): one GROUP.org +# per group, Message-IDs as IDs, In-Reply-To as PARENT +emacs --batch -L ~/.emacs.d/claude/elel -l elel-import \ + --eval '(elel-import-drafts "~/News/drafts/drafts" "~/notes")' +# an org-roam v1 directory: notes are collected into few files, each named +# by a tag chosen to cover many notes (see elel-import-roam-min-group) +emacs --batch -L ~/.emacs.d/claude/elel -l elel-import \ + --eval '(elel-import-roam "~/org/roam" "~/notes")' +#+end_src +Both are also commands (=M-x elel-import-drafts=, =M-x elel-import-roam=) +that ask before writing. + +* Embeddings +The "E" of elel. Install [[https://ollama.com][Ollama]] and run +=ollama pull qwen3-embedding:0.6b=. Then =elel index= also embeds the +notes (the first run takes a few minutes; later runs only embed what +changed). If Ollama is not running, indexing still works; the notes +stay pending and the next run catches up. =--no-embed= skips it. + +- *Units*: a note is cut into paragraph chunks (=src/chunk.rs=). A + paragraph that is only a formula joins its neighbour, small paragraphs + are merged, source blocks and =\begin..\end= stay whole, and the title is + put in front of each chunk. A note's own vector is the mean of its + chunks. +- *Storage*: =/vec/= holds =f16.bin= (exact vectors), + =bits.bin= (one sign bit per dimension, for a first Hamming pass when + there are more than 2000 chunks) and =meta.json=. All of it can be + rebuilt. Changing the model (=ELEL_MODEL=, default + =qwen3-embedding:0.6b=) re-embeds everything. =ELEL_OLLAMA= is the + server URL. +- *=like:X=*: notes close in meaning. X is a note ID (the whole note + is compared) or a text such as =like:"moduli of curves"= (each note + scores as its best chunk, which is shown as the snippet). Several + =like:= are merged by their mean; =-like:Y= pushes away from Y. + Other filters still apply: =like:ID tag:algebra date:2024..=. +- *Hybrid*: =search --hybrid= with words fuses the text ranking and the + meaning ranking by reciprocal rank fusion. In Emacs + =elel-hybrid-search= (default t) does this for every search; without + Ollama it falls back to the text search. +- *Tags as anchors*: =elel tags --like ID= scores each tag (the mean of + the other notes that carry it) against the note, to suggest tags. + +In the summary buffer, =#= marks a note as liked, =-= as disliked, and +=R= shows the notes closest to the marked ones (or to the note at point). +In the article, =R= shows the notes like this one. =S= (=elel-suggest=) +lists the notes that the note could link to (the ones it is already +related to are left out), with the best tags in the header; =i= there +inserts =[[note:ID][TITLE]]= where the suggestion was asked for. After a +new note is saved, the suggestions appear by themselves +(=elel-suggest-after-capture=), and =i= inserts at the point where you +typed =C-c C-c=. =elel-insert-link= lists the notes closest to the +current note first. + +* Not yet done +Anchors written as free phrases (instead of tags), a Gnus link +adapter, an optional daemon for faster calls. diff --git a/elel-import.el b/elel-import.el new file mode 100644 index 0000000..0dd2b9c --- /dev/null +++ b/elel-import.el @@ -0,0 +1,508 @@ +;;; elel-import.el --- Import old notes into elel -*- lexical-binding: t; -*- + +;; Copyright (C) 2026 Durand + +;; Author: Durand +;; Keywords: outlines, files + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Two importers, which only read their sources: +;; +;; `elel-import-drafts' -- notes kept as Gnus drafts, i.e. messages +;; whose Gcc is a nnmaildir+notes:GROUP group. Each GROUP becomes +;; one container file GROUP.org, and the Message-IDs become the +;; IDs, so the In-Reply-To threads become PARENT properties. +;; +;; `elel-import-roam' -- an org-roam (version 1, file based) directory. +;; Each file becomes one note, and the links to other roam files +;; become [[note:ID][TITLE]] links. +;; +;; Both take the source, the output directory, and an optional DRY-RUN. +;; With a nil output directory, they only report what they would do. +;; They never overwrite: if a target file exists, nothing is written. +;; +;; From a shell: +;; +;; emacs --batch -L DIR -l elel-import \ +;; --eval '(elel-import-drafts "~/News/drafts/drafts" "~/notes")' + +;;; Code: + +(require 'cl-lib) +(require 'subr-x) +(require 'rfc2047) +(require 'qp) +(require 'elel) + +;;; Common helpers + +(defun elel-import--read (path) + "The content of the file PATH, decoded as UTF-8." + (with-temp-buffer + (let ((coding-system-for-read 'utf-8)) + (insert-file-contents path)) + (buffer-string))) + +(defun elel-import--org-note (title tags props body) + "The text of a note in org format. +PROPS is an alist of (NAME . VALUE) for the property drawer." + (concat "* " title + (and tags (concat " :" (mapconcat #'identity tags ":") ":")) + "\n:PROPERTIES:\n" + (mapconcat (lambda (p) (format ":%s: %s\n" (car p) (cdr p))) + props "") + ":END:\n" + body "\n")) + +(defun elel-import--demote (body) + "Demote every heading in BODY by one level. +Return (NEW-BODY . NUMBER-OF-HEADINGS)." + (let ((count 0) + (pos 0)) + (while (string-match "^\\*+ " body pos) + (setq count (1+ count)) + (setq pos (match-end 0))) + (cons (replace-regexp-in-string "^\\(\\*+\\) " "*\\1 " body) + count))) + +(defun elel-import--finish (result out dry-run) + "Report RESULT, and write its files to OUT unless DRY-RUN or no OUT." + (let ((files (plist-get result :files))) + (message "%s" (mapconcat #'identity (plist-get result :report) "\n")) + (cond + ((or dry-run (null out)) + (message "elel import: dry run, nothing written.")) + (t + (let* ((out (expand-file-name out)) + (targets (mapcar (lambda (f) (expand-file-name (car f) out)) + files)) + (clash (cl-find-if #'file-exists-p targets))) + (cond + (clash + (user-error "elel import: %s already exists; nothing written" + clash)) + ((and (not noninteractive) + (not (y-or-n-p (format "Write %d files to %s? " + (length files) out)))) + (user-error "elel import: cancelled")) + (t + (cl-mapc + (lambda (f target) + (make-directory (file-name-directory target) t) + (let ((coding-system-for-write 'utf-8-unix)) + (write-region (cdr f) nil target nil 'silent))) + files targets) + (message "elel import: wrote %d files to %s" + (length files) out)))))))) + +;;; Gnus drafts + +(defun elel-import--header (headers name) + "The value of the header NAME in the string HEADERS, or nil." + (let ((case-fold-search t) + (flat (replace-regexp-in-string "\n[ \t]+" " " headers))) + (and (string-match + (concat "^" (regexp-quote name) ":[ \t]*\\(.*\\)$") flat) + (string-trim (match-string 1 flat))))) + +(defun elel-import--decode-body (body headers) + "Decode BODY according to the transfer encoding and charset in HEADERS." + (let* ((cte (downcase (or (elel-import--header + headers "Content-Transfer-Encoding") + ""))) + (type (or (elel-import--header headers "Content-Type") "")) + (charset (cond + ((string-match "charset=\"?\\([^\";[:space:]]+\\)" type) + (match-string 1 type)) + ("utf-8"))) + (coding (or (ignore-errors (coding-system-from-name charset)) + 'utf-8))) + (cond + ((string= cte "quoted-printable") + (decode-coding-string (quoted-printable-decode-string body) coding)) + ((string= cte "base64") + (decode-coding-string (base64-decode-string body) coding)) + (body)))) + +(defun elel-import--strip-signature (body) + "Remove a short signature (\"-- \" and up to three lines) from BODY." + (let ((pos nil) + (start 0)) + (while (string-match "\n-- \n" body start) + (setq pos (match-beginning 0)) + (setq start (match-end 0))) + (cond + ((and pos (<= (cl-count ?\n (substring body pos)) 4)) + (substring body 0 pos)) + (body)))) + +(defun elel-import--draft (path) + "Parse the draft file PATH. Return a plist, or nil if it is no note." + (let* ((text (elel-import--read path)) + (number (file-name-nondirectory path))) + (cond + ((not (string-match "^--text follows this line--\n" text)) + (list :skip (format "draft %s: no header separator" number))) + (t + (let* ((headers (substring text 0 (match-beginning 0))) + (body (substring text (match-end 0))) + (gcc (or (elel-import--header headers "Gcc") "")) + (subject (or (elel-import--header headers "Subject") "")) + (irt (elel-import--header headers "In-Reply-To")) + (mtime (file-attribute-modification-time + (file-attributes path)))) + (cond + ((not (string-match "\\`nnmaildir\\+notes:\\(.+\\)\\'" gcc)) + (list :skip (format "draft %s (%s): Gcc is %s, not a notes group" + number (rfc2047-decode-string subject) + (if (string-empty-p gcc) "empty" gcc)))) + (t + (list + :group (match-string 1 gcc) + :number number + :title (rfc2047-decode-string subject) + :id (or (elel-import--header headers "Message-ID") + (funcall elel-id-function)) + :parent (and irt (string-match "<[^>]+>" irt) + (match-string 0 irt)) + :time mtime + :body (string-trim + (elel-import--strip-signature + (string-trim + (elel-import--decode-body body headers)))))))))))) + +;;;###autoload +(defun elel-import-drafts (src out &optional dry-run) + "Import the notes kept in the Gnus draft directory SRC into OUT. +Only drafts whose Gcc is a nnmaildir+notes:GROUP group are notes; +each GROUP becomes the container file GROUP.org, and a tag. With a +nil OUT, or a non-nil DRY-RUN, only report." + (interactive + (list (read-directory-name "Draft directory: " "~/News/drafts/drafts/") + (read-directory-name "Write to: " elel-root) + current-prefix-arg)) + (let* ((paths (sort (directory-files (expand-file-name src) t "\\`[0-9]+\\'") + (lambda (a b) + (< (string-to-number (file-name-nondirectory a)) + (string-to-number (file-name-nondirectory b)))))) + (parsed (mapcar #'elel-import--draft paths)) + (notes (cl-remove-if (lambda (n) (plist-get n :skip)) parsed)) + (ids (mapcar (lambda (n) (plist-get n :id)) notes)) + (report (list (format "source : %s" (expand-file-name src)) + (format "drafts : %d, notes: %d" + (length paths) (length notes)))) + (groups nil) + (files nil) + (demoted 0)) + (dolist (n parsed) + (cond + ((plist-get n :skip) + (push (concat " skipped " (plist-get n :skip)) report)))) + (dolist (n notes) + (let ((cell (assoc (plist-get n :group) groups))) + (cond + (cell (setcdr cell (cons n (cdr cell)))) + ((push (list (plist-get n :group) n) groups))))) + (dolist (g (reverse groups)) + (let* ((group (car g)) + (sorted (sort (cdr g) + (lambda (a b) + (time-less-p (plist-get a :time) + (plist-get b :time))))) + (texts nil)) + (dolist (n sorted) + (let* ((demotion (elel-import--demote (plist-get n :body))) + (parent (plist-get n :parent)) + (known (and parent (member parent ids)))) + (setq demoted (+ demoted (cdr demotion))) + (cond + ((and parent (not known)) + (push (format " note %s: parent %s is not among the notes" + (plist-get n :number) parent) + report))) + (push (elel-import--org-note + (plist-get n :title) + (list group) + (append + (list (cons "ID" (plist-get n :id)) + (cons "DATE" (elel--now (plist-get n :time)))) + (and known (list (cons "PARENT" parent))) + (list (cons "DRAFT_FILE" (plist-get n :number)))) + (car demotion)) + texts))) + (push (cons (concat group ".org") + (mapconcat #'identity (nreverse texts) "\n")) + files) + (push (format " %s.org: %d notes" group (length sorted)) report))) + (push (format "headings demoted : %d" demoted) report) + (push "DATE is the modification time of the draft file (drafts have no Date header)." + report) + (elel-import--finish + (list :files (nreverse files) :report (reverse report)) + out dry-run))) + +;;; org-roam + +(defconst elel-import--file-link-regexp + "\\[\\[file:\\([^]\n]*\\)\\]\\(?:\\[\\([^]\n]*\\)\\]\\)?\\]" + "Matches an org file link. Group 1 is the path, group 2 the description.") + +(defun elel-import--keyword (text name) + "The value of the keyword #+NAME: in TEXT, or nil." + (let ((case-fold-search t)) + (and (string-match + (concat "^#\\+" name ":[ \t]*\\(.*?\\)[ \t]*$") text) + (not (string-empty-p (match-string 1 text))) + (match-string 1 text)))) + +(defun elel-import--stamp (path) + "A compact time stamp such as 20200514T232108 for the roam file PATH." + (let ((name (file-name-nondirectory path))) + (cond + ((or (string-match + (concat "\\`\\([0-9]\\{4\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)" + "\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)-") + name) + (string-match + (concat "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)" + "--\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-") + name)) + (format "%s%s%sT%s%s%s" + (match-string 1 name) (match-string 2 name) + (match-string 3 name) (match-string 4 name) + (match-string 5 name) (match-string 6 name))) + ((string-match + "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)\\.org\\'" + name) + (format "%s%s%sT000000" + (match-string 1 name) (match-string 2 name) + (match-string 3 name))) + ((format-time-string + "%Y%m%dT%H%M%S" + (file-attribute-modification-time (file-attributes path))))))) + +(defun elel-import--stamp-time (stamp) + "The time of STAMP, a string such as 20200514T232108, in local time." + (let ((n (lambda (a b) (string-to-number (substring stamp a b))))) + (encode-time (list (funcall n 13 15) (funcall n 11 13) (funcall n 9 11) + (funcall n 6 8) (funcall n 4 6) (funcall n 0 4) + nil -1 nil)))) + +(defun elel-import--cover (items min) + "Choose one tag for each of ITEMS, so that few tags are used. +ITEMS is a list of (KEY . TAGS). Return a hash table from KEY to the +chosen tag, or to nothing when no good tag exists. Tags are picked +greedily, each time the one that covers most of the notes left, until +no tag covers MIN notes; the notes left are not in the table." + (let ((left (cl-remove-if-not #'cdr items)) + (chosen (make-hash-table :test 'equal)) + (counts (make-hash-table :test 'equal)) + (continue t)) + (while (and left continue) + (clrhash counts) + (dolist (item left) + (dolist (tag (cdr item)) + (puthash tag (1+ (gethash tag counts 0)) counts))) + (let (best (best-count 0)) + (maphash (lambda (tag count) + (cond + ((or (> count best-count) + (and (= count best-count) best (string< tag best))) + (setq best tag best-count count)))) + counts) + (cond + ((or (null best) (< best-count min)) (setq continue nil)) + (t + (dolist (item left) + (cond ((member best (cdr item)) (puthash (car item) best chosen)))) + (setq left (cl-remove-if (lambda (item) (member best (cdr item))) + left)))))) + chosen)) + +(defun elel-import--target (rel _tags &optional chosen) + "The container file for the roam file REL, whose tags are TAGS. +A journal goes to journal.org; otherwise the tag CHOSEN names the +file; otherwise the directory does; otherwise the file is roam.org." + (let ((dir (file-name-directory rel))) + (cond + ((string-prefix-p "journal/" rel) "journal.org") + (chosen (concat chosen ".org")) + (dir (concat (replace-regexp-in-string + "[ \t]+" "-" (directory-file-name dir)) + ".org")) + ("roam.org")))) + +(defun elel-import--tags (text) + "The #+roam_tags of TEXT, made into valid org tags." + (let ((raw (elel-import--keyword text "roam_tags")) + (tags nil)) + (dolist (w (and raw (or (ignore-errors (split-string-and-unquote raw)) + (split-string raw)))) + (let ((tag (string-trim + (replace-regexp-in-string "[^[:alnum:]_@#%]" "_" w) + "_+" "_+"))) + (cond + ((and (not (string-empty-p tag)) (not (member tag tags))) + (push tag tags))))) + (nreverse tags))) + +(defcustom elel-import-roam-min-group 2 + "The fewest notes that a file made from a tag may hold. +A note whose tags would make smaller files goes to roam.org instead." + :type 'integer + :group 'elel) + +;;;###autoload +(defun elel-import-roam (src out &optional dry-run) + "Import the org-roam directory SRC into the directory OUT. +Each roam file becomes one note. The notes are collected into +container files, one per first tag (see `elel-import--target'), sorted +by date. With a nil OUT, or a non-nil DRY-RUN, only report." + (interactive + (list (read-directory-name "Roam directory: " "~/org/roam/") + (read-directory-name "Write to: " (expand-file-name "roam" elel-root)) + current-prefix-arg)) + (let* ((src (file-name-as-directory (expand-file-name src))) + (paths (cl-remove-if + (lambda (p) + (string-match-p "\\(\\`\\|/\\)\\." + (file-relative-name p src))) + (sort (directory-files-recursively src "\\.org\\'") + #'string<))) + (notes (make-hash-table :test 'equal)) + (links 0) + (demoted 0) + (unresolved nil) + (drawers nil) + (cover nil) + (groups nil) + (group-report nil) + (files nil)) + (dolist (path paths) + (let* ((rel (file-relative-name path src)) + (text (elel-import--read path)) + (stamp (elel-import--stamp path))) + (puthash path + (list :rel rel + :text text + :title (or (elel-import--keyword text "title") + (file-name-base path)) + :stamp stamp + :id (concat stamp "-" + (substring (md5 rel nil nil 'utf-8) 0 4)) + :tags (elel-import--tags text)) + notes))) + (setq cover + (elel-import--cover + (cl-loop for path in paths + for n = (gethash path notes) + unless (string-prefix-p "journal/" (plist-get n :rel)) + collect (cons (plist-get n :rel) (plist-get n :tags))) + elel-import-roam-min-group)) + (dolist (path paths) + (let* ((n (gethash path notes)) + (rel (plist-get n :rel)) + (body (let ((case-fold-search t)) + (string-trim + (replace-regexp-in-string + "^#\\+\\(?:title\\|roam_tags\\):.*\n?" "" + (plist-get n :text)) + "\n+" "\n+"))) + (rewritten + (replace-regexp-in-string + elel-import--file-link-regexp + (lambda (m) + (string-match elel-import--file-link-regexp m) + (let* ((target (match-string 1 m)) + (desc (match-string 2 m)) + (p (car (split-string target "::" t))) + (found (and p + (gethash + (expand-file-name + p (file-name-directory path)) + notes)))) + (cond + (found + (setq links (1+ links)) + (concat "[[note:" (plist-get found :id) "][" + (cond + ((or (null desc) (string-empty-p desc)) + (plist-get found :title)) + (desc)) + "]]")) + (t + (cond + ((and p (string-suffix-p ".org" p)) + (push (cons rel target) unresolved))) + m)))) + body t t)) + (demotion (elel-import--demote rewritten))) + (cond + ((string-match-p "\\`[ \t\n]*:PROPERTIES:" body) + (push rel drawers))) + (setq demoted (+ demoted (cdr demotion))) + (let* ((target (elel-import--target rel (plist-get n :tags) + (gethash rel cover))) + (cell (assoc target groups)) + (item (cons + (plist-get n :stamp) + (elel-import--org-note + (plist-get n :title) + (plist-get n :tags) + (append + (list (cons "ID" (plist-get n :id)) + (cons "DATE" + (elel--now (elel-import--stamp-time + (plist-get n :stamp))))) + (and (string-prefix-p "journal/" rel) + (list (cons "KIND" "journal"))) + (list (cons "ROAM_FILE" rel))) + (car demotion))))) + (cond + (cell (setcdr cell (cons item (cdr cell)))) + ((push (list target item) groups)))))) + (dolist (g (sort groups (lambda (a b) (string< (car a) (car b))))) + (let ((sorted (sort (cdr g) (lambda (a b) (string< (car a) (car b)))))) + (push (cons (car g) + (mapconcat #'cdr sorted "\n")) + files) + (push (format " %s: %d notes" (car g) (length sorted)) group-report))) + (elel-import--finish + (list + :files (nreverse files) + :report + (append + (list (format "source : %s" src) + (format "notes : %d" (length paths)) + (format "with tags : %d" + (cl-count-if (lambda (p) + (plist-get (gethash p notes) :tags)) + paths)) + (format "links to notes rewritten : %d" links) + (format "headings demoted : %d" demoted)) + (reverse group-report) + (mapcar (lambda (u) + (format " unresolved .org link in %s: %s" + (car u) (cdr u))) + (reverse unresolved)) + (mapcar (lambda (rel) + (format " warning: %s starts with a property drawer" rel)) + (reverse drawers)))) + out dry-run))) + +(provide 'elel-import) +;;; elel-import.el ends here @@ -0,0 +1,1816 @@ +;;; elel.el --- Front end for the elel notes engine -*- lexical-binding: t; -*- + +;; Copyright (C) 2026 Durand + +;; Author: Durand +;; Keywords: outlines, files +;; Package-Requires: ((emacs "27.1")) + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;; This is the Emacs side of elel. The engine (a Rust program in the +;; same directory) reads the note files and keeps an index; this file +;; only shows what the engine reports and helps to write notes. +;; +;; The interface follows Gnus: +;; +;; Group buffer -> the views buffer, M-x elel +;; Summary buffer -> the summary buffer, threaded by Parent +;; Article buffer -> the article buffer, the note as it is in its +;; file, in the major mode of its file, with the +;; minor mode `elel-article-mode' on top +;; +;; The summary and the article are shown together; close either. +;; The engine never writes to the note files. Creating notes and +;; adding missing IDs is done here. + +;;; Code: + +(require 'cl-lib) +(require 'subr-x) +(require 'button) +(require 'outline) +(require 'rx) +(require 'hl-line) + +(declare-function org-link-set-parameters "ol" (type &rest parameters)) +(declare-function org-entry-get "org" (epom property &optional inherit literal-nil)) +(defvar elel-article-id) +(defvar elel-capture-id) + +(declare-function org-next-link "org" (&optional search-backward)) +(declare-function org-previous-link "org" ()) +(declare-function org-fold-hide-drawer-all "org-fold" ()) +(declare-function org-hide-drawer-all "org" ()) +(declare-function org-open-at-point "org" (&optional arg)) +(declare-function org-back-to-heading "org" (&optional invisible-ok)) +(declare-function org-entry-get "org" (pom property &optional inherit literal-nil)) + +;;; Customization + +(defgroup elel nil + "A notes system with full-text search and links." + :group 'applications + :prefix "elel-") + +(defconst elel--directory + (file-name-directory (or load-file-name buffer-file-name default-directory)) + "The directory of this file.") + +(defcustom elel-root (expand-file-name "~/notes") + "The directory that holds the note files." + :type 'directory) + +(defcustom elel-index-directory nil + "The directory of the derived index. +If nil, the engine chooses its own default." + :type '(choice (const :tag "Engine default" nil) directory)) + +(defcustom elel-program + (let ((release (expand-file-name "target/release/elel" elel--directory)) + (debug (expand-file-name "target/debug/elel" elel--directory))) + (cond + ((file-executable-p release) release) + ((file-executable-p debug) debug) + ((executable-find "elel")) + ("elel"))) + "The engine executable." + :type 'file) + +(defcustom elel-capture-file "inbox.org" + "The file, relative to `elel-root', where new notes are written. +The extension decides the format: \".org\" or \".txt\"." + :type 'string) + +(defcustom elel-auto-index t + "If non-nil, update the index after saving a file under `elel-root'." + :type 'boolean) + +(defcustom elel-id-function #'elel-generate-id + "The function that makes the ID of a new note." + :type 'function) + +(defcustom elel-views + (list + (list "Inbox" "is:inbox") + (list "Recent" "" :limit 30) + (list "All" "")) + "The views listed in the views buffer, like the groups of Gnus. +Each element is (NAME QUERY . PLIST). QUERY is a search query (see +the README), and PLIST may have these properties: + :limit N show at most N notes; + :hidden t list the view only after \\`L' (list all views). +Views for a tag or a file are just entries such as + (\"Math\" \"tag:math\") or (\"Journal\" \"file:journal\" :hidden t)." + :type '(repeat (cons string (cons string plist)))) + +(defcustom elel-major-mode-alist + (list (cons (rx-to-string (list 'seq ".org" 'eos) t) 'org-mode) + (cons (rx-to-string (list 'seq 'any) t) 'text-mode)) + "Alist of (REGEXP . MODE): the major mode for showing a note. +The first REGEXP that matches the file name of the note decides." + :type '(alist :key-type regexp :value-type function)) + +(defcustom elel-summary-window-height 0.3 + "The fraction of the height that the summary window keeps. +It applies when an article window is made below the summary." + :type 'number) + +(defcustom elel-article-keep-summary t + "Non-nil to show the article below the summary. +If nil, the article replaces the summary window; press \\`h' in the +article to go back to the summary." + :type 'boolean) + +(defcustom elel-thread-root "" + "What to draw before the title of a note without a parent." + :type 'string) + +(defcustom elel-thread-false-root "─┬> " + "What to draw before a note whose parent is not in the summary." + :type 'string) + +(defcustom elel-thread-indent " " + "What to draw under a note that has no younger siblings." + :type 'string) + +(defcustom elel-thread-vertical "│" + "What to draw under a note that has younger siblings." + :type 'string) + +(defcustom elel-thread-leaf-with-other "├─> " + "What to draw before a note that has younger siblings." + :type 'string) + +(defcustom elel-thread-single-leaf "└─> " + "What to draw before a note that is the last child of its parent." + :type 'string) + +;;; Faces + +(defface elel-title + '((t :weight bold)) + "Face of note titles.") + +(defface elel-mark + '((t :inherit font-lock-warning-face :weight bold)) + "Face of the mark column of the summary buffer." + :group 'elel) + +(defcustom elel-suggest-after-capture t + "Non-nil means a new note is followed by suggestions of links and tags." + :type 'boolean + :group 'elel) + +(defcustom elel-suggest-limit 15 + "How many notes to suggest." + :type 'integer + :group 'elel) + +(defface elel-date + '((t :inherit shadow)) + "Face of dates.") + +(defface elel-tag + '((t :inherit font-lock-type-face)) + "Face of tags.") + +(defface elel-hit + '((t :inherit highlight)) + "Face of the words that matched a search.") + +(defface elel-thread + '((t :inherit shadow)) + "Face of the lines that draw the threads.") + +(defface elel-heading + '((t :inherit font-lock-keyword-face :weight bold)) + "Face of section headings.") + +;;; Talking to the engine + +(defun elel--environment () + "The process environment for running the engine." + (append + (list (concat "ELEL_ROOT=" (expand-file-name elel-root))) + (cond + (elel-index-directory + (list (concat "ELEL_INDEX=" (expand-file-name elel-index-directory))))) + process-environment)) + +(defun elel--parse-lines () + "Parse the JSON objects, one per line, in the current buffer." + (goto-char (point-min)) + (let (objects) + (while (not (eobp)) + (cond + ((eq (char-after) ?{) + (push (json-parse-buffer :object-type 'alist + :array-type 'list + :null-object nil + :false-object nil) + objects)) + ((forward-line 1)))) + (nreverse objects))) + +(defun elel--call (&rest args) + "Run the engine with ARGS. Return the list of objects it prints." + (let ((process-environment (elel--environment)) + (coding-system-for-read 'utf-8) + (coding-system-for-write 'utf-8)) + (with-temp-buffer + (let ((status (apply #'call-process elel-program nil t nil args))) + (cond + ((eq status 0) (elel--parse-lines)) + ((user-error "elel: %s" (string-trim (buffer-string))))))))) + +(defvar elel--index-process nil + "The running asynchronous index process, if any.") + +(defvar elel--index-pending nil + "Non-nil if another index was requested while one was running.") + +(defun elel-index () + "Update the index now, and wait for it." + (interactive) + (let ((r (car (elel--call "index")))) + (message "elel: %d files, %d notes (%d changed, %d removed)" + (alist-get 'files r) (alist-get 'notes r) + (alist-get 'changed r) (alist-get 'removed r)) + r)) + +(defvar elel--index-callbacks nil + "Functions to call once the running index has finished.") + +(defun elel-index-async () + "Update the index in the background." + (interactive) + (cond + ((process-live-p elel--index-process) + (setq elel--index-pending t)) + (t + (let ((process-environment (elel--environment)) + (buffer (generate-new-buffer " *elel-index*"))) + (setq elel--index-pending nil) + (setq elel--index-process + (make-process + :name "elel-index" + :buffer buffer + :command (list elel-program "index") + :connection-type 'pipe + :noquery t + :sentinel + (lambda (process _event) + (cond + ((not (process-live-p process)) + (let ((ok (eq (process-exit-status process) 0)) + (out (with-current-buffer (process-buffer process) + (string-trim (buffer-string))))) + (kill-buffer (process-buffer process)) + (cond + ((not ok) + (setq elel--index-callbacks nil) + (message "elel: indexing failed: %s" out)) + (elel--index-pending (elel-index-async)) + (t (elel--run-index-callbacks))))))))))))) + +(defun elel--run-index-callbacks () + "Call and forget the functions waiting for the index." + (let ((callbacks (nreverse elel--index-callbacks))) + (setq elel--index-callbacks nil) + (dolist (f callbacks) + (condition-case err + (funcall f) + (error (message "elel: %s" (error-message-string err))))))) + +(defun elel--maybe-index () + "Index after saving a file under `elel-root'." + (cond + ((and elel-auto-index + buffer-file-name + (string-prefix-p + (file-name-as-directory (expand-file-name elel-root)) + (expand-file-name buffer-file-name))) + (elel-index-async)))) + +(add-hook 'after-save-hook #'elel--maybe-index) + +;;; Rendering helpers + +(defun elel--field (note key) + "The value of KEY in NOTE." + (alist-get key note)) + +(defun elel--short-date (note) + "The date of NOTE, cut to ten characters." + (let ((d (elel--field note 'date))) + (cond + ((stringp d) (substring d 0 (min 10 (length d)))) + ((make-string 10 ?\s))))) + +(defun elel--tags (note) + "The tags of NOTE as a propertized string." + (let ((tags (elel--field note 'tags))) + (cond + (tags (propertize (concat ":" (mapconcat #'identity tags ":") ":") + 'face 'elel-tag)) + ("")))) + +(defun elel--fontify-snippet (string) + "Turn the highlight markers of the engine in STRING into faces." + (with-temp-buffer + (insert string) + (goto-char (point-min)) + (while (search-forward (string #xe000) nil t) + (let ((beg (match-beginning 0))) + (delete-region beg (point)) + (cond + ((search-forward (string #xe001) nil t) + (let ((end (match-beginning 0))) + (delete-region end (point)) + (put-text-property beg end 'face 'elel-hit)))))) + (buffer-string))) + +;;; Asking the engine for notes + +(defun elel--get (id) + "The engine's answer for the note ID. +If the files moved since the last index, update the index and retry." + (condition-case err + (car (elel--call "get" id)) + (user-error + (cond + ((string-match-p "index out of date" (error-message-string err)) + (elel-index) + (car (elel--call "get" id))) + ((signal (car err) (cdr err))))))) + +(defun elel--quote-id (id) + "ID, quoted if needed to be the value of a filter in a query." + (cond + ((string-match-p "[ \t\"]" id) (concat "\"" id "\"")) + (id))) + +(defun elel--filter (key id) + "The query term that selects with KEY (such as \"thread\") the note ID." + (concat key ":" (elel--quote-id id))) + +(defcustom elel-hybrid-search t + "Non-nil means a search also ranks by meaning. +The words of the query are embedded and the result is fused with the +text search. If Ollama is not running, only the text search is used." + :type 'boolean + :group 'elel) + +(defun elel--search (query &optional limit) + "The notes matching QUERY, at most LIMIT (default: all)." + (apply #'elel--call "search" "-n" (number-to-string (or limit 1000000)) + (append (and elel-hybrid-search + (not (string-empty-p (string-trim (or query "")))) + (list "--hybrid")) + (list "--" (or query ""))))) + +(defun elel--counts (queries) + "The number of notes matching each of QUERIES." + (mapcar (lambda (o) (alist-get 'count o)) + (apply #'elel--call "count" "--" queries))) + +;;; Threads + +(defun elel--thread (notes) + "Order NOTES so that each child follows its parent. +Return a list of (NOTE PREFIX CONTINUATION), where PREFIX is the +drawing in front of the title (see `elel-thread-root' and friends) and +CONTINUATION is what to put in front of a further line of the note. A +note whose parent is not among NOTES is a root." + (let ((ids (mapcar (lambda (n) (elel--field n 'id)) notes)) + (kids (make-hash-table :test 'equal)) + (seen (make-hash-table :test 'equal)) + roots out) + (dolist (n notes) + (let ((p (elel--field n 'parent))) + (cond + ((and p (member p ids) (not (equal p (elel--field n 'id)))) + (puthash p (cons n (gethash p kids)) kids)) + ((push n roots))))) + (cl-labels + ((walk (n depth ancestors lastp) + (let ((id (elel--field n 'id))) + (cond + ((gethash id seen)) + (t + (puthash id t seen) + (let* ((leaf (cond + ((= depth 0) + (cond + ((elel--field n 'parent) elel-thread-false-root) + (elel-thread-root))) + (lastp elel-thread-single-leaf) + (elel-thread-leaf-with-other))) + (kids-of (cl-remove-if + (lambda (c) (gethash (elel--field c 'id) seen)) + (reverse (gethash id kids)))) + (below (cond + ((= depth 0) "") + (t (concat ancestors + (cond + (lastp elel-thread-indent) + (elel-thread-vertical))))))) + (push (list n + (concat (cond ((= depth 0) "") (ancestors)) leaf) + (concat (cond ((= depth 0) "") (ancestors)) + (make-string (string-width leaf) ?\s))) + out) + (let ((rest kids-of)) + (while rest + (walk (car rest) (1+ depth) below (null (cdr rest))) + (setq rest (cdr rest)))))))))) + (dolist (r (nreverse roots)) (walk r 0 "" t)) + ;; Notes caught in a cycle of parents have no root. + (dolist (n notes) (walk n 0 "" t))) + (nreverse out))) + +;;; The summary buffer + +(defconst elel--summary-buffer-name "*elel summary*") +(defconst elel--article-buffer-name "*elel article*") + +(defvar-local elel-summary-title nil + "The title of this summary buffer.") + +(defvar-local elel-summary-query nil + "The query of this summary buffer, or nil if it has a function.") + +(defvar-local elel-summary-limit nil + "The maximum number of notes of the query, or nil.") + +(defvar-local elel-summary-function nil + "A function that returns the notes of this summary, instead of a query.") + +(defvar-local elel-summary-extra nil + "Queries whose notes are added to this summary (see `elel-summary-refer-thread').") + +(defvar-local elel-summary-threaded t + "Non-nil if the notes are threaded by their parents.") + +(defvar-local elel-summary-show-snippets nil + "Non-nil if snippets are shown below the notes.") + +(defvar-local elel-summary-marks nil + "Alist of (ID . CHAR) for the marked notes: ?+ is liked, ?- is disliked.") + +(defvar-local elel-summary-insert-marker nil + "Where `elel-summary-insert-link' puts the link, a marker, or nil.") + +(defvar elel-search-history nil + "History of search queries.") + +(defvar elel-summary-mode-map (make-sparse-keymap) + "Keymap of `elel-summary-mode'.") + +;; Filled in separately, so that loading this file again updates the keys. +(let ((map elel-summary-mode-map)) + (set-keymap-parent map special-mode-map) + (define-key map (vector ?\r) #'elel-summary-open) + (define-key map (vector ?\s) #'elel-summary-scroll-up) + (define-key map (vector ?\d) #'elel-summary-scroll-down) + (define-key map (vector ?n) #'elel-summary-next) + (define-key map (vector ?p) #'elel-summary-previous) + (define-key map (vector ?g) #'elel-summary-show-article) + (define-key map (vector ?s) #'elel-summary-toggle-snippets) + (define-key map (vector ?/) #'elel-summary-narrow) + (define-key map (vector ?G) #'elel-search) + (define-key map (vector ?a) #'elel-summary-refer-thread) + (define-key map (vector ?^) #'elel-summary-parent) + (define-key map (vector ?c) #'elel-summary-children) + (define-key map (vector ?o) #'elel-summary-outlinks) + (define-key map (vector ?b) #'elel-summary-backlinks) + (define-key map (vector ?H) #'elel-summary-hood) + (define-key map (vector ?e) #'elel-summary-edit) + (define-key map (vector ?r) #'elel-summary-new-child) + (define-key map (vector ?N) #'elel-new-note) + (define-key map (vector ?#) #'elel-summary-mark-like) + (define-key map (vector ?-) #'elel-summary-mark-unlike) + (define-key map (vector ?R) #'elel-summary-related) + (define-key map (vector ?S) #'elel-suggest) + (define-key map (vector ?i) #'elel-summary-insert-link) + (define-key map (vector ?=) #'elel-summary-expand-window) + (define-key map (vector ?A) #'elel-summary-select-article-buffer) + (define-key map (vector ?q) #'elel-summary-quit)) + +(define-derived-mode elel-summary-mode special-mode "elel-summary" + "Major mode for a list of notes, like a Gnus summary buffer. +\\{elel-summary-mode-map}" + (setq truncate-lines t) + (hl-line-mode 1) + (setq-local revert-buffer-function + (lambda (&rest _) (elel-summary-refresh)))) + +(defun elel--id-at-point () + "The ID of the note at point, in a summary buffer." + (or (get-text-property (point) 'elel-id) + (user-error "No note here"))) + +(defun elel--note-at-point () + "The note at point, in a summary buffer." + (or (get-text-property (point) 'elel-note) + (user-error "No note here"))) + +(defun elel--insert-note-line (entry show-snippets) + "Insert the lines of ENTRY, an element of the value of `elel--thread'." + (let* ((note (nth 0 entry)) + (prefix (nth 1 entry)) + (continuation (nth 2 entry)) + (beg (point)) + (rel (elel--field note 'rel)) + (score (elel--field note 'score)) + (back (elel--field note 'backlinks)) + (snippet (elel--field note 'snippet))) + (insert + (propertize (string (or (alist-get (elel--field note 'id) elel-summary-marks + nil nil #'equal) + ?\s)) + 'face 'elel-mark) + (propertize (elel--short-date note) 'face 'elel-date) + " " + (cond + (rel (propertize (format "%-8s " rel) 'face 'elel-date)) + ("")) + (propertize prefix 'face 'elel-thread) + (propertize (elel--field note 'title) 'face 'elel-title) + " " (elel--tags note) + (cond + ((and (numberp back) (> back 0)) + (propertize (format " ←%d" back) 'face 'elel-date)) + ("")) + (cond + (score (propertize (format " %.2f" score) 'face 'elel-date)) + ("")) + "\n") + (cond + ((and show-snippets snippet) + (insert (make-string 13 ?\s) + (propertize continuation 'face 'elel-thread) + (elel--fontify-snippet snippet) "\n"))) + (put-text-property beg (point) 'elel-id (elel--field note 'id)) + (put-text-property beg (point) 'elel-note note))) + +(defun elel--summary-notes () + "The notes of the current summary buffer, asked of the engine." + (let ((notes (cond + (elel-summary-function (funcall elel-summary-function)) + (t (elel--search elel-summary-query elel-summary-limit))))) + (dolist (q elel-summary-extra) + (dolist (n (elel--search q)) + (unless (cl-find (elel--field n 'id) notes + :key (lambda (x) (elel--field x 'id)) + :test #'equal) + (setq notes (append notes (list n)))))) + notes)) + +(defun elel--render-summary (notes) + "Show NOTES in the current summary buffer." + (let ((inhibit-read-only t) + (id (get-text-property (point) 'elel-id)) + (line (line-number-at-pos))) + (erase-buffer) + (insert (propertize (or elel-summary-title "Notes") 'face 'elel-heading) + (format " (%d)\n\n" (length notes))) + (dolist (entry (cond + (elel-summary-threaded (elel--thread notes)) + ((mapcar (lambda (n) (list n "" "")) notes)))) + (elel--insert-note-line entry elel-summary-show-snippets)) + (goto-char (point-min)) + (cond + ((and id (elel--goto-note id))) + ((forward-line (max 2 (1- line))))))) + +(defun elel--goto-note (id) + "Move to the line of the note ID in this summary buffer. +Return non-nil if it is there." + (let ((pos (point-min)) + found) + (while (and (not found) + (setq pos (next-single-property-change pos 'elel-id))) + (cond + ((equal (get-text-property pos 'elel-id) id) + (goto-char pos) + (setq found t)))) + found)) + +(defun elel--summary-buffer () + "The summary buffer, or nil." + (get-buffer elel--summary-buffer-name)) + +(cl-defun elel--show-summary (title &key query limit function (threaded t) quiet) + "Show a summary buffer named TITLE and select its window. +The notes are those of the QUERY (at most LIMIT of them), or the value +of FUNCTION. THREADED says whether to thread them. With QUIET, only +display the buffer, and leave the selection alone." + (let ((buffer (get-buffer-create elel--summary-buffer-name))) + (with-current-buffer buffer + (elel-summary-mode) + (let ((inhibit-read-only t)) (erase-buffer)) + (setq elel-summary-title title) + (setq elel-summary-query query) + (setq elel-summary-limit limit) + (setq elel-summary-function function) + (setq elel-summary-threaded threaded) + (let ((notes (elel--summary-notes))) + (setq elel-summary-show-snippets + (and (seq-some (lambda (n) (elel--field n 'snippet)) notes) t)) + (elel--render-summary notes))) + (let ((window (get-buffer-window buffer))) + (cond + (quiet (display-buffer buffer)) + (window (select-window window)) + (t (switch-to-buffer buffer)))) + buffer)) + +(defun elel-summary-refresh () + "Ask the engine again and redraw." + (interactive) + (elel--render-summary (elel--summary-notes))) + +(defun elel-summary-toggle-snippets () + "Show or hide the snippets." + (interactive) + (setq elel-summary-show-snippets (not elel-summary-show-snippets)) + (elel-summary-refresh)) + +(defun elel-summary-show-article () + "Show the note at point again, read fresh from its file, like Gnus's \`g'. +The summary stays as it is; search again (\`G') or use `revert-buffer' to +refresh it." + (interactive) + (elel--show-article (elel--id-at-point))) + +(defun elel-summary-open () + "Show the note at point in the article window." + (interactive) + (elel--show-article (elel--id-at-point))) + +(defun elel-summary-expand-window (&optional arg) + "Make the summary window take up the whole frame, like Gnus's `='. +This works from the article window too: the summary is what expands. +With a prefix argument ARG, go back to the summary and article layout, +showing the note at point (or else the note last shown)." + (interactive "P") + (let* ((summary (or (elel--summary-buffer) (user-error "No summary"))) + (article (get-buffer elel--article-buffer-name)) + (id (or (cond + ((eq (current-buffer) summary) + (get-text-property (point) 'elel-id)) + ((bound-and-true-p elel-article-mode) elel-article-id)) + (and article (buffer-local-value 'elel-article-id article)))) + (window (get-buffer-window summary))) + (cond + (window (select-window window)) + ((switch-to-buffer summary))) + (delete-other-windows) + (cond + ((not arg)) + (id (elel--show-article id)) + ((message "No note to show"))))) + +(defun elel-summary-select-article-buffer () + "Select the article window, like `gnus-summary-select-article-buffer'. +If the article is not shown (say, after `elel-summary-expand-window'), +show it again first. If there is no article yet, show the note at point." + (interactive) + (let ((buffer (get-buffer elel--article-buffer-name)) + (window (elel--article-visible-window))) + (cond + ((null buffer) (elel--show-article (elel--id-at-point) nil t)) + (window (select-window window)) + (t + (let ((new (elel--article-window))) + (set-window-buffer new buffer) + (select-window new)))))) + +(defun elel--article-visible-window () + "The window that shows the article buffer, or nil." + (get-buffer-window elel--article-buffer-name)) + +(defun elel-summary-scroll-up () + "Scroll the article; at its end, show the next note." + (interactive) + (let ((window (elel--article-visible-window))) + (cond + ((null window) (elel-summary-open)) + (t + (condition-case nil + (with-selected-window window (scroll-up-command)) + (end-of-buffer (elel-summary-next))))))) + +(defun elel-summary-scroll-down () + "Scroll the article backward." + (interactive) + (let ((window (elel--article-visible-window))) + (cond + ((null window) (user-error "No article")) + ((condition-case nil + (with-selected-window window (scroll-down-command)) + (beginning-of-buffer (message "Beginning of the note"))))))) + +(defun elel--move-to-note (step) + "Move STEP (1 or -1) notes in this summary buffer. Return non-nil if moved." + (let ((start (point)) + found) + (while (and (not found) + (zerop (forward-line step)) + (not (eobp))) + (cond + ((get-text-property (point) 'elel-id) + (let ((here (get-text-property (point) 'elel-id)) + (there (get-text-property start 'elel-id))) + (cond + ((not (equal here there)) (setq found t))))))) + (cond + (found + ;; Land on the first line of the note. + (beginning-of-line) + (while (and (> (point) (point-min)) + (equal (get-text-property (1- (point)) 'elel-id) + (get-text-property (point) 'elel-id))) + (forward-line -1))) + ((goto-char start))) + found)) + +(defun elel-summary-next () + "Show the next note." + (interactive) + (cond + ((not (get-text-property (point) 'elel-id)) + (goto-char (point-min)) + (cond ((next-single-property-change (point) 'elel-id) + (goto-char (next-single-property-change (point) 'elel-id)) + (elel-summary-open)))) + ((elel--move-to-note 1) (elel-summary-open)) + ((message "No more notes")))) + +(defun elel-summary-previous () + "Show the previous note." + (interactive) + (cond + ((elel--move-to-note -1) (elel-summary-open)) + ((message "No previous note")))) + +(defun elel-summary-narrow (query) + "Search again, with QUERY, which starts as the current query." + (interactive + (list (read-string "Limit to: " + (cond + (elel-summary-query (concat elel-summary-query " ")) + ("")) + 'elel-search-history))) + (elel--show-summary (format "Search: %s" query) :query query :threaded nil)) + +;;; Related notes and suggestions (the semantic side) + +(defun elel--set-mark (char) + "Toggle the mark CHAR on the note at point, then go to the next note." + (let* ((id (elel--id-at-point)) + (note (get-text-property (point) 'elel-note)) + (old (alist-get id elel-summary-marks nil nil #'equal))) + (setf (alist-get id elel-summary-marks nil t #'equal) + (cond ((eq old char) nil) (char))) + (let ((inhibit-read-only t)) + (save-excursion + (goto-char (line-beginning-position)) + (delete-char 1) + (insert (propertize (string (or (alist-get id elel-summary-marks + nil nil #'equal) + ?\s)) + 'face 'elel-mark 'elel-id id 'elel-note note)))) + (elel--move-to-note 1))) + +(defun elel-summary-mark-like () + "Mark the note at point as one that the wanted notes resemble." + (interactive) + (elel--set-mark ?+)) + +(defun elel-summary-mark-unlike () + "Mark the note at point as one that the wanted notes should not resemble." + (interactive) + (elel--set-mark ?-)) + +(defun elel--related-query (positive negative) + "The query for the notes like those of IDs POSITIVE and unlike NEGATIVE." + (mapconcat #'identity + (append (mapcar (lambda (id) (format "like:\"%s\"" id)) positive) + (mapcar (lambda (id) (format "-like:\"%s\"" id)) negative)) + " ")) + +(defun elel--related (positive negative) + "Show the notes like those of IDs POSITIVE and unlike NEGATIVE." + (elel--show-summary "Related" + :query (elel--related-query positive negative) + :limit 30 :threaded nil)) + +(defun elel-summary-related () + "Show the notes closest in meaning to the marked notes. +Notes marked with \\`#' count for, notes marked with \\`-' count against. +If none is marked, use the note at point." + (interactive) + (let (positive negative) + (dolist (m elel-summary-marks) + (cond + ((eq (cdr m) ?+) (push (car m) positive)) + ((eq (cdr m) ?-) (push (car m) negative)))) + (cond + ((null positive) (setq positive (list (elel--id-at-point))))) + (elel--related positive negative))) + +(defun elel-article-related () + "Show the notes closest in meaning to this note." + (interactive) + (elel--related (list elel-article-id) nil)) + +(defun elel--suggested-tags (id) + "The tags that fit the note ID best and that it does not have yet." + (let (tags) + (dolist (x (elel--call "tags" "--like" id)) + (cond + ((and (not (alist-get 'has x)) (< (length tags) 5)) + (push (alist-get 'tag x) tags)))) + (nreverse tags))) + +;;;###autoload +(defun elel-suggest (&optional id marker quiet) + "Show the notes that a note ID could link to, and tags that fit it. +The notes it already has relations with are left out. If MARKER is a +marker, then \\`i' in the summary inserts a link at that place. With +QUIET, do not select the summary window." + (interactive) + (let* ((id (or id (elel--note-id-here) (user-error "No note here"))) + (data (elel--get id)) + (excluded (cons id + (mapcar (lambda (n) (elel--field n 'id)) + (append (list (alist-get 'parent data)) + (alist-get 'children data) + (alist-get 'links data))))) + (tags (elel--suggested-tags id)) + (title (format "Suggestions for %s%s" + (elel--field (alist-get 'note data) 'title) + (cond (tags (concat " tags: " (string-join tags " "))) + ("")))) + (buffer + (elel--show-summary + title + :function (lambda () + (cl-remove-if + (lambda (n) (member (elel--field n 'id) excluded)) + (elel--search (format "like:\"%s\"" id) + (+ elel-suggest-limit + (length excluded))))) + :threaded nil + :quiet quiet))) + (with-current-buffer buffer + (setq elel-summary-insert-marker (and (markerp marker) marker))))) + +(defun elel-summary-insert-link () + "Insert a link to the note at point where `elel-suggest' was asked for." + (interactive) + (let ((note (elel--note-at-point)) + (marker elel-summary-insert-marker)) + (cond + ((not (and marker (marker-buffer marker))) + (user-error "No place to insert a link; use \\`S' from a capture")) + (t + (with-current-buffer (marker-buffer marker) + (save-excursion + (goto-char marker) + (cond + ((and (not (bolp)) (not (memq (char-before) (list ?\s ?\t)))) + (insert " "))) + (insert (elel--link-string (elel--field note 'id) + (elel--field note 'title)) + " ") + (set-marker marker (point)))) + (message "elel: linked %s" (elel--field note 'title)))))) + +(defun elel-summary-refer-thread (&optional id) + "Add the whole thread of the note ID (default: at point) to this summary." + (interactive) + (let ((id (or id (elel--id-at-point)))) + (cl-pushnew (elel--filter "thread" id) elel-summary-extra :test #'equal) + (setq elel-summary-threaded t) + (let ((inhibit-read-only t)) + (elel--render-summary (elel--summary-notes))) + (elel--goto-note id))) + +(defun elel--summary-query (title query) + "Show a summary of the notes matching QUERY, named TITLE." + (elel--show-summary title :query query :threaded t)) + +(defun elel-summary-parent () + "Show the parent of the note at point." + (interactive) + (elel--show-parent (elel--note-at-point))) + +(defun elel-summary-children () + "Summarize the children of the note at point." + (interactive) + (elel--children (elel--id-at-point))) + +(defun elel-summary-outlinks () + "Summarize the notes that the note at point links to." + (interactive) + (elel--outlinks (elel--id-at-point))) + +(defun elel-summary-backlinks () + "Summarize the notes that link to the note at point." + (interactive) + (elel--backlinks (elel--id-at-point))) + +(defun elel-summary-hood () + "Summarize the neighbourhood of the note at point." + (interactive) + (elel-hood (elel--id-at-point) (prefix-numeric-value current-prefix-arg))) + +(defun elel-summary-edit () + "Edit the file of the note at point." + (interactive) + (elel-edit-note (elel--id-at-point))) + +(defun elel-summary-new-child () + "Write a new note whose parent is the note at point." + (interactive) + (elel-new-note (read-string "Title: ") (elel--id-at-point))) + +(defun elel-summary-quit () + "Leave the summary, and close the article window. +The summary and article buffers are killed, as in Gnus, so that quitting +the views buffer afterwards does not bring the summary back." + (interactive) + (let ((window (elel--article-visible-window)) + (summary (current-buffer)) + (article (get-buffer elel--article-buffer-name)) + (views (get-buffer "*elel*"))) + (cond + ((and window (cdr (window-list nil 'nomini))) (delete-window window))) + (cond + (views (switch-to-buffer views)) + ((quit-window))) + (cond ((buffer-live-p article) (kill-buffer article))) + (cond ((buffer-live-p summary) (kill-buffer summary))))) + + +;;; The same queries, from anywhere + +(defun elel--children (id) + "Summarize the children of the note ID." + (elel--summary-query (format "Children of %s" id) + (elel--filter "parent" id))) + +(defun elel--outlinks (id) + "Summarize the notes that the note ID links to." + (elel--summary-query (format "Links of %s" id) (elel--filter "from" id))) + +(defun elel--backlinks (id) + "Summarize the notes that link to the note ID." + (elel--summary-query (format "Backlinks of %s" id) (elel--filter "to" id))) + +(defun elel--show-parent (note) + "Show the parent of NOTE in the article window." + (let ((parent (elel--field note 'parent))) + (cond + (parent (elel--show-article parent t)) + ((user-error "This note has no parent"))))) + +;;;###autoload +(defun elel-search (query) + "Search the notes for QUERY and show the results. +QUERY may hold filters such as tag:X or date:2025..; see the README." + (interactive (list (read-string "Search: " nil 'elel-search-history))) + (elel--show-summary (format "Search: %s" query) + :query query :threaded nil)) + +(defun elel--hood-notes (id hops) + "The notes within HOPS steps of ID, with the center first." + (let* ((objects (elel--call "hood" id "--hops" (number-to-string hops))) + (center (alist-get 'center (car objects)))) + (cons (cons (cons 'rel "this") center) (cdr objects)))) + +;;;###autoload +(defun elel-hood (id &optional hops) + "Show the notes around ID, within HOPS steps (default 1)." + (interactive (list (cond + ((bound-and-true-p elel-article-mode) elel-article-id) + ((elel--id-at-point))) + (prefix-numeric-value current-prefix-arg))) + (let ((hops (or hops 1))) + (elel--show-summary + (format "Neighbourhood of %s" id) + :function (lambda () (elel--hood-notes id hops)) + :threaded nil))) + +;;; The article buffer + +(defvar-local elel-article-id nil + "The ID of the note shown in this article buffer.") + +(defvar-local elel-article-note nil + "The note shown in this article buffer, as the engine reported it.") + +(defvar elel--history nil + "The IDs of the notes visited before, newest first.") + +(defvar elel-article-mode-map (make-sparse-keymap) + "Keymap of `elel-article-mode'.") + +;; Filled in separately, so that loading this file again updates the keys. +(let ((map elel-article-mode-map)) + (define-key map (vector ?\s) #'elel-article-scroll-up) + (define-key map (vector ?j) #'elel-article-scroll-up) + (define-key map (vector ?\d) #'elel-article-scroll-down) + (define-key map (vector ?k) #'elel-article-scroll-down) + (define-key map (vector ?n) #'elel-article-next) + (define-key map (vector ?p) #'elel-article-previous) + (define-key map (vector ?\t) #'elel-article-next-link) + (define-key map (vector 'backtab) #'elel-article-previous-link) + (define-key map (vector ?\r) #'elel-article-follow) + (define-key map (vector ?l) #'elel-article-back) + (define-key map (vector ?^) #'elel-article-parent) + (define-key map (vector ?c) #'elel-article-children) + (define-key map (vector ?o) #'elel-article-outlinks) + (define-key map (vector ?b) #'elel-article-backlinks) + (define-key map (vector ?H) #'elel-hood) + (define-key map (vector ?a) #'elel-article-refer-thread) + (define-key map (vector ?=) #'elel-summary-expand-window) + (define-key map (vector ?R) #'elel-article-related) + (define-key map (vector ?S) #'elel-suggest) + (define-key map (vector ?h) #'elel-article-goto-summary) + (define-key map (vector ?q) #'elel-article-quit) + (define-key map (vector ?e) #'elel-article-edit) + (define-key map (vector ?r) #'elel-article-new-child) + (define-key map (vector ?g) #'elel-article-refresh) + (define-key map (vector ?/) #'elel-search) + (define-key map (vector ?N) #'elel-new-note)) + +(define-minor-mode elel-article-mode + "Minor mode for reading a note, like `view-mode' but aware of the summary. +The major mode of the buffer is chosen by the file of the note, through +`elel-major-mode-alist'. +\\{elel-article-mode-map}" + :lighter " elel" + :keymap elel-article-mode-map + (setq truncate-lines nil) + (setq buffer-read-only elel-article-mode)) + +(defun elel--major-mode (path) + "The major mode in which to show a note of the file PATH." + (let ((mode (assoc-default path elel-major-mode-alist + #'string-match-p 'text-mode))) + (cond + ((fboundp mode) mode) + ('text-mode)))) + +(defun elel--article-window () + "The window in which to show the article. +It is the window that shows the article already. Failing that, it is +a new window below the summary window if `elel-article-keep-summary'; +failing that, the selected window." + (let ((summary (get-buffer-window elel--summary-buffer-name))) + (cond + ((elel--article-visible-window)) + ((and elel-article-keep-summary summary + (ignore-errors + (let* ((total (window-total-height summary)) + (new (split-window summary nil 'below)) + (want (round (* elel-summary-window-height total)))) + (ignore-errors + (window-resize summary (- want (window-total-height summary)))) + new)))) + ((selected-window))))) + +(defun elel--hide-drawers () + "Fold the property drawers of the current org buffer." + (cond + ((fboundp 'org-fold-hide-drawer-all) (org-fold-hide-drawer-all)) + ((fboundp 'org-hide-drawer-all) (org-hide-drawer-all)))) + +(defun elel--show-article (id &optional history select) + "Show the note ID in the article window. +With HISTORY, remember the note shown before. With SELECT, select +the article window. The note is shown as it is in its file, in the +major mode chosen by `elel-major-mode-alist'." + (let* ((data (elel--get id)) + (note (alist-get 'note data)) + (text (or (alist-get 'raw data) (alist-get 'body data) "")) + (mode (elel--major-mode (elel--field note 'path))) + (buffer (get-buffer-create elel--article-buffer-name)) + (previous (buffer-local-value 'elel-article-id buffer))) + (cond + ((and history previous (not (equal previous id))) + (push previous elel--history))) + (with-current-buffer buffer + (let ((inhibit-read-only t)) + (kill-all-local-variables t) + (erase-buffer) + (insert text) + (cond + ((not (bolp)) (insert "\n"))) + (funcall mode) + (setq elel-article-id id) + (setq elel-article-note note) + (elel-article-mode 1) + (cond + ((derived-mode-p 'org-mode) (elel--hide-drawers)) + ;; Without org, make the links buttons. + ((elel--buttonize-body (point-min) (point-max)))) + (goto-char (point-min)) + (set-buffer-modified-p nil))) + (cond + ((alist-get 'stale data) + (message "elel: the file changed since the last index; update the index"))) + (let ((window (elel--article-window))) + (set-window-buffer window buffer) + (set-window-point window (point-min)) + (set-window-start window (point-min)) + (cond (select (select-window window)))) + buffer)) + +;;;###autoload +(defun elel-open-note (id) + "Show the note with ID in the article window, and select that window." + (interactive "sNote ID: ") + (elel--show-article id (bound-and-true-p elel-article-mode) t)) + +(defun elel--with-summary (function) + "Call FUNCTION in the summary window, or else in the summary buffer." + (let ((buffer (elel--summary-buffer))) + (cond + ((null buffer) (user-error "No summary; search first")) + ((get-buffer-window buffer) + (with-selected-window (get-buffer-window buffer) + (funcall function))) + ((with-current-buffer buffer (funcall function)))))) + +(defun elel-article-scroll-up () + "Scroll forward; at the end of the note, go to the next note." + (interactive) + (condition-case nil + (scroll-up-command) + (end-of-buffer (elel-article-next)))) + +(defun elel-article-scroll-down () + "Scroll backward." + (interactive) + (condition-case nil + (scroll-down-command) + (beginning-of-buffer (message "Beginning of the note")))) + +(defun elel-article-next () + "Show the next note of the summary." + (interactive) + (elel--with-summary #'elel-summary-next)) + +(defun elel-article-previous () + "Show the previous note of the summary." + (interactive) + (elel--with-summary #'elel-summary-previous)) + +(defun elel-article-next-link () + "Move to the next link." + (interactive) + (cond + ((derived-mode-p 'org-mode) (org-next-link)) + ((forward-button 1)))) + +(defun elel-article-previous-link () + "Move to the previous link." + (interactive) + (cond + ((derived-mode-p 'org-mode) (org-previous-link)) + ((backward-button 1)))) + +(defun elel-article-follow () + "Follow the link at point, in the article window." + (interactive) + (let ((id (elel--link-at-point))) + (cond + (id (elel--show-article id t)) + ((button-at (point)) (push-button)) + ((derived-mode-p 'org-mode) (call-interactively #'org-open-at-point)) + ((user-error "No link here"))))) + +(defun elel-article-back () + "Go back to the note shown before." + (interactive) + (cond + (elel--history (elel--show-article (pop elel--history))) + ((user-error "No previous note")))) + +(defun elel-article-parent () + "Show the parent of this note." + (interactive) + (elel--show-parent elel-article-note)) + +(defun elel-article-children () + "Summarize the children of this note." + (interactive) + (elel--children elel-article-id)) + +(defun elel-article-outlinks () + "Summarize the notes this note links to." + (interactive) + (elel--outlinks elel-article-id)) + +(defun elel-article-backlinks () + "Summarize the notes that link to this note." + (interactive) + (elel--backlinks elel-article-id)) + +(defun elel-article-refer-thread () + "Add the whole thread of this note to the summary." + (interactive) + (let ((id elel-article-id)) + (cond + ((elel--summary-buffer) + (elel--with-summary (lambda () (elel-summary-refer-thread id)))) + ((elel--summary-query (format "Thread of %s" id) + (elel--filter "thread" id)))))) + +(defun elel-article-goto-summary () + "Select the summary window." + (interactive) + (let ((buffer (elel--summary-buffer))) + (cond + ((null buffer) (user-error "No summary")) + ((get-buffer-window buffer) (select-window (get-buffer-window buffer))) + ((switch-to-buffer buffer))))) + +(defun elel-article-quit () + "Close the article window." + (interactive) + (let ((summary (elel--summary-buffer))) + (cond + ((and summary (not (get-buffer-window summary))) + (switch-to-buffer summary)) + ((cdr (window-list nil 'nomini)) (delete-window)) + ((bury-buffer))))) + +(defun elel-article-edit () + "Edit the file of this note." + (interactive) + (elel-edit-note elel-article-id)) + +(defun elel-article-new-child () + "Write a new note whose parent is this note." + (interactive) + (elel-new-note (read-string "Title: ") elel-article-id)) + +(defun elel-article-refresh () + "Read the note again from its file, as if opening it anew. +Point goes back to the start, like `elel-summary-show-article'." + (interactive) + (elel--show-article elel-article-id)) + +(defconst elel--body-link-regexp + (concat "\\[\\[note:" + ;; The kind, for typed links: [[note::KIND:ID]]. + "\\(?::\\([[:alnum:]_-]+\\):\\)?" + ;; The ID. + "\\([^] \t\n]+\\)\\]" + ;; The description, which is for humans. + "\\(?:\\[\\([^]\n]*\\)\\]\\)?" + "\\]") + "Matches a link to a note: [[note:ID][DESCRIPTION]]. +Group 1 is the kind (nil if untyped), group 2 the ID and group 3 the +description, if any. The typed form is [[note::KIND:ID][DESCRIPTION]].") + +(defun elel--open-button (button) + "Open the note of BUTTON." + (elel--show-article (button-get button 'elel-id) t)) + +(defun elel--buttonize-body (beg end) + "Replace the links between BEG and END by buttons. +A button shows the description of the link, or the ID if there is none." + (save-excursion + (let ((end (copy-marker end))) + (goto-char beg) + (while (re-search-forward elel--body-link-regexp end t) + (let* ((id (match-string-no-properties 2)) + (label (or (match-string-no-properties 3) id)) + (start (match-beginning 0))) + (replace-match label t t) + (make-text-button start (+ start (length label)) + 'elel-id id + 'action #'elel--open-button + 'follow-link t + 'face 'link)))))) + +(defun elel-edit-note (id) + "Visit the file of the note ID, at the note." + (interactive "sNote ID: ") + (let* ((data (elel--get id)) + (note (alist-get 'note data))) + (find-file (elel--field note 'path)) + (goto-char (point-min)) + (forward-line (1- (elel--field note 'line))) + (cond + ((derived-mode-p 'org-mode) (outline-show-entry))))) + +;;; The views buffer + +(defvar-local elel-views-show-hidden nil + "Non-nil if the views marked :hidden are listed.") + +(defvar elel-views-mode-map (make-sparse-keymap) + "Keymap of `elel-views-mode'.") + +;; Filled in separately, so that loading this file again updates the keys. +(let ((map elel-views-mode-map)) + (set-keymap-parent map special-mode-map) + (define-key map (vector ?\r) #'elel-views-open) + (define-key map (vector ?\s) #'elel-views-open) + (define-key map (vector ?n) #'next-line) + (define-key map (vector ?p) #'previous-line) + (define-key map (vector ?g) #'elel-views-refresh) + (define-key map (vector ?u) #'elel-views-update) + (define-key map (vector ?L) #'elel-views-toggle-hidden) + (define-key map (vector ?/) #'elel-search) + (define-key map (vector ?G) #'elel-search) + (define-key map (vector ?N) #'elel-new-note) + (define-key map (vector ?C) #'elel-check) + (define-key map (vector ?q) #'quit-window)) + +(define-derived-mode elel-views-mode special-mode "elel-views" + "Major mode for the list of views, like a Gnus group buffer. +\\{elel-views-mode-map}" + (setq truncate-lines t) + (hl-line-mode 1)) + +(defun elel--view-plist (view) + "The property list of the view VIEW, which is (NAME QUERY . PLIST)." + (cddr view)) + +(defun elel-views-refresh () + "Redraw the views buffer." + (interactive) + (let* ((inhibit-read-only t) + (line (line-number-at-pos)) + (shown (cl-remove-if + (lambda (v) + (and (plist-get (elel--view-plist v) :hidden) + (not elel-views-show-hidden))) + elel-views)) + (counts (and shown (elel--counts (mapcar #'cadr shown))))) + (erase-buffer) + (insert (propertize "elel" 'face 'elel-heading) + " " (abbreviate-file-name (expand-file-name elel-root)) + (cond (elel-views-show-hidden " (all views)") ("")) + "\n\n") + (cl-mapc + (lambda (v count) + (let* ((beg (point)) + (limit (plist-get (elel--view-plist v) :limit)) + (count (cond (limit (min count limit)) (count)))) + (insert (format "%6d %s%s\n" count (car v) + (cond + ((plist-get (elel--view-plist v) :hidden) + (propertize " (hidden)" 'face 'elel-date)) + ("")))) + (put-text-property beg (point) 'elel-view v))) + shown counts) + (goto-char (point-min)) + (forward-line (max 2 (1- line))))) + +(defun elel-views-toggle-hidden () + "List the hidden views, or hide them again." + (interactive) + (setq elel-views-show-hidden (not elel-views-show-hidden)) + (elel-views-refresh)) + +(defun elel-views-update () + "Update the index, then redraw." + (interactive) + (elel-index) + (elel-views-refresh)) + +(defun elel-views-open () + "Open the view at point." + (interactive) + (let ((v (get-text-property (point) 'elel-view))) + (cond + ((null v) (user-error "No view here")) + (t + (elel--show-summary (car v) + :query (cadr v) + :limit (plist-get (elel--view-plist v) :limit) + :threaded t))))) + +;;;###autoload +(defun elel () + "Show the views buffer." + (interactive) + (let ((buffer (get-buffer-create "*elel*"))) + (with-current-buffer buffer + (elel-views-mode) + (condition-case err + (elel-views-refresh) + (user-error + (cond + ;; The first time, there is no index yet: make one. + ((string-match-p "no index yet" (error-message-string err)) + (elel-index) + (elel-views-refresh)) + ((signal (car err) (cdr err))))))) + (switch-to-buffer buffer))) + +;;; Writing notes + +(defun elel-generate-id () + "Make an ID that sorts by time, such as 20261117T143005-7f3a." + (format "%s-%04x" (format-time-string "%Y%m%dT%H%M%S") (random #x10000))) + +(defun elel--now (&optional time) + "TIME (default: now) in ISO 8601, with a colon in the zone." + (let ((s (format-time-string "%Y-%m-%dT%H:%M:%S%z" time))) + (concat (substring s 0 -2) ":" (substring s -2)))) + +(defun elel--context () + "Where the user is: the file of the current buffer, if outside the notes." + (cond + ((and buffer-file-name + (not (string-prefix-p + (file-name-as-directory (expand-file-name elel-root)) + (expand-file-name buffer-file-name)))) + (abbreviate-file-name buffer-file-name)))) + +(defun elel--skeleton (format title id parent context) + "The text of a new note in FORMAT (\"org\" or \"txt\")." + (let ((date (elel--now))) + (cond + ((string= format "org") + (concat "* " title "\n:PROPERTIES:\n" + ":ID: " id "\n" + ":DATE: " date "\n" + (and parent (concat ":PARENT: " parent "\n")) + (and context (concat ":CONTEXT: " context "\n")) + ":END:\n")) + ((concat "ID: " id "\n" + "Title: " title "\n" + "Date: " date "\n" + (and parent (concat "Parent: " parent "\n")) + (and context (concat "Context: " context "\n")) + "\n"))))) + +(defvar elel-capture-mode-map (make-sparse-keymap) + "Keymap of `elel-capture-mode'.") + +;; Filled in separately, so that loading this file again updates the keys. +(let ((map elel-capture-mode-map)) + (define-key map (vector 3 3) #'elel-capture-finish)) + +(define-minor-mode elel-capture-mode + "Minor mode while writing a new note. +\\<elel-capture-mode-map>Type \\[elel-capture-finish] to save the note and update the index." + :lighter " elel" + :keymap elel-capture-mode-map) + +(defvar-local elel-capture-id nil + "The ID of the note being written in this buffer.") + +(defun elel-capture-finish () + "Save the new note, update the index, and leave the capture mode. +If `elel-suggest-after-capture' is non-nil, then show suggestions of +links and tags when the index is ready." + (interactive) + (let ((id elel-capture-id) + (marker (point-marker))) + (cond + ((and id elel-suggest-after-capture) + (push (lambda () (elel-suggest id marker t)) elel--index-callbacks) + (save-buffer) + (elel-capture-mode -1) + (elel-index-async) + (message "elel: saved; indexing, then suggestions")) + (t + (save-buffer) + (elel-capture-mode -1) + (message "elel: saved; indexing"))))) + +;;;###autoload +(defun elel-new-note (title &optional parent) + "Start a new note with TITLE, as a child of PARENT if given. +The note is appended to `elel-capture-file'." + (interactive (list (read-string "Title: ") nil)) + (let* ((file (expand-file-name elel-capture-file elel-root)) + (format (or (file-name-extension file) "org")) + (context (elel--context)) + (id (funcall elel-id-function))) + (make-directory (file-name-directory file) t) + (find-file file) + (goto-char (point-max)) + (cond + ((and (not (bobp)) (not (bolp))) (insert "\n"))) + (cond + ((> (buffer-size) 0) + (insert (cond ((string= format "txt") "\n\f\n") ("\n"))))) + (insert (elel--skeleton format title id parent context)) + (setq elel-capture-id id) + (elel-capture-mode 1) + (message "Write the note, then type C-c C-c"))) + +;;; Fixing missing IDs + +(defconst elel--txt-key-regexp + "^\\(id\\|title\\|date\\|kind\\|tags\\|parent\\|links\\|context\\|summary\\):" + "Matches the first line of the header of a plain text note.") + +(defun elel--insert-id (path line id) + "Give the note that starts at LINE of PATH the ID." + (with-current-buffer (find-file-noselect path) + (save-excursion + (goto-char (point-min)) + (forward-line (1- line)) + (cond + ((string= (file-name-extension path) "org") + (forward-line 1) + (while (looking-at "[ \t]*\\(SCHEDULED\\|DEADLINE\\|CLOSED\\):") + (forward-line 1)) + (cond + ((looking-at "[ \t]*:PROPERTIES:[ \t]*$") + (forward-line 1) + (insert ":ID: " id "\n")) + ((insert ":PROPERTIES:\n:ID: " id "\n:END:\n")))) + (t + (while (and (looking-at "[ \t]*$") (not (eobp))) + (forward-line 1)) + (let ((case-fold-search t)) + (cond + ((looking-at elel--txt-key-regexp) (insert "ID: " id "\n")) + ((insert "ID: " id "\n\n"))))))) + (current-buffer))) + +(defun elel-assign-ids () + "Add an ID to every note that has none." + (interactive) + (elel-index) + (let ((issues (cl-remove-if-not + (lambda (i) (equal (alist-get 'issue i) "missing-id")) + (elel--call "check")))) + (cond + ((null issues) (message "elel: every note has an ID")) + ((y-or-n-p (format "Add IDs to %d notes? " (length issues))) + ;; Work from the end of each file, so that line numbers stay valid. + (let ((sorted (sort (copy-sequence issues) + (lambda (a b) + (let ((pa (alist-get 'path a)) + (pb (alist-get 'path b))) + (cond + ((string< pa pb) t) + ((string= pa pb) + (> (alist-get 'line a) (alist-get 'line b))))))))) + (dolist (i sorted) + (let ((b (get-file-buffer (alist-get 'path i)))) + (cond + ((and b (buffer-modified-p b)) + (user-error "Save %s first" (buffer-name b)))))) + (let (buffers) + (dolist (i sorted) + (cl-pushnew (elel--insert-id (alist-get 'path i) + (alist-get 'line i) + (funcall elel-id-function)) + buffers)) + (dolist (b buffers) + (with-current-buffer b (save-buffer)))) + (elel-index)))))) + +;;; Checking + +(defun elel--check-button (button) + "Visit the place of BUTTON." + (find-file (button-get button 'path)) + (goto-char (point-min)) + (forward-line (1- (button-get button 'line)))) + +(defun elel-check () + "Report notes without IDs, duplicate IDs and dangling links." + (interactive) + (let ((issues (elel--call "check")) + (buffer (get-buffer-create "*elel check*"))) + (with-current-buffer buffer + (let ((inhibit-read-only t)) + (special-mode) + (erase-buffer) + (insert (propertize "elel check" 'face 'elel-heading) + (format " (%d)\n\n" (length issues))) + (dolist (i issues) + (insert (format "%-16s " (alist-get 'issue i))) + (insert-text-button + (or (alist-get 'title i) "(untitled)") + 'path (alist-get 'path i) + 'line (alist-get 'line i) + 'action #'elel--check-button + 'follow-link t) + (cond + ((alist-get 'target i) + (insert " -> " (alist-get 'target i)))) + (insert "\n")) + (goto-char (point-min)))) + (switch-to-buffer buffer))) + +;;; Links in the note files +;; +;; A link to a note is written [[note:ID][TITLE]] in every format, and +;; [[note::KIND:ID][TITLE]] when it has a type. The title is for the +;; reader; the ID is how the engine finds the note. In org, "note" is a +;; registered link type, so org follows, completes and exports these +;; links itself (see the end of this section). + +(defun elel--note-id-here () + "The ID of the note that the current buffer or point is about, or nil." + (cond + (elel-capture-id) + ((derived-mode-p 'elel-summary-mode) (get-text-property (point) 'elel-id)) + ((bound-and-true-p elel-article-mode) elel-article-id) + ((derived-mode-p 'org-mode) (org-entry-get nil "ID" t)))) + +(defun elel--choose-note () + "Ask for a note by title, and return it. +The notes closest in meaning to the current note come first." + (let* ((here (ignore-errors (elel--note-id-here))) + (near (and here + (ignore-errors + (elel--search (format "like:\"%s\"" here) 50)))) + (notes (append near + (cl-remove-if + (lambda (n) + (cl-find (elel--field n 'id) near + :key (lambda (x) (elel--field x 'id)) + :test #'equal)) + (elel--search "")))) + (table (mapcar + (lambda (n) + (cons (format "%s (%s)" + (elel--field n 'title) + (string-trim (elel--short-date n))) + n)) + notes))) + (cond + (table + (cdr (assoc (completing-read + "Link to: " + (lambda (string pred action) + (cond + ((eq action 'metadata) + (list 'metadata + (cons 'display-sort-function #'identity) + (cons 'cycle-sort-function #'identity))) + ((complete-with-action action table string pred)))) + nil t) + table))) + ((user-error "No notes in the index"))))) + +(defun elel--link-string (id description &optional kind) + "The text of a link to the note ID, with DESCRIPTION and maybe KIND." + (concat "[[note:" (and kind (concat ":" kind ":")) id "]" + (and description (concat "[" description "]")) + "]")) + +(defun elel-insert-link (&optional typed) + "Insert a link to a note, chosen by title. +If the region is active, its text becomes the description of the +link; otherwise the title of the note does. With a prefix argument +TYPED, also ask for the type of the link." + (interactive "P") + (let* ((note (elel--choose-note)) + (kind (and typed + (let ((k (string-trim (read-string "Type of link: ")))) + (and (not (string-empty-p k)) k)))) + (description + (cond + ((use-region-p) + (prog1 (buffer-substring-no-properties + (region-beginning) (region-end)) + (delete-region (region-beginning) (region-end)))) + ((elel--field note 'title))))) + (insert (elel--link-string (elel--field note 'id) description kind)))) + +(defun elel--link-at-point () + "The ID of the note link at point, or nil." + (let ((pos (point)) + found) + (save-excursion + (beginning-of-line) + (while (and (not found) + (re-search-forward elel--body-link-regexp + (line-end-position) t)) + (cond + ((and (<= (match-beginning 0) pos) (<= pos (match-end 0))) + (setq found (match-string-no-properties 2)))))) + found)) + +(defun elel-follow-link () + "Open the note linked at point. +If there is no note link at point, do what \\`C-c C-o' normally does." + (interactive) + (let ((id (elel--link-at-point))) + (cond + (id (elel-open-note id)) + ((derived-mode-p 'org-mode) (call-interactively #'org-open-at-point)) + ((user-error "No note link at point"))))) + +(defun elel-open-this-note () + "Open the note around point in the note buffer." + (interactive) + (let ((id (save-excursion + (cond + ((string= (file-name-extension (or buffer-file-name "")) "org") + (org-back-to-heading t) + (org-entry-get nil "ID")) + ((progn + (cond ((search-backward "\f" nil t) (forward-line 1)) + ((goto-char (point-min)))) + (let ((case-fold-search t)) + (re-search-forward "^id:[ \t]*\\(.+\\)$" + (line-end-position 8) t))) + (match-string-no-properties 1)))))) + (cond + (id (elel-open-note (string-trim id))) + ((user-error "No ID for this note"))))) + +(defvar elel-link-mode-map (make-sparse-keymap) + "Keymap of `elel-link-mode'.") + +;; Filled in separately, so that loading this file again updates the keys. +(let ((map elel-link-mode-map)) + (define-key map (vector 3 15) #'elel-follow-link) + (define-key map (vector 3 ?n ?l) #'elel-insert-link) + (define-key map (vector 3 ?n ?o) #'elel-open-this-note)) + +(define-minor-mode elel-link-mode + "Minor mode for the note files: follow and insert note links. +\\{elel-link-mode-map}" + :lighter " elel" + :keymap elel-link-mode-map) + +(defun elel--maybe-link-mode () + "Turn on `elel-link-mode' in org and text files under `elel-root'." + (cond + ((and buffer-file-name + (member (file-name-extension buffer-file-name) (list "org" "txt")) + (string-prefix-p + (file-name-as-directory (expand-file-name elel-root)) + (expand-file-name buffer-file-name))) + (elel-link-mode 1)))) + +(add-hook 'find-file-hook #'elel--maybe-link-mode) + +;;;; The org link type + +(defun elel--parse-path (path) + "Split the PATH of a note link into (KIND . ID). +KIND is nil for an untyped link, whose path is just the ID." + (cond + ((string-match "\\`:\\([[:alnum:]_-]+\\):\\(.+\\)\\'" path) + (cons (match-string 1 path) (match-string 2 path))) + ((cons nil path)))) + +(defun elel--org-follow (path _arg) + "Open the note of the org link whose path is PATH." + (elel-open-note (cdr (elel--parse-path path)))) + +(defun elel--org-complete (&optional _arg) + "Choose a note by title and return the path of a link to it." + (concat "note:" (elel--field (elel--choose-note) 'id))) + +(defun elel--org-description (link desc) + "The description for LINK, if DESC is nil: the title of the note." + (or desc + (ignore-errors + (let* ((id (cdr (elel--parse-path (string-remove-prefix "note:" link)))) + (data (car (elel--call "get" id)))) + (elel--field (alist-get 'note data) 'title))))) + +(defun elel--org-export (path description _backend _info) + "Export the link to PATH as just its DESCRIPTION." + (or description (cdr (elel--parse-path path)))) + +(with-eval-after-load 'org + (org-link-set-parameters + "note" + :follow #'elel--org-follow + :complete #'elel--org-complete + :insert-description #'elel--org-description + :export #'elel--org-export)) + +(provide 'elel) +;;; elel.el ends here diff --git a/src/adapter/mod.rs b/src/adapter/mod.rs new file mode 100644 index 0000000..8f56b9b --- /dev/null +++ b/src/adapter/mod.rs @@ -0,0 +1,275 @@ +//! Format adapters. Each adapter turns the text of a file into notes. + +pub mod org; +pub mod txt; + +use crate::note::{Link, Note}; +use std::path::Path; + +/// Parse the file at `path`, whose content is `text`. Returns `None` +/// when no adapter handles the extension. +pub fn parse_file(path: &Path, text: &str) -> Option<Vec<Note>> { + let p = path.to_string_lossy().to_string(); + match path.extension().and_then(|e| e.to_str()) { + Some("org") => Some(org::parse(&p, text)), + Some("txt") => Some(txt::parse(&p, text)), + _ => None, + } +} + +/// The fields every adapter collects before building a note. +#[derive(Default)] +pub struct Fields { + pub id: Option<String>, + pub title: String, + pub date: Option<String>, + pub kind: Option<String>, + pub tags: Vec<String>, + pub parent: Option<String>, + pub links: Option<String>, + pub context: Option<String>, + pub summary: Option<String>, +} + +impl Fields { + /// Store the value of a metadata key. Returns false for unknown keys. + pub fn set(&mut self, key: &str, value: &str) -> bool { + let value = value.trim(); + let some = |v: &str| { + if v.is_empty() { + None + } else { + Some(v.to_string()) + } + }; + match key.to_ascii_lowercase().as_str() { + "id" => self.id = some(value), + "title" => self.title = value.to_string(), + "date" => self.date = some(value), + "kind" => self.kind = some(value), + "tags" => { + for t in split_list(value) { + if !self.tags.contains(&t) { + self.tags.push(t); + } + } + } + "parent" => self.parent = some(value), + "links" => self.links = some(value), + "context" => self.context = some(value), + "summary" => self.summary = some(value), + _ => return false, + } + true + } +} + +/// Assemble a note from its fields, its body, and its place in the file. +#[allow(clippy::too_many_arguments)] +pub fn build( + path: &str, + text: &str, + start: usize, + end: usize, + line: usize, + fields: Fields, + body: String, +) -> Note { + let mut links = fields + .links + .as_deref() + .map(parse_links_prop) + .unwrap_or_default(); + for l in body_links(&body) { + if !links.iter().any(|x| x.dst == l.dst) { + links.push(l); + } + } + Note { + id: fields.id, + path: path.to_string(), + start, + end, + line, + hash: blake3::hash(text[start..end].as_bytes()).to_hex().to_string(), + title: fields.title, + date: fields.date, + kind: fields.kind, + tags: fields.tags, + parent: fields.parent, + links, + context: fields.context, + summary: fields.summary, + body, + } +} + +/// Split on whitespace, commas and colons. +pub fn split_list(s: &str) -> Vec<String> { + s.split(|c: char| c.is_whitespace() || c == ',' || c == ':') + .filter(|t| !t.is_empty()) + .map(|t| t.to_string()) + .collect() +} + +fn is_kind(s: &str) -> bool { + !s.is_empty() + && s + .chars() + .all(|c| c.is_ascii_alphabetic() || c == '-' || c == '_') +} + +/// Parse the value of a LINKS property: tokens `ID` or `kind:ID`. +pub fn parse_links_prop(s: &str) -> Vec<Link> { + let mut out = Vec::new(); + for tok in s.split_whitespace() { + let link = match tok.split_once(':') { + Some(("notes" | "note", rest)) if !rest.is_empty() => Link { + dst: rest.to_string(), + kind: None, + }, + Some((k, rest)) if is_kind(k) && !rest.is_empty() => Link { + dst: rest.to_string(), + kind: Some(k.to_string()), + }, + _ => Link { + dst: tok.to_string(), + kind: None, + }, + }; + out.push(link); + } + out +} + +/// Split the path of a `note:` link into its kind and its ID. +/// +/// The path is `ID` for an untyped link and `:KIND:ID` for a typed one, +/// so `[[note:ID]]` and `[[note::KIND:ID]]` are the two forms. +fn split_note_path(p: &str) -> Option<(Option<String>, String)> { + if p.is_empty() || p.contains(char::is_whitespace) { + return None; + } + match p.strip_prefix(':') { + Some(rest) => { + let (k, id) = rest.split_once(':')?; + let ok = !k.is_empty() + && k.chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + (ok && !id.is_empty()).then(|| (Some(k.to_string()), id.to_string())) + } + None => Some((None, p.to_string())), + } +} + +/// Find the `[[note:ID][description]]` links in a body, and their kinds. +/// The description is for humans (usually the title of the note); only +/// the ID is used to locate the note. +pub fn body_links(body: &str) -> Vec<Link> { + let mut out: Vec<Link> = Vec::new(); + let mut pos = 0; + while let Some(i) = body[pos..].find("[[note:") { + let start = pos + i + "[[note:".len(); + let rest = &body[start..]; + match rest.find(']') { + Some(j) if !rest[..j].contains('\n') => { + let closes = rest[j..].starts_with("]]") || rest[j..].starts_with("]["); + if closes { + if let Some((kind, dst)) = split_note_path(&rest[..j]) { + match out.iter_mut().find(|l| l.dst == dst) { + // A typed link is more informative than an untyped one. + Some(l) => l.kind = l.kind.take().or(kind), + None => out.push(Link { dst, kind }), + } + } + } + pos = start + j; + } + _ => pos = start, + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn body_links_read_the_id_and_ignore_the_description() { + let l = body_links( + "see [[note:abc-1][A title: with colon]] and [[note:<x.y@z>][d]] or [[note:q]] twice [[note:q][again]]", + ); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!(got, vec![("abc-1", None), ("<x.y@z>", None), ("q", None)]); + } + + #[test] + fn body_links_may_be_typed() { + let l = body_links("[[note::supports:a1][A]] [[note::contra:<b@c>]] [[note:d]] [[note:d2][x]]"); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!( + got, + vec![("a1", Some("supports")), ("<b@c>", Some("contra")), ("d", None), ("d2", None)] + ); + // The typed form wins when the same note is linked twice. + let l = body_links("[[note:a]] and [[note::cites:a]]"); + assert_eq!(l.len(), 1); + assert_eq!(l[0].kind.as_deref(), Some("cites")); + } + + #[test] + fn body_links_ignore_other_links() { + let body = "[[https://a.b][web]] [[file:x.org]] [[*heading]] [[#custom]] [[./rel][r]] \\ + [[two words]] [[n1][fuzzy]] [[note:]] [[note:two words]] [[note::bad]] \\ + [[notes:x]] [[note:a\nb]]"; + assert!(body_links(body).is_empty(), "{:?}", body_links(body)); + } + + #[test] + fn links_property_has_typed_and_plain_tokens() { + let l = parse_links_prop("a1 supports:b2 notes:c3 <d@e>"); + let got: Vec<(&str, Option<&str>)> = + l.iter().map(|x| (x.dst.as_str(), x.kind.as_deref())).collect(); + assert_eq!( + got, + vec![("a1", None), ("b2", Some("supports")), ("c3", None), ("<d@e>", None)] + ); + } + + #[test] + fn org_notes_are_top_level_headings_only() { + let text = "intro\n* One :a:b:\n:PROPERTIES:\n:ID: 1\n:PARENT: 0\n:END:\nbody\n** sub\nmore\n* Two\ntext\n"; + let notes = org::parse("/x.org", text); + assert_eq!(notes.len(), 2); + assert_eq!(notes[0].id.as_deref(), Some("1")); + assert_eq!(notes[0].title, "One"); + assert_eq!(notes[0].tags, vec!["a", "b"]); + assert_eq!(notes[0].parent.as_deref(), Some("0")); + assert!(notes[0].body.contains("** sub")); + assert_eq!(notes[0].line, 2); + assert_eq!(&text[notes[0].start..notes[0].end].lines().next(), &Some("* One :a:b:")); + assert_eq!(notes[1].id, None); + } + + #[test] + fn txt_notes_are_split_at_form_feeds() { + let text = "ID: 1\nTitle: A\nTags: x y\n\nbody a\n\x0c\nID: 2\nTitle: B\n\nbody b\n\x0c\n\n"; + let notes = txt::parse("/x.txt", text); + assert_eq!(notes.len(), 2); + assert_eq!(notes[0].tags, vec!["x", "y"]); + assert_eq!(notes[0].body, "body a"); + assert_eq!(notes[1].id.as_deref(), Some("2")); + assert_eq!(notes[1].line, 7); + } + + #[test] + fn txt_note_without_header_takes_its_title_from_the_text() { + let notes = txt::parse("/x.txt", "just a thought\nmore\n"); + assert_eq!(notes.len(), 1); + assert_eq!(notes[0].title, "just a thought"); + assert_eq!(notes[0].id, None); + } +} diff --git a/src/adapter/org.rs b/src/adapter/org.rs new file mode 100644 index 0000000..a999814 --- /dev/null +++ b/src/adapter/org.rs @@ -0,0 +1,84 @@ +//! Org adapter: each top-level heading is a note. Metadata lives in +//! the heading (title, tags) and in its property drawer. + +use super::{Fields, build}; +use crate::note::Note; + +fn is_tag_string(s: &str) -> bool { + s.len() > 2 + && s.starts_with(':') + && s.ends_with(':') + && s.chars() + .all(|c| c.is_alphanumeric() || "_@#%:.".contains(c)) +} + +/// Split a heading line into its title and its tags. +fn parse_heading(line: &str) -> (String, Vec<String>) { + let h = line.trim_start_matches('*').trim(); + if let Some(idx) = h.rfind(char::is_whitespace) { + let last = &h[idx + 1..]; + if is_tag_string(last) { + let tags = last + .split(':') + .filter(|t| !t.is_empty()) + .map(|t| t.to_string()) + .collect(); + return (h[..idx].trim().to_string(), tags); + } + } + (h.to_string(), Vec::new()) +} + +fn is_planning(line: &str) -> bool { + let t = line.trim_start(); + t.starts_with("SCHEDULED:") || t.starts_with("DEADLINE:") || t.starts_with("CLOSED:") +} + +pub fn parse(path: &str, text: &str) -> Vec<Note> { + // Find the top-level headings: a single star at column 0. + let mut heads: Vec<(usize, usize)> = Vec::new(); + let mut off = 0; + for (ln, line) in text.split_inclusive('\n').enumerate() { + let t = line.trim_end(); + if line.starts_with("* ") || t == "*" { + heads.push((off, ln + 1)); + } + off += line.len(); + } + let mut notes = Vec::new(); + for (i, &(start, line_no)) in heads.iter().enumerate() { + let end = heads.get(i + 1).map(|h| h.0).unwrap_or(text.len()); + let chunk = &text[start..end]; + let mut lines = chunk.lines(); + let heading = lines.next().unwrap_or(""); + let (title, tags) = parse_heading(heading); + let mut fields = Fields { + title, + tags, + ..Default::default() + }; + let mut rest: Vec<&str> = lines.collect(); + // Skip planning lines, then read the property drawer, if any. + let mut k = 0; + while k < rest.len() && is_planning(rest[k]) { + k += 1; + } + let mut body_from = 0; + if k < rest.len() && rest[k].trim() == ":PROPERTIES:" { + let mut j = k + 1; + while j < rest.len() && rest[j].trim() != ":END:" { + let l = rest[j].trim(); + if let Some(r) = l.strip_prefix(':') { + if let Some((key, val)) = r.split_once(':') { + fields.set(key, val); + } + } + j += 1; + } + body_from = (j + 1).min(rest.len()); + } + let body = rest.split_off(body_from).join("\n"); + notes.push(build(path, text, start, end, line_no, fields, body)); + } + notes +} diff --git a/src/adapter/txt.rs b/src/adapter/txt.rs new file mode 100644 index 0000000..ddea799 --- /dev/null +++ b/src/adapter/txt.rs @@ -0,0 +1,85 @@ +//! Plain text adapter. Notes are separated by a form-feed line. A +//! note starts with `Key: value` lines, then a blank line, then the +//! body. + +use super::{Fields, build}; +use crate::note::Note; + +const KEYS: [&str; 9] = [ + "id", "title", "date", "kind", "tags", "parent", "links", "context", "summary", +]; + +/// Parse `Key: value` if the key is a known one. +fn header_line(line: &str) -> Option<(&str, &str)> { + let (k, v) = line.split_once(':')?; + let lk = k.trim().to_ascii_lowercase(); + if KEYS.contains(&lk.as_str()) && k.trim().len() == k.len() { + Some((k, v)) + } else { + None + } +} + +pub fn parse(path: &str, text: &str) -> Vec<Note> { + // Split into chunks at the form-feed lines. + let mut chunks: Vec<(usize, usize, usize)> = Vec::new(); // start, end, line + let mut off = 0; + let mut cur_start = 0; + let mut cur_line = 1; + for (ln, line) in text.split_inclusive('\n').enumerate() { + if line.trim_end_matches(['\r', '\n']) == "\x0c" { + chunks.push((cur_start, off, cur_line)); + cur_start = off + line.len(); + cur_line = ln + 2; + } + off += line.len(); + } + chunks.push((cur_start, text.len(), cur_line)); + + let mut notes = Vec::new(); + for (start, end, line_no) in chunks { + let chunk = &text[start..end]; + if chunk.trim().is_empty() { + continue; + } + let mut fields = Fields::default(); + let mut body_lines: Vec<&str> = Vec::new(); + let mut in_header = true; + let mut seen = 0; + for l in chunk.lines() { + if in_header { + if seen == 0 && l.trim().is_empty() { + continue; + } + if let Some((k, v)) = header_line(l) { + fields.set(k, v); + seen += 1; + continue; + } + in_header = false; + // The blank line that ends the header is not part of the body. + if seen > 0 && l.trim().is_empty() { + continue; + } + } + body_lines.push(l); + } + if fields.title.is_empty() { + fields.title = body_lines + .iter() + .find(|l| !l.trim().is_empty()) + .map(|l| l.trim().chars().take(80).collect()) + .unwrap_or_default(); + } + notes.push(build( + path, + text, + start, + end, + line_no, + fields, + body_lines.join("\n"), + )); + } + notes +} diff --git a/src/chunk.rs b/src/chunk.rs new file mode 100644 index 0000000..b17316b --- /dev/null +++ b/src/chunk.rs @@ -0,0 +1,314 @@ +//! Cut a note into chunks for the embeddings. +//! +//! A chunk is a paragraph or a few paragraphs. Blocks that must stay +//! whole (source blocks, `\begin..\end`, `$$..$$`) are never cut. A +//! paragraph that is only a formula is merged into a neighbour, because +//! alone it would be embedded as "some mathematics" and look like every +//! other formula. Each chunk is prefixed with the title, so a bare +//! "this follows from the lemma" still knows what it is about. + +/// Paragraphs shorter than this are merged with a neighbour. +const MIN: usize = 200; +/// A chunk is not grown beyond this (a single paragraph may exceed it). +const MAX: usize = 1500; +/// A paragraph with fewer words than this, once the math is removed, +/// is a formula paragraph. +const MIN_WORDS: usize = 5; + +#[derive(Debug, Clone, PartialEq)] +pub struct Chunk { + pub text: String, + /// 0-based line of the body where the chunk starts. + pub line: usize, +} + +/// A paragraph of the body: its text and its first line. +struct Para { + text: String, + line: usize, +} + +/// Replace `[[note:ID][desc]]` by `desc` and org links by their text. +pub fn strip_links(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find("[[") { + out.push_str(&rest[..i]); + let tail = &rest[i + 2..]; + match tail.find("]]") { + Some(j) => { + let inner = &tail[..j]; + match inner.split_once("][") { + Some((_, desc)) => out.push_str(desc), + None => out.push_str(inner), + } + rest = &tail[j + 2..]; + } + None => { + out.push_str(&rest[i..]); + rest = ""; + } + } + } + out.push_str(rest); + out +} + +/// The text with math (`$..$`, `\(..\)`, `\[..\]`, `\begin..\end`) removed. +fn without_math(s: &str) -> String { + let mut out = String::new(); + let mut rest = s; + 'outer: while !rest.is_empty() { + for (open, close) in [ + ("$$", "$$"), + ("\\[", "\\]"), + ("\\(", "\\)"), + ("$", "$"), + ] { + if let Some(r) = rest.strip_prefix(open) { + match r.find(close) { + Some(j) => rest = &r[j + close.len()..], + None => rest = "", + } + out.push(' '); + continue 'outer; + } + } + if rest.starts_with("\\begin{") { + match rest.find("\\end{") { + Some(j) => { + let after = &rest[j..]; + rest = match after.find('}') { + Some(k) => &after[k + 1..], + None => "", + }; + out.push(' '); + continue 'outer; + } + None => break, + } + } + let c = rest.chars().next().unwrap(); + out.push(c); + rest = &rest[c.len_utf8()..]; + } + out +} + +/// Does the paragraph hold almost nothing but math? +fn is_formula(p: &str) -> bool { + let t = p.trim(); + if t.is_empty() { + return false; + } + let has_math = t.contains('$') || t.contains("\\[") || t.contains("\\(") || t.contains("\\begin{"); + if !has_math { + return false; + } + let left = without_math(t); + // CJK has no spaces: count its characters as words. + let words = left + .split(|c: char| !c.is_alphanumeric()) + .filter(|w| !w.is_empty()) + .map(|w| if w.chars().any(|c| c as u32 > 0x2E80) { w.chars().count() } else { 1 }) + .sum::<usize>(); + words < MIN_WORDS +} + +/// Split into paragraphs, keeping blocks whole. +fn paragraphs(body: &str) -> Vec<Para> { + let mut out: Vec<Para> = Vec::new(); + let mut cur: Vec<&str> = Vec::new(); + let mut start = 0; + // Inside a block that ends with this line (lowercased, trimmed prefix). + let mut block_end: Option<String> = None; + let mut dollars = false; + let flush = |cur: &mut Vec<&str>, start: usize, out: &mut Vec<Para>| { + if !cur.is_empty() { + out.push(Para { text: cur.join("\n"), line: start }); + cur.clear(); + } + }; + for (i, l) in body.lines().enumerate() { + let t = l.trim(); + let tl = t.to_lowercase(); + if let Some(end) = &block_end { + cur.push(l); + if tl.starts_with(end.as_str()) { + block_end = None; + } + continue; + } + if dollars { + cur.push(l); + if t.contains("$$") { + dollars = false; + } + continue; + } + if t.is_empty() { + flush(&mut cur, start, &mut out); + continue; + } + let heading = l.starts_with('*') && l.trim_start_matches('*').starts_with(' '); + if heading { + flush(&mut cur, start, &mut out); + } + if cur.is_empty() { + start = i; + } + if let Some(rest) = tl.strip_prefix("#+begin_") { + let name = rest.split_whitespace().next().unwrap_or(""); + block_end = Some(format!("#+end_{}", name)); + } else if let Some(rest) = t.strip_prefix("\\begin{") { + let name = rest.split('}').next().unwrap_or(""); + block_end = Some(format!("\\end{{{}}}", name.to_lowercase())); + } else if t.matches("$$").count() % 2 == 1 { + dollars = true; + } + // A one-line block (\begin{x} .. \end{x}) closes at once. + if let Some(end) = &block_end { + if tl[1..].contains(end.as_str()) { + block_end = None; + } + } + cur.push(l); + } + flush(&mut cur, start, &mut out); + out +} + +/// The chunks of a note. A note without a body gets one chunk from its +/// title, tags and summary. +pub fn chunks(title: &str, tags: &[String], summary: Option<&str>, body: &str) -> Vec<Chunk> { + let body = strip_links(body); + let paras = paragraphs(&body); + // Merge formula paragraphs into the previous paragraph (or the next one). + let mut merged: Vec<Para> = Vec::new(); + let mut carry: Option<Para> = None; + for p in paras { + let formula = is_formula(&p.text); + let p = match carry.take() { + Some(c) => Para { text: format!("{}\n{}", c.text, p.text), line: c.line }, + None => p, + }; + if formula && merged.is_empty() { + // Opens the note: it joins the next paragraph. + carry = Some(p); + } else if formula { + let last = merged.last_mut().unwrap(); + last.text = format!("{}\n{}", last.text, p.text); + } else { + merged.push(p); + } + } + if let Some(c) = carry { + merged.push(c); + } + // Merge small paragraphs, up to MAX. + let mut out: Vec<Chunk> = Vec::new(); + let mut small = false; + for p in merged { + match out.last_mut() { + Some(last) if (small || p.text.len() < MIN) && last.text.len() + p.text.len() < MAX => { + last.text.push_str("\n\n"); + last.text.push_str(&p.text); + small = last.text.len() < MIN; + } + _ => { + small = p.text.len() < MIN; + out.push(Chunk { text: p.text, line: p.line }); + } + } + } + if out.is_empty() { + let mut t = String::new(); + if let Some(s) = summary { + t.push_str(s); + } + if !tags.is_empty() { + t.push_str("\n"); + t.push_str(&tags.join(" ")); + } + out.push(Chunk { text: t, line: 0 }); + } + for c in &mut out { + c.text = format!("{}\n\n{}", title, c.text.trim()); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn long(s: &str) -> String { + format!("{} {}", s, "word ".repeat(60)) + } + + #[test] + fn formula_paragraph_joins_the_previous_one() { + let body = format!("{}\n\n\\[ x^2 + y^2 = z^2 \\]\n\n{}", long("First"), long("Second")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 2, "{:?}", c); + assert!(c[0].text.contains("x^2")); + assert!(!c[1].text.contains("x^2")); + } + + #[test] + fn formula_at_the_start_joins_the_next() { + let body = format!("$$a=b$$\n\n{}", long("Prose")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("a=b") && c[0].text.contains("Prose")); + } + + #[test] + fn prose_with_inline_math_is_not_a_formula() { + assert!(!is_formula("Let $x$ be a point on the curve and consider its tangent.")); + assert!(is_formula("\\[ \\int_0^1 f \\]")); + assert!(is_formula("$a$ and $b$")); + } + + #[test] + fn source_blocks_stay_whole() { + let body = format!("{}\n\n#+begin_src elisp\n(a)\n\n(b)\n#+end_src\n\n{}", long("One"), long("Two")); + let p = paragraphs(&body); + assert_eq!(p.len(), 3, "{:?}", p.iter().map(|x| &x.text).collect::<Vec<_>>()); + assert!(p[1].text.contains("(b)")); + } + + #[test] + fn environment_stays_whole() { + let body = "text\n\n\\begin{align}\na\n\nb\n\\end{align}\n\nmore"; + let p = paragraphs(body); + assert_eq!(p.len(), 3); + assert!(p[1].text.contains("\\end{align}")); + } + + #[test] + fn small_paragraphs_are_merged() { + let c = chunks("T", &[], None, "a short one\n\nanother short\n\nand a third"); + assert_eq!(c.len(), 1); + assert!(c[0].text.starts_with("T\n\n")); + } + + #[test] + fn links_become_their_description() { + assert_eq!(strip_links("see [[note:a1][Foo]] and [[note:a2]]"), "see Foo and note:a2"); + } + + #[test] + fn empty_body_uses_title_and_tags() { + let c = chunks("Title", &["x".into()], Some("sum"), ""); + assert_eq!(c.len(), 1); + assert!(c[0].text.contains("sum") && c[0].text.contains("x")); + } + + #[test] + fn line_numbers_point_at_the_paragraph() { + let body = format!("{}\n\n{}\n\n{}", long("A"), long("B"), long("C")); + let c = chunks("T", &[], None, &body); + assert_eq!(c.iter().map(|x| x.line).collect::<Vec<_>>(), vec![0, 2, 4]); + } +} diff --git a/src/graph.rs b/src/graph.rs new file mode 100644 index 0000000..460bdec --- /dev/null +++ b/src/graph.rs @@ -0,0 +1,187 @@ +//! The manifest (what was indexed) and the graph built from it. +//! +//! The manifest is a small JSON file. The graph is rebuilt in memory +//! on every call; with thousands of notes this takes microseconds. + +use crate::note::Note; +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::path::Path; + +#[derive(Serialize, Deserialize)] +pub struct FileEntry { + pub hash: String, + pub notes: Vec<Note>, +} + +#[derive(Default, Serialize, Deserialize)] +pub struct Manifest { + pub files: BTreeMap<String, FileEntry>, +} + +impl Manifest { + pub fn load(path: &Path) -> Manifest { + std::fs::read_to_string(path) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default() + } + + /// Write through a temporary file so a crash cannot leave half a file. + pub fn save(&self, path: &Path) -> Result<()> { + let tmp = path.with_extension("json.tmp"); + std::fs::write(&tmp, serde_json::to_vec(self)?)?; + std::fs::rename(&tmp, path)?; + Ok(()) + } +} + +/// How a note is reached from the note we started at. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Rel { + Parent, + Child, + Link, + Backlink, +} + +impl Rel { + pub fn name(self) -> &'static str { + match self { + Rel::Parent => "parent", + Rel::Child => "child", + Rel::Link => "link", + Rel::Backlink => "backlink", + } + } +} + +pub struct Graph { + pub notes: Vec<Note>, + by_id: HashMap<String, usize>, + children: HashMap<String, Vec<usize>>, + backlinks: HashMap<String, Vec<usize>>, + pub duplicates: Vec<(String, Vec<usize>)>, +} + +impl Graph { + pub fn build(m: &Manifest) -> Graph { + let notes: Vec<Note> = m.files.values().flat_map(|f| f.notes.clone()).collect(); + let mut by_id: HashMap<String, usize> = HashMap::new(); + let mut seen: HashMap<String, Vec<usize>> = HashMap::new(); + for (i, n) in notes.iter().enumerate() { + if let Some(id) = &n.id { + seen.entry(id.clone()).or_default().push(i); + by_id.entry(id.clone()).or_insert(i); + } + } + let mut duplicates: Vec<(String, Vec<usize>)> = + seen.into_iter().filter(|(_, v)| v.len() > 1).collect(); + duplicates.sort(); + let mut children: HashMap<String, Vec<usize>> = HashMap::new(); + let mut backlinks: HashMap<String, Vec<usize>> = HashMap::new(); + for (i, n) in notes.iter().enumerate() { + if n.id.is_none() { + continue; + } + if let Some(p) = &n.parent { + children.entry(p.clone()).or_default().push(i); + } + for l in &n.links { + backlinks.entry(l.dst.clone()).or_default().push(i); + } + } + Graph { + notes, + by_id, + children, + backlinks, + duplicates, + } + } + + pub fn get(&self, id: &str) -> Option<&Note> { + self.by_id.get(id).map(|&i| &self.notes[i]) + } + + pub fn parent_of(&self, id: &str) -> Option<&Note> { + self.get(id) + .and_then(|n| n.parent.as_deref()) + .and_then(|p| self.get(p)) + } + + pub fn children_of(&self, id: &str) -> Vec<&Note> { + self.idx(self.children.get(id)) + } + + pub fn backlinks_of(&self, id: &str) -> Vec<&Note> { + self.idx(self.backlinks.get(id)) + } + + pub fn outlinks_of(&self, id: &str) -> Vec<&Note> { + match self.get(id) { + Some(n) => n.links.iter().filter_map(|l| self.get(&l.dst)).collect(), + None => Vec::new(), + } + } + + fn idx(&self, v: Option<&Vec<usize>>) -> Vec<&Note> { + match v { + Some(v) => v.iter().map(|&i| &self.notes[i]).collect(), + None => Vec::new(), + } + } + + /// Does the note have a link or a backlink or a parent or a child? + pub fn is_connected(&self, n: &Note) -> bool { + let id = match &n.id { + Some(id) => id, + None => return false, + }; + n.parent.is_some() + || !n.links.is_empty() + || self.children.contains_key(id) + || self.backlinks.contains_key(id) + } + + /// Breadth-first search over all relations, ignoring direction. + /// Returns (note, distance, relation at the first step). + pub fn reach(&self, id: &str, hops: usize) -> Vec<(&Note, usize, Rel)> { + let mut out = Vec::new(); + let mut seen: HashSet<&str> = HashSet::new(); + seen.insert(id); + let mut queue: VecDeque<(&str, usize, Option<Rel>)> = VecDeque::new(); + queue.push_back((id, 0, None)); + while let Some((cur, d, first)) = queue.pop_front() { + if d == hops { + continue; + } + let mut next: Vec<(&Note, Rel)> = Vec::new(); + if let Some(p) = self.parent_of(cur) { + next.push((p, Rel::Parent)); + } + for c in self.children_of(cur) { + next.push((c, Rel::Child)); + } + for l in self.outlinks_of(cur) { + next.push((l, Rel::Link)); + } + for b in self.backlinks_of(cur) { + next.push((b, Rel::Backlink)); + } + for (n, rel) in next { + let nid = match n.id.as_deref() { + Some(x) => x, + None => continue, + }; + if seen.insert(nid) { + let r = first.unwrap_or(rel); + out.push((n, d + 1, r)); + queue.push_back((nid, d + 1, Some(r))); + } + } + } + out + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9824351 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,588 @@ +//! elel: Emacs Lexical Embedding Link. A notes engine. +//! +//! This program only reads the note files. It never writes to them. +//! Everything it stores lives in the index directory and can be rebuilt. + +mod adapter; +mod chunk; +mod graph; +mod note; +mod query; +mod text; +mod vec; + +use anyhow::{Context, Result, bail}; +use clap::{Parser, Subcommand}; +use graph::{FileEntry, Graph, Manifest}; +use note::Note; +use query::Query; +use serde::Serialize; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use text::Text; +use vec::{Ollama, Store}; + +#[derive(Parser)] +#[command(version, about = "Search and link notes kept in plain files")] +struct Cli { + /// Directory that holds the note files. + #[arg(long, env = "ELEL_ROOT", global = true)] + root: Option<PathBuf>, + /// Directory for the derived index. + #[arg(long, env = "ELEL_INDEX", global = true)] + index: Option<PathBuf>, + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Scan the note files and update the index. + Index { + /// Re-read every file, even the unchanged ones. + #[arg(long)] + force: bool, + /// Do not compute embeddings (no Ollama needed). + #[arg(long)] + no_embed: bool, + }, + /// Search notes. The query may hold filters such as `tag:X`, + /// `date:2025..`, `from:ID`, `thread:ID`, `is:inbox` (see the README); + /// the other words go to the full text index. An empty query lists + /// every note. + Search { + query: Option<String>, + #[arg(short, default_value_t = 50)] + n: usize, + /// Also rank by meaning: fuse the text search with the embeddings. + #[arg(long)] + hybrid: bool, + }, + /// Print how many notes each query matches, one line per query. + Count { queries: Vec<String> }, + /// Show the neighbourhood of a note. + Hood { + id: String, + #[arg(long, default_value_t = 1)] + hops: usize, + }, + /// Print a note with its body, read fresh from its file. + Get { id: String }, + /// Report notes without IDs, duplicate IDs and dangling links. + Check, + /// List the tags with the number of notes that carry each. + Tags { + /// Rank the tags by how close their notes are to this note. + #[arg(long)] + like: Option<String>, + }, +} + +struct Ctx { + root: PathBuf, + index: PathBuf, +} + +impl Ctx { + fn new(cli: &Cli) -> Result<Ctx> { + let home = PathBuf::from(std::env::var("HOME").context("HOME is not set")?); + let root = cli.root.clone().unwrap_or_else(|| home.join("notes")); + let index = cli + .index + .clone() + .unwrap_or_else(|| home.join(".local/share/elel")); + Ok(Ctx { root, index }) + } + + fn manifest_path(&self) -> PathBuf { + self.index.join("graph.json") + } + + fn tantivy_dir(&self) -> PathBuf { + self.index.join("tantivy") + } +} + +/// A note as shown to the front end. +#[derive(Serialize)] +struct View<'a> { + #[serde(flatten)] + note: &'a Note, + #[serde(skip_serializing_if = "Option::is_none")] + score: Option<f32>, + #[serde(skip_serializing_if = "Option::is_none")] + snippet: Option<String>, + #[serde(skip_serializing_if = "Option::is_none")] + dist: Option<usize>, + #[serde(skip_serializing_if = "Option::is_none")] + rel: Option<&'static str>, + /// For hits by meaning: the line of the body where the best chunk starts. + #[serde(skip_serializing_if = "Option::is_none")] + chunk_line: Option<usize>, + /// Number of notes that link here. + backlinks: usize, +} + +fn view<'a>(g: &Graph, n: &'a Note) -> View<'a> { + View { + note: n, + score: None, + snippet: None, + dist: None, + rel: None, + chunk_line: None, + backlinks: n.id.as_deref().map(|i| g.backlinks_of(i).len()).unwrap_or(0), + } +} + +fn print<T: Serialize>(v: &T) -> Result<()> { + println!("{}", serde_json::to_string(v)?); + Ok(()) +} + +fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) -> Result<()> { + let mut entries: Vec<_> = std::fs::read_dir(dir) + .with_context(|| format!("cannot read {}", dir.display()))? + .filter_map(|e| e.ok()) + .collect(); + entries.sort_by_key(|e| e.file_name()); + for e in entries { + let p = e.path(); + let name = e.file_name().to_string_lossy().to_string(); + if name.starts_with('.') || name.ends_with('~') || name.starts_with('#') { + continue; + } + if p.is_dir() { + collect_files(&p, out)?; + } else if matches!(p.extension().and_then(|x| x.to_str()), Some("org" | "txt")) { + out.push(p); + } + } + Ok(()) +} + +fn cmd_index(ctx: &Ctx, force: bool, no_embed: bool) -> Result<()> { + std::fs::create_dir_all(&ctx.index)?; + let text = Text::open(&ctx.tantivy_dir())?; + let old = if text.fresh || force { + Manifest::default() + } else { + Manifest::load(&ctx.manifest_path()) + }; + let root = ctx + .root + .canonicalize() + .with_context(|| format!("no notes directory at {}", ctx.root.display()))?; + let mut files = Vec::new(); + collect_files(&root, &mut files)?; + + let mut new = Manifest::default(); + let mut changed: Vec<(String, Vec<Note>)> = Vec::new(); + for f in &files { + let key = f.to_string_lossy().to_string(); + let content = match std::fs::read_to_string(f) { + Ok(c) => c, + Err(e) => { + eprintln!("skipping {}: {}", key, e); + continue; + } + }; + let hash = blake3::hash(content.as_bytes()).to_hex().to_string(); + match old.files.get(&key) { + Some(e) if e.hash == hash => { + new.files.insert( + key, + FileEntry { + hash, + notes: e.notes.clone(), + }, + ); + } + _ => { + let notes = adapter::parse_file(f, &content).unwrap_or_default(); + changed.push((key.clone(), notes.clone())); + new.files.insert(key, FileEntry { hash, notes }); + } + } + } + let removed: Vec<String> = old + .files + .keys() + .filter(|k| !new.files.contains_key(*k)) + .cloned() + .collect(); + text.update(&removed, &changed)?; + new.save(&ctx.manifest_path())?; + let g = Graph::build(&new); + let mut report = serde_json::json!({ + "files": new.files.len(), + "changed": changed.len(), + "removed": removed.len(), + "notes": g.notes.len(), + }); + if !no_embed { + let r = vec::sync(&ctx.index, &g, &Ollama::from_env())?; + report["embedded"] = r.embedded.into(); + report["pending"] = r.pending.into(); + if let Some(e) = r.error { + eprintln!("elel: embeddings stopped: {}", e); + report["embed_error"] = e.into(); + } + } + print(&report) +} + +fn load_graph(ctx: &Ctx) -> Result<Graph> { + let p = ctx.manifest_path(); + if !p.exists() { + bail!("no index yet; run `elel index` first"); + } + Ok(Graph::build(&Manifest::load(&p))) +} + +/// One answer of a query: the note, and for text queries the score +/// and the snippet. +struct Hit<'a> { + note: &'a Note, + score: Option<f32>, + snippet: Option<String>, + /// The best chunk, for hits by meaning. + row: Option<vec::Row>, +} + +/// The words of a text query that can be embedded: no `-x`, `OR`, or +/// field prefixes, and no quotes. +fn plain_words(words: &str) -> String { + words + .split_whitespace() + .filter(|w| !w.starts_with('-') && *w != "OR" && *w != "AND") + .map(|w| w.rsplit(':').next().unwrap_or(w).trim_matches('"')) + .filter(|w| !w.is_empty()) + .collect::<Vec<_>>() + .join(" ") +} + +/// Rank by meaning for the `like:` filters (or, with `hybrid`, for the words). +fn semantic( + ctx: &Ctx, + g: &Graph, + q: &Query, + m: &query::Matcher, + hybrid: bool, +) -> Result<Vec<vec::VecHit>> { + let store = Store::load(&ctx.index); + if store.is_empty() { + bail!("no embeddings yet; start Ollama and run `elel index`"); + } + let ollama = Ollama::from_env(); + if store.model != ollama.model { + bail!( + "the embeddings were made with {}, but the model is {}; run `elel index`", + store.model, + ollama.model + ); + } + let mut by_chunk = false; + let mut exclude: HashSet<String> = HashSet::new(); + let mut resolve = |s: &str| -> Result<Vec<f32>> { + if g.get(s).is_some() { + exclude.insert(s.to_string()); + store + .note_vec(s) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", s)) + } else { + by_chunk = true; + ollama.embed_query(s) + } + }; + let mut pos = Vec::new(); + for s in &q.like { + pos.push(resolve(s)?); + } + let mut neg = Vec::new(); + for s in &q.unlike { + neg.push(resolve(s)?); + } + if pos.is_empty() { + if !neg.is_empty() { + bail!("-like: needs a like: to go with"); + } + by_chunk = true; + pos.push(ollama.embed_query(&plain_words(&q.words))?); + } + let qv = vec::rocchio(&pos, &neg).unwrap(); + let _ = hybrid; + let allow = |id: &str| match g.get(id) { + Some(n) => !exclude.contains(id) && m.pass(n), + None => false, + }; + Ok(store.rank(&qv, by_chunk, &allow)) +} + +fn run_query<'a>( + ctx: &Ctx, + g: &'a Graph, + query: &str, + n: usize, + hybrid: bool, +) -> Result<Vec<Hit<'a>>> { + let q = Query::parse(query)?; + let root = ctx + .root + .canonicalize() + .unwrap_or_else(|_| ctx.root.clone()) + .to_string_lossy() + .to_string(); + let m = q.matcher(g, &root); + let has_words = !q.words.trim().is_empty(); + let has_like = !q.like.is_empty() || !q.unlike.is_empty(); + let mut out = Vec::new(); + + // The ranking by meaning, if the query asks for it. + let mut by_meaning: Option<Vec<vec::VecHit>> = None; + if has_like || (hybrid && has_words && !plain_words(&q.words).is_empty()) { + match semantic(ctx, g, &q, &m, hybrid) { + Ok(v) => by_meaning = Some(v), + Err(e) if has_like => return Err(e), + Err(e) => eprintln!("elel: {}; using the text search only", e), + } + } + + // The ranking by words. + let mut lexical: Vec<(&'a Note, f32, Option<String>)> = Vec::new(); + if has_words { + let text = Text::open(&ctx.tantivy_dir())?; + // Filters apply after the text search, so ask for plenty. + for f in text.search(&q.words, 100_000)? { + let note = match g.get(&f.id) { + Some(note) => note, + None => continue, + }; + if m.pass(note) { + lexical.push((note, f.score, f.snippet)); + } + } + } + + match by_meaning { + None if has_words => { + for (note, score, snippet) in lexical.into_iter().take(n) { + out.push(Hit { note, score: Some(score), snippet, row: None }); + } + } + None => { + let mut all: Vec<&Note> = g.notes.iter().filter(|x| m.pass(x)).collect(); + // Newest first; notes without a date go last. + all.sort_by(|a, b| b.date.cmp(&a.date)); + for note in all.into_iter().take(n) { + out.push(Hit { note, score: None, snippet: None, row: None }); + } + } + Some(v) if !has_words => { + for h in v.into_iter().take(n) { + if let Some(note) = g.get(&h.id) { + out.push(Hit { note, score: Some(h.score), snippet: None, row: h.row }); + } + } + } + Some(v) => { + let lists = vec![ + lexical.iter().filter_map(|(n, _, _)| n.id.clone()).collect::<Vec<_>>(), + v.iter().map(|h| h.id.clone()).collect::<Vec<_>>(), + ]; + let mut rows: HashMap<String, vec::Row> = HashMap::new(); + for h in v { + if let Some(r) = h.row { + rows.insert(h.id, r); + } + } + let mut snippets: HashMap<String, Option<String>> = HashMap::new(); + for (note, _, s) in lexical { + if let Some(id) = ¬e.id { + snippets.insert(id.clone(), s); + } + } + for (id, score) in vec::rrf(&lists).into_iter().take(n) { + if let Some(note) = g.get(&id) { + out.push(Hit { + note, + score: Some(score), + snippet: snippets.remove(&id).flatten(), + row: rows.remove(&id), + }); + } + } + } + } + Ok(out) +} + +/// The first text of a chunk, for a hit by meaning. +fn chunk_snippet(h: &Hit, cache: &mut HashMap<String, Vec<Note>>) -> Option<String> { + let row = h.row.as_ref()?; + let path = &h.note.path; + if !cache.contains_key(path) { + let content = std::fs::read_to_string(path).ok()?; + let v = adapter::parse_file(Path::new(path), &content).unwrap_or_default(); + cache.insert(path.clone(), v); + } + let fresh = cache[path].iter().find(|x| x.id == h.note.id && x.hash == row.hash)?; + let cs = vec::note_chunks(fresh); + let text = &cs.get(row.chunk)?.text; + let body = text.strip_prefix(&fresh.title).unwrap_or(text); + let flat = body.split_whitespace().collect::<Vec<_>>().join(" "); + Some(flat.chars().take(200).collect()) +} + +fn cmd_search(ctx: &Ctx, query: Option<String>, n: usize, hybrid: bool) -> Result<()> { + let g = load_graph(ctx)?; + let mut cache = HashMap::new(); + for h in run_query(ctx, &g, query.as_deref().unwrap_or(""), n, hybrid)? { + let mut v = view(&g, h.note); + v.score = h.score; + v.snippet = h.snippet.clone().or_else(|| chunk_snippet(&h, &mut cache)); + v.chunk_line = h.row.as_ref().map(|r| r.line); + print(&v)?; + } + Ok(()) +} + +fn cmd_count(ctx: &Ctx, queries: Vec<String>) -> Result<()> { + let g = load_graph(ctx)?; + for q in queries { + let n = run_query(ctx, &g, &q, usize::MAX, false)?.len(); + print(&serde_json::json!({ "count": n }))?; + } + Ok(()) +} + +fn cmd_hood(ctx: &Ctx, id: &str, hops: usize) -> Result<()> { + let g = load_graph(ctx)?; + let center = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + print(&serde_json::json!({ "center": view(&g, center) }))?; + for (n, d, rel) in g.reach(id, hops.max(1)) { + let mut v = view(&g, n); + v.dist = Some(d); + v.rel = Some(rel.name()); + print(&v)?; + } + Ok(()) +} + +fn cmd_get(ctx: &Ctx, id: &str) -> Result<()> { + let g = load_graph(ctx)?; + let n = match g.get(id) { + Some(n) => n, + None => bail!("no note with id {}", id), + }; + // Read the file again, so the body is never stale. + let path = PathBuf::from(&n.path); + let content = std::fs::read_to_string(&path) + .with_context(|| format!("index out of date: cannot read {}", path.display()))?; + let fresh = adapter::parse_file(&path, &content) + .unwrap_or_default() + .into_iter() + .find(|x| x.id.as_deref() == Some(id)); + let note = fresh.as_ref().unwrap_or(n); + print(&serde_json::json!({ + "note": view(&g, note), + "body": note.body, + // The exact text of the note in its file, for display. + "raw": fresh.as_ref().map(|f| content[f.start..f.end].to_string()), + "stale": fresh.is_none(), + "parent": g.parent_of(id).map(|x| view(&g, x)), + "children": g.children_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "links": g.outlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + "backlinks": g.backlinks_of(id).iter().map(|x| view(&g, x)).collect::<Vec<_>>(), + })) +} + +fn cmd_check(ctx: &Ctx) -> Result<()> { + let g = load_graph(ctx)?; + let mut issues: Vec<serde_json::Value> = Vec::new(); + for n in &g.notes { + match &n.id { + None => issues.push(serde_json::json!({ + "issue": "missing-id", "path": n.path, "line": n.line, + "start": n.start, "title": n.title, + })), + Some(id) => { + if let Some(p) = &n.parent { + if g.get(p).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-parent", "id": id, "target": p, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for l in &n.links { + if g.get(&l.dst).is_none() { + issues.push(serde_json::json!({ + "issue": "dangling-link", "id": id, "target": l.dst, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + } + } + } + let mut by_path: BTreeMap<&str, usize> = BTreeMap::new(); + for (id, idxs) in &g.duplicates { + for &i in idxs { + let n = &g.notes[i]; + *by_path.entry(&n.path).or_default() += 1; + issues.push(serde_json::json!({ + "issue": "duplicate-id", "id": id, + "path": n.path, "line": n.line, "title": n.title, + })); + } + } + for i in issues { + print(&i)?; + } + Ok(()) +} + +fn cmd_tags(ctx: &Ctx, like: Option<String>) -> Result<()> { + let g = load_graph(ctx)?; + if let Some(id) = like { + let store = Store::load(&ctx.index); + let v = store + .note_vec(&id) + .ok_or_else(|| anyhow::anyhow!("note {} has no embedding yet; run `elel index`", id))?; + for (tag, count, score, has) in store.tag_scores(&g, &id, &v) { + print(&serde_json::json!({ "tag": tag, "count": count, "score": score, "has": has }))?; + } + return Ok(()); + } + let mut counts: BTreeMap<&str, usize> = BTreeMap::new(); + for n in g.notes.iter().filter(|n| n.id.is_some()) { + for t in &n.tags { + *counts.entry(t).or_default() += 1; + } + } + let mut v: Vec<(&str, usize)> = counts.into_iter().collect(); + // Most used first, then alphabetical. + v.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(b.0))); + for (tag, count) in v { + print(&serde_json::json!({ "tag": tag, "count": count }))?; + } + Ok(()) +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + let ctx = Ctx::new(&cli)?; + match cli.cmd { + Cmd::Index { force, no_embed } => cmd_index(&ctx, force, no_embed), + Cmd::Search { query, n, hybrid } => cmd_search(&ctx, query, n, hybrid), + Cmd::Count { queries } => cmd_count(&ctx, queries), + Cmd::Hood { id, hops } => cmd_hood(&ctx, &id, hops), + Cmd::Get { id } => cmd_get(&ctx, &id), + Cmd::Check => cmd_check(&ctx), + Cmd::Tags { like } => cmd_tags(&ctx, like), + } +} diff --git a/src/note.rs b/src/note.rs new file mode 100644 index 0000000..ef38094 --- /dev/null +++ b/src/note.rs @@ -0,0 +1,36 @@ +//! The data model shared by all adapters and by the index. + +use serde::{Deserialize, Serialize}; + +/// A link from one note to another. Types are optional. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Link { + pub dst: String, + pub kind: Option<String>, +} + +/// One note. A file is a container of notes; the note is the unit. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Note { + /// `None` for a note that has no ID yet; `check` reports these. + pub id: Option<String>, + pub path: String, + /// Byte span of the note inside its file. + pub start: usize, + pub end: usize, + /// 1-based line number of the first line of the note. + pub line: usize, + /// Hash of the span, used to detect changed notes. + pub hash: String, + pub title: String, + pub date: Option<String>, + pub kind: Option<String>, + pub tags: Vec<String>, + pub parent: Option<String>, + pub links: Vec<Link>, + pub context: Option<String>, + pub summary: Option<String>, + /// The text to index. It is never written to the manifest. + #[serde(skip)] + pub body: String, +} diff --git a/src/query.rs b/src/query.rs new file mode 100644 index 0000000..2b53ae6 --- /dev/null +++ b/src/query.rs @@ -0,0 +1,335 @@ +//! The query language. +//! +//! A query is one string. Words such as `tag:X` are filters on the +//! metadata and the graph; everything else is passed on to the full +//! text index. See the README for the list. + +use crate::graph::Graph; +use crate::note::Note; +use anyhow::{Result, bail}; +use std::collections::HashSet; + +/// A pattern for tags and kinds: `*` (anything), `abc*` (prefix) or `abc`. +#[derive(Debug, PartialEq)] +pub enum Pat { + Any, + Prefix(String), + Exact(String), +} + +impl Pat { + fn new(v: &str) -> Pat { + match v.strip_suffix('*') { + Some("") => Pat::Any, + Some(p) => Pat::Prefix(p.to_string()), + None => Pat::Exact(v.to_string()), + } + } + + fn matches(&self, s: &str) -> bool { + match self { + Pat::Any => true, + Pat::Prefix(p) => s.starts_with(p.as_str()), + Pat::Exact(e) => s == e, + } + } +} + +/// A range of dates, compared as prefixes: `2025` is the whole year. +#[derive(Debug, PartialEq)] +pub struct DateRange { + pub from: Option<String>, + pub to: Option<String>, +} + +impl DateRange { + fn contains(&self, date: &str) -> bool { + let cut = |n: usize| date.get(..n).unwrap_or(date); + self.from.as_ref().is_none_or(|a| cut(a.len()) >= a.as_str()) + && self.to.as_ref().is_none_or(|b| cut(b.len()) <= b.as_str()) + } +} + +#[derive(Debug, Default, PartialEq)] +pub struct Query { + pub tags: Vec<Pat>, + pub no_tags: Vec<Pat>, + pub kinds: Vec<Pat>, + pub no_kinds: Vec<Pat>, + pub files: Vec<String>, + pub no_files: Vec<String>, + pub from: Vec<String>, + pub to: Vec<String>, + pub parent: Vec<String>, + pub thread: Vec<String>, + pub dates: Vec<DateRange>, + pub inbox: bool, + /// `like:X` and `-like:X`: note IDs or free text, for the embeddings. + pub like: Vec<String>, + pub unlike: Vec<String>, + /// What is left for the full text index. + pub words: String, +} + +/// Split on white space, but keep double-quoted stretches together. +/// The quotes stay in the tokens. +fn tokens(q: &str) -> Vec<String> { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quoted = false; + for c in q.chars() { + match c { + '"' => { + quoted = !quoted; + cur.push(c); + } + c if c.is_whitespace() && !quoted => { + if !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + } + } + c => cur.push(c), + } + } + if !cur.is_empty() { + out.push(cur); + } + out +} + +fn unquote(v: &str) -> &str { + v.strip_prefix('"') + .map(|r| r.strip_suffix('"').unwrap_or(r)) + .unwrap_or(v) +} + +impl Query { + pub fn parse(q: &str) -> Result<Query> { + let mut out = Query::default(); + let mut words: Vec<String> = Vec::new(); + for tok in tokens(q) { + let (neg, rest) = match tok.strip_prefix('-') { + Some(r) if !r.is_empty() => (true, r), + _ => (false, tok.as_str()), + }; + let (key, value) = match rest.split_once(':') { + Some((k, v)) if !k.starts_with('"') => (k, v), + _ => { + words.push(tok); + continue; + } + }; + let known = matches!( + key, + "tag" | "kind" | "file" | "from" | "to" | "parent" | "thread" | "date" | "is" | "like" + ); + if !known { + // `title:foo` and friends belong to the text index. + words.push(tok); + continue; + } + let value = unquote(value); + if value.is_empty() { + bail!("empty value for {}:", key); + } + if neg && !matches!(key, "tag" | "kind" | "file" | "like") { + bail!("cannot negate {}:", key); + } + match key { + "tag" if neg => out.no_tags.push(Pat::new(value)), + "tag" => out.tags.push(Pat::new(value)), + "kind" if neg => out.no_kinds.push(Pat::new(value)), + "kind" => out.kinds.push(Pat::new(value)), + "file" if neg => out.no_files.push(value.to_string()), + "file" => out.files.push(value.to_string()), + "like" if neg => out.unlike.push(value.to_string()), + "like" => out.like.push(value.to_string()), + "from" => out.from.push(value.to_string()), + "to" => out.to.push(value.to_string()), + "parent" => out.parent.push(value.to_string()), + "thread" => out.thread.push(value.to_string()), + "date" => { + let range = match value.split_once("..") { + Some((a, b)) => DateRange { + from: (!a.is_empty()).then(|| a.to_string()), + to: (!b.is_empty()).then(|| b.to_string()), + }, + None => DateRange { + from: Some(value.to_string()), + to: Some(value.to_string()), + }, + }; + out.dates.push(range); + } + "is" => match value { + "inbox" => out.inbox = true, + _ => bail!("unknown is:{}", value), + }, + _ => unreachable!(), + } + } + out.words = words.join(" "); + Ok(out) + } + + /// Resolve the parts that need the graph, and return a matcher. + pub fn matcher<'a>(&'a self, g: &'a Graph, root: &'a str) -> Matcher<'a> { + let ids = |v: Vec<&Note>| -> HashSet<String> { + v.into_iter().filter_map(|n| n.id.clone()).collect() + }; + let mut sets: Vec<HashSet<String>> = Vec::new(); + for id in &self.from { + sets.push(ids(g.outlinks_of(id))); + } + for id in &self.to { + sets.push(ids(g.backlinks_of(id))); + } + for id in &self.parent { + sets.push(ids(g.children_of(id))); + } + for id in &self.thread { + sets.push(thread_of(g, id)); + } + Matcher { + q: self, + g, + root, + sets, + } + } +} + +/// All the notes in the thread of `id`: the root above it and every +/// descendant of that root. +fn thread_of(g: &Graph, id: &str) -> HashSet<String> { + let mut out = HashSet::new(); + if g.get(id).is_none() { + return out; + } + let mut root = id.to_string(); + let mut seen = HashSet::new(); + while seen.insert(root.clone()) { + match g.parent_of(&root).and_then(|p| p.id.clone()) { + Some(p) => root = p, + None => break, + } + } + let mut stack = vec![root]; + while let Some(cur) = stack.pop() { + if out.insert(cur.clone()) { + for c in g.children_of(&cur) { + if let Some(cid) = &c.id { + stack.push(cid.clone()); + } + } + } + } + out +} + +pub struct Matcher<'a> { + q: &'a Query, + g: &'a Graph, + root: &'a str, + sets: Vec<HashSet<String>>, +} + +impl Matcher<'_> { + pub fn pass(&self, note: &Note) -> bool { + let id = match ¬e.id { + Some(i) => i, + None => return false, + }; + let q = self.q; + let any_tag = |p: &Pat| note.tags.iter().any(|t| p.matches(t)); + let rel = note + .path + .strip_prefix(self.root) + .unwrap_or(¬e.path) + .trim_start_matches('/'); + q.tags.iter().all(|p| match p { + Pat::Any => !note.tags.is_empty(), + _ => any_tag(p), + }) && q.no_tags.iter().all(|p| match p { + Pat::Any => note.tags.is_empty(), + _ => !any_tag(p), + }) && q + .kinds + .iter() + .all(|p| note.kind.as_deref().is_some_and(|k| p.matches(k))) + && q.no_kinds.iter().all(|p| match p { + Pat::Any => note.kind.is_none(), + _ => !note.kind.as_deref().is_some_and(|k| p.matches(k)), + }) + && q.files.iter().all(|f| rel.contains(f.as_str())) + && q.no_files.iter().all(|f| !rel.contains(f.as_str())) + && self.sets.iter().all(|s| s.contains(id)) + && q.dates + .iter() + .all(|r| note.date.as_deref().is_some_and(|d| r.contains(d))) + && (!q.inbox || (!self.g.is_connected(note) && note.tags.is_empty())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filters_are_split_from_words() { + let q = Query::parse("tag:math -tag:old kind:idea \"exact phrase\" word title:foo").unwrap(); + assert_eq!(q.tags, vec![Pat::Exact("math".into())]); + assert_eq!(q.no_tags, vec![Pat::Exact("old".into())]); + assert_eq!(q.kinds, vec![Pat::Exact("idea".into())]); + assert_eq!(q.words, "\"exact phrase\" word title:foo"); + } + + #[test] + fn star_patterns() { + let q = Query::parse("tag:* -tag:alg* kind:*").unwrap(); + assert_eq!(q.tags, vec![Pat::Any]); + assert_eq!(q.no_tags, vec![Pat::Prefix("alg".into())]); + assert_eq!(q.kinds, vec![Pat::Any]); + assert_eq!(q.words, ""); + } + + #[test] + fn dates_and_quoted_values() { + let q = Query::parse("date:2025-03..2025-06 date:..2024 date:2020 file:\"my notes\"").unwrap(); + assert_eq!(q.dates.len(), 3); + assert!(q.dates[0].contains("2025-04-01T10:00:00")); + assert!(!q.dates[0].contains("2025-07-01")); + assert!(q.dates[0].contains("2025-06-30")); + assert!(q.dates[1].contains("2023-01-01")); + assert!(!q.dates[1].contains("2025-01-01")); + assert!(q.dates[2].contains("2020-05-14T23:21:08+08:00")); + assert_eq!(q.files, vec!["my notes"]); + } + + #[test] + fn errors() { + assert!(Query::parse("tag:").is_err()); + assert!(Query::parse("-from:x").is_err()); + assert!(Query::parse("is:weird").is_err()); + // A colon in an ordinary word is fine. + assert_eq!(Query::parse("a:b").unwrap().words, "a:b"); + // A lone dash is a word. + assert_eq!(Query::parse("- x").unwrap().words, "- x"); + } + + #[test] + fn like_filters() { + let q = Query::parse("like:abc-1 -like:\"moduli of curves\" tag:x word").unwrap(); + assert_eq!(q.like, vec!["abc-1"]); + assert_eq!(q.unlike, vec!["moduli of curves"]); + assert_eq!(q.words, "word"); + } + + #[test] + fn ids_with_angle_brackets() { + let q = Query::parse("thread:<a.b@c> to:abc-1 is:inbox").unwrap(); + assert_eq!(q.thread, vec!["<a.b@c>"]); + assert_eq!(q.to, vec!["abc-1"]); + assert!(q.inbox); + } +} diff --git a/src/text.rs b/src/text.rs new file mode 100644 index 0000000..466e065 --- /dev/null +++ b/src/text.rs @@ -0,0 +1,188 @@ +//! Full-text index on top of Tantivy. +//! +//! Two views of the text are indexed: the normal one, split into words, +//! and a bigram one, so that Chinese (which has no spaces) can be +//! matched by substring. + +use crate::note::Note; +use anyhow::Result; +use std::path::Path; +use tantivy::collector::TopDocs; +use tantivy::directory::MmapDirectory; +use tantivy::query::{BooleanQuery, Occur, Query, QueryParser, TermQuery}; +use tantivy::schema::{ + Field, IndexRecordOption, STORED, STRING, Schema, TextFieldIndexing, TextOptions, Value, +}; +use tantivy::snippet::SnippetGenerator; +use tantivy::tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer}; +use tantivy::{Index, IndexWriter, TantivyDocument, Term, doc}; + +/// Marks around highlighted words in snippets. These are private-use +/// characters, so they cannot appear in normal text. +pub const HL_START: char = '\u{e000}'; +pub const HL_END: char = '\u{e001}'; + +pub struct Text { + index: Index, + id: Field, + path: Field, + title: Field, + body: Field, + tags: Field, + grams: Field, + /// True if the index directory was created by this call. + pub fresh: bool, +} + +pub struct Found { + pub id: String, + pub score: f32, + pub snippet: Option<String>, +} + +fn has_cjk(s: &str) -> bool { + s.chars().any(|c| !c.is_ascii() && c.is_alphabetic()) +} + +impl Text { + pub fn open(dir: &Path) -> Result<Text> { + let fresh = !dir.join("meta.json").exists(); + std::fs::create_dir_all(dir)?; + let mut sb = Schema::builder(); + let id = sb.add_text_field("id", STRING | STORED); + let path = sb.add_text_field("path", STRING | STORED); + let title = sb.add_text_field("title", tantivy::schema::TEXT | STORED); + let body = sb.add_text_field("body", tantivy::schema::TEXT | STORED); + let tags = sb.add_text_field("tags", tantivy::schema::TEXT); + let grams = sb.add_text_field( + "grams", + TextOptions::default().set_indexing_options( + TextFieldIndexing::default() + .set_tokenizer("bigram") + .set_index_option(IndexRecordOption::Basic), + ), + ); + let index = Index::open_or_create(MmapDirectory::open(dir)?, sb.build())?; + index.tokenizers().register( + "bigram", + TextAnalyzer::builder(NgramTokenizer::new(2, 2, false)?) + .filter(LowerCaser) + .build(), + ); + Ok(Text { + index, + id, + path, + title, + body, + tags, + grams, + fresh, + }) + } + + /// Remove the notes of `removed` files, and replace the notes of the + /// `changed` files. Only notes with an ID are indexed. + pub fn update(&self, removed: &[String], changed: &[(String, Vec<Note>)]) -> Result<()> { + let mut w: IndexWriter = self.index.writer(50_000_000)?; + for p in removed { + w.delete_term(Term::from_field_text(self.path, p)); + } + for (p, notes) in changed { + w.delete_term(Term::from_field_text(self.path, p)); + for n in notes { + let id = match &n.id { + Some(id) => id, + None => continue, + }; + let mut text = String::new(); + if let Some(s) = &n.summary { + text.push_str(s); + text.push('\n'); + } + text.push_str(&n.body); + let all = format!("{}\n{}", n.title, text); + w.add_document(doc!( + self.id => id.as_str(), + self.path => p.as_str(), + self.title => n.title.as_str(), + self.body => text, + self.tags => n.tags.join(" "), + self.grams => all, + ))?; + } + } + w.commit()?; + Ok(()) + } + + /// Build a query. Words go through the query parser (so phrases and + /// `title:foo` work); words with CJK characters become a conjunction + /// of bigrams. + fn build_query(&self, q: &str) -> Result<Box<dyn Query>> { + let mut parser = QueryParser::for_index(&self.index, vec![self.title, self.body, self.tags]); + parser.set_conjunction_by_default(); + if !has_cjk(q) { + return Ok(parser.parse_query(q)?); + } + let mut clauses: Vec<(Occur, Box<dyn Query>)> = Vec::new(); + for word in q.split_whitespace() { + if has_cjk(word) { + let chars: Vec<char> = word.to_lowercase().chars().collect(); + if chars.len() == 1 { + // A single character has no bigram; match any bigram with it. + let t = Term::from_field_text(self.grams, &chars[0].to_string()); + clauses.push((Occur::Must, Box::new(TermQuery::new(t, IndexRecordOption::Basic)))); + } else { + for w in chars.windows(2) { + let g: String = w.iter().collect(); + let t = Term::from_field_text(self.grams, &g); + clauses.push(( + Occur::Must, + Box::new(TermQuery::new(t, IndexRecordOption::Basic)), + )); + } + } + } else { + clauses.push((Occur::Must, parser.parse_query(word)?)); + } + } + Ok(Box::new(BooleanQuery::new(clauses))) + } + + pub fn search(&self, q: &str, limit: usize) -> Result<Vec<Found>> { + let reader = self.index.reader()?; + let searcher = reader.searcher(); + let query = self.build_query(q)?; + let top = searcher.search(&query, &TopDocs::with_limit(limit).order_by_score())?; + let snipgen = SnippetGenerator::create(&searcher, &*query, self.body).ok(); + let mut out = Vec::new(); + for (score, addr) in top { + let d: TantivyDocument = searcher.doc(addr)?; + let id = match d.get_first(self.id).and_then(|v| v.as_str()) { + Some(s) => s.to_string(), + None => continue, + }; + let snippet = snipgen.as_ref().and_then(|g| { + let s = g.snippet_from_doc(&d); + let frag = s.fragment(); + if frag.is_empty() { + return None; + } + let mut out = String::new(); + let mut last = 0; + for r in s.highlighted() { + out.push_str(&frag[last..r.start]); + out.push(HL_START); + out.push_str(&frag[r.start..r.end]); + out.push(HL_END); + last = r.end; + } + out.push_str(&frag[last..]); + Some(out.replace('\n', " ")) + }); + out.push(Found { id, score, snippet }); + } + Ok(out) + } +} diff --git a/src/vec.rs b/src/vec.rs new file mode 100644 index 0000000..2c30ab5 --- /dev/null +++ b/src/vec.rs @@ -0,0 +1,651 @@ +//! The semantic side: embeddings from Ollama, a small vector store, and +//! the ranking on top of it. +//! +//! One vector per chunk of a note (see `chunk.rs`). The store keeps two +//! files next to each other: the vectors as f16 for exact scores, and one +//! sign bit per dimension for a fast first pass by Hamming distance when +//! there are very many chunks. Everything here can be rebuilt from the +//! note files, so it lives in the index directory. + +use crate::chunk; +use crate::graph::Graph; +use crate::note::Note; +use anyhow::{Context, Result, anyhow, bail}; +use half::f16; +use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub const DEFAULT_MODEL: &str = "qwen3-embedding:0.6b"; +const BATCH: usize = 16; +/// Above this many chunks, a Hamming pass narrows the candidates first. +const EXACT_LIMIT: usize = 2000; + +// ---------------------------------------------------------------- Ollama + +pub struct Ollama { + url: String, + pub model: String, +} + +/// Why a call to Ollama failed: it is not running, or it is something else. +#[derive(Debug)] +pub struct Unreachable(pub String); + +impl std::fmt::Display for Unreachable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::error::Error for Unreachable {} + +impl Ollama { + pub fn from_env() -> Ollama { + Ollama { + url: std::env::var("ELEL_OLLAMA").unwrap_or_else(|_| "http://localhost:11434".into()), + model: std::env::var("ELEL_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.into()), + } + } + + /// Embed texts, already formatted for the model. The vectors are + /// normalized to length 1. + pub fn embed(&self, inputs: &[String]) -> Result<Vec<Vec<f32>>> { + let agent = ureq::AgentBuilder::new() + .timeout_connect(Duration::from_secs(3)) + .timeout(Duration::from_secs(300)) + .build(); + let resp = agent + .post(&format!("{}/api/embed", self.url.trim_end_matches('/'))) + .send_json(serde_json::json!({ + "model": self.model, + "input": inputs, + "truncate": true, + })); + let v: serde_json::Value = match resp { + Ok(r) => r.into_json()?, + Err(ureq::Error::Status(404, _)) => bail!( + "model {} is not available in Ollama; run `ollama pull {}`", + self.model, + self.model + ), + Err(ureq::Error::Status(code, r)) => { + let body = r.into_string().unwrap_or_default(); + bail!("Ollama answered {}: {}", code, body) + } + Err(e @ ureq::Error::Transport(_)) => { + return Err(anyhow::Error::new(Unreachable(format!( + "cannot reach Ollama at {} ({})", + self.url, e + )))); + } + }; + let arr = v["embeddings"] + .as_array() + .ok_or_else(|| anyhow!("Ollama gave no embeddings: {}", v))?; + let mut out = Vec::new(); + for e in arr { + let mut x: Vec<f32> = e + .as_array() + .ok_or_else(|| anyhow!("bad embedding"))? + .iter() + .map(|n| n.as_f64().unwrap_or(0.0) as f32) + .collect(); + normalize(&mut x); + out.push(x); + } + if out.len() != inputs.len() { + bail!("asked for {} embeddings, got {}", inputs.len(), out.len()); + } + Ok(out) + } + + /// A note chunk, as the model wants a document. + pub fn doc_text(&self, text: &str) -> String { + if self.model.starts_with("embeddinggemma") { + format!("title: none | text: {}", text) + } else if self.model.starts_with("nomic-embed") { + format!("search_document: {}", text) + } else { + text.to_string() + } + } + + /// A search text, as the model wants a query. + pub fn query_text(&self, text: &str) -> String { + if self.model.starts_with("qwen3-embedding") { + format!( + "Instruct: Given a search query, retrieve the notes that are about it\nQuery: {}", + text + ) + } else if self.model.starts_with("embeddinggemma") { + format!("task: search result | query: {}", text) + } else if self.model.starts_with("nomic-embed") { + format!("search_query: {}", text) + } else { + text.to_string() + } + } + + pub fn embed_query(&self, text: &str) -> Result<Vec<f32>> { + Ok(self.embed(&[self.query_text(text)])?.remove(0)) + } +} + +// ------------------------------------------------------------ vector math + +pub fn normalize(v: &mut [f32]) { + let n = v.iter().map(|x| x * x).sum::<f32>().sqrt(); + if n > 0.0 { + for x in v.iter_mut() { + *x /= n; + } + } +} + +pub fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +/// The sign bits of a vector, packed eight to a byte. +pub fn to_bits(v: &[f32]) -> Vec<u8> { + let mut out = vec![0u8; v.len().div_ceil(8)]; + for (i, x) in v.iter().enumerate() { + if *x > 0.0 { + out[i / 8] |= 1 << (i % 8); + } + } + out +} + +pub fn hamming(a: &[u8], b: &[u8]) -> u32 { + a.iter().zip(b).map(|(x, y)| (x ^ y).count_ones()).sum() +} + +/// The normalized mean of some vectors. +pub fn mean(vs: &[&[f32]]) -> Option<Vec<f32>> { + let first = vs.first()?; + let mut m = vec![0.0f32; first.len()]; + for v in vs { + for (a, b) in m.iter_mut().zip(*v) { + *a += b; + } + } + normalize(&mut m); + Some(m) +} + +/// The merged vector of some liked and some disliked vectors: +/// `normalize(mean(pos) - 0.5 * mean(neg))`. +pub fn rocchio(pos: &[Vec<f32>], neg: &[Vec<f32>]) -> Option<Vec<f32>> { + let p: Vec<&[f32]> = pos.iter().map(|v| v.as_slice()).collect(); + let mut q = mean(&p)?; + let n: Vec<&[f32]> = neg.iter().map(|v| v.as_slice()).collect(); + if let Some(nm) = mean(&n) { + for (a, b) in q.iter_mut().zip(&nm) { + *a -= 0.5 * b; + } + normalize(&mut q); + } + Some(q) +} + +/// Reciprocal rank fusion of ranked lists of IDs: `sum 1/(60 + rank)`. +pub fn rrf(lists: &[Vec<String>]) -> Vec<(String, f32)> { + let mut score: HashMap<&str, f32> = HashMap::new(); + for l in lists { + for (r, id) in l.iter().enumerate() { + *score.entry(id).or_default() += 1.0 / (60.0 + r as f32 + 1.0); + } + } + let mut v: Vec<(String, f32)> = score.into_iter().map(|(k, s)| (k.to_string(), s)).collect(); + v.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap().then(a.0.cmp(&b.0))); + v +} + +// ------------------------------------------------------------------ store + +#[derive(Clone, Serialize, Deserialize)] +pub struct Row { + pub id: String, + /// `Note.hash` when the vector was made. + pub hash: String, + /// Index of the chunk in the note. + pub chunk: usize, + /// 0-based line in the body where the chunk starts. + pub line: usize, +} + +#[derive(Default, Serialize, Deserialize)] +struct Meta { + model: String, + dim: usize, + rows: Vec<Row>, +} + +#[derive(Default)] +pub struct Store { + pub model: String, + pub dim: usize, + pub rows: Vec<Row>, + vecs: Vec<f32>, + bits: Vec<u8>, +} + +/// A hit of the vector ranking. +pub struct VecHit { + pub id: String, + pub score: f32, + /// The best chunk, when ranked by chunk. + pub row: Option<Row>, +} + +pub fn dir(index: &Path) -> PathBuf { + index.join("vec") +} + +impl Store { + pub fn load(index: &Path) -> Store { + let d = dir(index); + let meta: Meta = std::fs::read_to_string(d.join("meta.json")) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let raw = std::fs::read(d.join("f16.bin")).unwrap_or_default(); + let want = meta.rows.len() * meta.dim * 2; + if meta.rows.is_empty() || raw.len() != want { + return Store::default(); + } + let vecs: Vec<f32> = raw + .chunks_exact(2) + .map(|b| f16::from_le_bytes([b[0], b[1]]).to_f32()) + .collect(); + let bits = std::fs::read(d.join("bits.bin")).unwrap_or_default(); + Store { + model: meta.model, + dim: meta.dim, + rows: meta.rows, + vecs, + bits, + } + } + + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub fn save(&self, index: &Path) -> Result<()> { + let d = dir(index); + std::fs::create_dir_all(&d)?; + let write = |name: &str, bytes: &[u8]| -> Result<()> { + let tmp = d.join(format!("{}.tmp", name)); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, d.join(name))?; + Ok(()) + }; + let mut f = Vec::with_capacity(self.vecs.len() * 2); + for x in &self.vecs { + f.extend_from_slice(&f16::from_f32(*x).to_le_bytes()); + } + // The bit file must be consistent with the (rounded) f16 file. + write("f16.bin", &f)?; + write("bits.bin", &self.bits)?; + let meta = Meta { + model: self.model.clone(), + dim: self.dim, + rows: self.rows.clone(), + }; + write("meta.json", &serde_json::to_vec(&meta)?) + } + + fn vec_of(&self, i: usize) -> &[f32] { + &self.vecs[i * self.dim..(i + 1) * self.dim] + } + + fn bits_of(&self, i: usize) -> &[u8] { + let w = self.dim.div_ceil(8); + &self.bits[i * w..(i + 1) * w] + } + + /// Keep the rows for which `keep` is true. + pub fn retain(&mut self, keep: impl Fn(&Row) -> bool) { + let w = self.dim.div_ceil(8); + let mut rows = Vec::new(); + let mut vecs = Vec::new(); + let mut bits = Vec::new(); + for (i, r) in self.rows.iter().enumerate() { + if keep(r) { + rows.push(r.clone()); + vecs.extend_from_slice(self.vec_of(i)); + bits.extend_from_slice(&self.bits[i * w..(i + 1) * w]); + } + } + self.rows = rows; + self.vecs = vecs; + self.bits = bits; + } + + pub fn push(&mut self, row: Row, v: &[f32]) { + if self.rows.is_empty() { + self.dim = v.len(); + } + self.bits.extend_from_slice(&to_bits(v)); + self.vecs.extend_from_slice(v); + self.rows.push(row); + } + + #[cfg(test)] + pub fn has(&self, id: &str, hash: &str) -> bool { + self.rows.iter().any(|r| r.id == id && r.hash == hash) + } + + /// The vector of a whole note: the mean of its chunks. + pub fn note_vec(&self, id: &str) -> Option<Vec<f32>> { + let vs: Vec<&[f32]> = self + .rows + .iter() + .enumerate() + .filter(|(_, r)| r.id == id) + .map(|(i, _)| self.vec_of(i)) + .collect(); + mean(&vs) + } + + /// All note vectors at once. + pub fn note_vecs(&self) -> HashMap<String, Vec<f32>> { + let mut groups: HashMap<&str, Vec<&[f32]>> = HashMap::new(); + for (i, r) in self.rows.iter().enumerate() { + groups.entry(&r.id).or_default().push(self.vec_of(i)); + } + groups + .into_iter() + .filter_map(|(id, vs)| mean(&vs).map(|m| (id.to_string(), m))) + .collect() + } + + /// Rank the notes for the query vector `q`. With `by_chunk` a note + /// scores as its best chunk (for text queries); otherwise as the mean + /// of its chunks (for "like this note"). `allow` filters by ID. + pub fn rank(&self, q: &[f32], by_chunk: bool, allow: &dyn Fn(&str) -> bool) -> Vec<VecHit> { + let mut out: Vec<VecHit> = Vec::new(); + if self.is_empty() || q.len() != self.dim { + return out; + } + if by_chunk { + let mut cand: Vec<usize> = (0..self.rows.len()) + .filter(|&i| allow(&self.rows[i].id)) + .collect(); + if cand.len() > EXACT_LIMIT { + let qb = to_bits(q); + cand.sort_by_key(|&i| hamming(&qb, self.bits_of(i))); + cand.truncate(EXACT_LIMIT); + } + let mut best: HashMap<&str, (f32, usize)> = HashMap::new(); + for i in cand { + let s = dot(q, self.vec_of(i)); + let e = best.entry(&self.rows[i].id).or_insert((f32::MIN, i)); + if s > e.0 { + *e = (s, i); + } + } + for (id, (s, i)) in best { + out.push(VecHit { + id: id.to_string(), + score: s, + row: Some(self.rows[i].clone()), + }); + } + } else { + for (id, v) in self.note_vecs() { + if allow(&id) { + out.push(VecHit { + score: dot(q, &v), + id, + row: None, + }); + } + } + } + out.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap().then(a.id.cmp(&b.id))); + out + } + + /// Rank the tags for a note vector. The anchor of a tag is the mean + /// of the vectors of the other notes that carry it; tags with fewer + /// than two such notes are skipped. + pub fn tag_scores(&self, g: &Graph, id: &str, v: &[f32]) -> Vec<(String, usize, f32, bool)> { + let vecs = self.note_vecs(); + let mut by_tag: HashMap<&str, Vec<&[f32]>> = HashMap::new(); + let mut count: HashMap<&str, usize> = HashMap::new(); + let mut has: HashSet<&str> = HashSet::new(); + for n in g.notes.iter().filter(|n| n.id.is_some()) { + let nid = n.id.as_deref().unwrap(); + for t in &n.tags { + *count.entry(t).or_default() += 1; + if nid == id { + has.insert(t); + } else if let Some(x) = vecs.get(nid) { + by_tag.entry(t).or_default().push(x); + } + } + } + let mut out = Vec::new(); + for (t, vs) in by_tag { + if vs.len() < 2 { + continue; + } + if let Some(a) = mean(&vs) { + out.push((t.to_string(), count[t], dot(v, &a), has.contains(t))); + } + } + out.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap().then(a.0.cmp(&b.0))); + out + } +} + +// ------------------------------------------------------------------ sync + +/// The text of the chunks of a note, formatted for the model. +pub fn note_chunks(n: &Note) -> Vec<chunk::Chunk> { + chunk::chunks(&n.title, &n.tags, n.summary.as_deref(), &n.body) +} + +pub struct SyncReport { + pub embedded: usize, + pub pending: usize, + pub error: Option<String>, +} + +/// Bring the store up to date with the graph. Notes without a vector are +/// embedded; vectors of notes that changed or vanished are dropped. +/// If Ollama fails, the notes done so far are kept and the rest stay +/// pending for the next run. +pub fn sync(index: &Path, g: &Graph, ollama: &Ollama) -> Result<SyncReport> { + let mut store = Store::load(index); + if store.model != ollama.model { + store = Store::default(); + } + let current: HashMap<&str, &str> = g + .notes + .iter() + .filter_map(|n| n.id.as_deref().map(|i| (i, n.hash.as_str()))) + .collect(); + store.retain(|r| current.get(r.id.as_str()) == Some(&r.hash.as_str())); + store.model = ollama.model.clone(); + let have: HashSet<String> = store.rows.iter().map(|r| r.id.clone()).collect(); + let mut seen: HashSet<&str> = HashSet::new(); + let todo: Vec<&Note> = g + .notes + .iter() + .filter(|n| match n.id.as_deref() { + Some(i) => !have.contains(i) && seen.insert(i), + None => false, + }) + .collect(); + let total = todo.len(); + let mut embedded = 0; + let mut error = None; + let mut parsed: HashMap<String, Vec<Note>> = HashMap::new(); + // Work in groups of notes, so a failure never leaves half a note. + let mut group: Vec<(&Note, Vec<chunk::Chunk>)> = Vec::new(); + let mut size = 0; + let flush = |group: &mut Vec<(&Note, Vec<chunk::Chunk>)>, + store: &mut Store| + -> Result<usize> { + if group.is_empty() { + return Ok(0); + } + let texts: Vec<String> = group + .iter() + .flat_map(|(_, cs)| cs.iter().map(|c| ollama.doc_text(&c.text))) + .collect(); + let mut vs = Vec::new(); + for b in texts.chunks(BATCH) { + let t0 = std::time::Instant::now(); + vs.extend(ollama.embed(b)?); + if std::env::var_os("ELEL_DEBUG").is_some() { + eprintln!( + "embed batch of {}: {} chars, max {}, {:.1}s", + b.len(), + b.iter().map(|x| x.len()).sum::<usize>(), + b.iter().map(|x| x.len()).max().unwrap_or(0), + t0.elapsed().as_secs_f32() + ); + } + } + let mut k = 0; + let mut notes = 0; + for (n, cs) in group.iter() { + for (ci, c) in cs.iter().enumerate() { + store.push( + Row { + id: n.id.clone().unwrap(), + hash: n.hash.clone(), + chunk: ci, + line: c.line, + }, + &vs[k], + ); + k += 1; + } + notes += 1; + } + group.clear(); + Ok(notes) + }; + 'work: for n in todo { + // The body is not in the manifest: read it from the file again. + let notes = match parsed.get(&n.path) { + Some(v) => v, + None => { + let content = match std::fs::read_to_string(&n.path) { + Ok(c) => c, + Err(_) => continue, + }; + let v = crate::adapter::parse_file(Path::new(&n.path), &content).unwrap_or_default(); + parsed.entry(n.path.clone()).or_insert(v) + } + }; + let fresh = match notes.iter().find(|x| x.id == n.id && x.hash == n.hash) { + Some(f) => f, + None => continue, + }; + let cs = note_chunks(fresh); + size += cs.len(); + group.push((n, cs)); + if size >= BATCH { + match flush(&mut group, &mut store) { + Ok(k) => embedded += k, + Err(e) => { + error = Some(e.to_string()); + break 'work; + } + } + size = 0; + } + } + if error.is_none() { + match flush(&mut group, &mut store) { + Ok(k) => embedded += k, + Err(e) => error = Some(e.to_string()), + } + } + store.save(index).context("cannot save the vectors")?; + Ok(SyncReport { + embedded, + pending: total - embedded, + error, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn unit(v: &[f32]) -> Vec<f32> { + let mut x = v.to_vec(); + normalize(&mut x); + x + } + + #[test] + fn bits_and_hamming() { + let a = to_bits(&[1.0, -1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, 1.0]); + assert_eq!(a, vec![0b0000_1101, 0b0000_0001]); + let b = to_bits(&[1.0, 1.0, 1.0, 1.0, -1.0, -1.0, -1.0, -1.0, -1.0]); + assert_eq!(hamming(&a, &b), 2); + } + + #[test] + fn f16_round_trip_keeps_the_order() { + let dir = std::env::temp_dir().join(format!("elel-vec-{}", std::process::id())); + let mut s = Store::default(); + s.model = "m".into(); + let row = |id: &str| Row { id: id.into(), hash: "h".into(), chunk: 0, line: 0 }; + s.push(row("a"), &unit(&[1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.push(row("b"), &unit(&[1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.push(row("c"), &unit(&[0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0])); + s.save(&dir).unwrap(); + let t = Store::load(&dir); + std::fs::remove_dir_all(&dir).ok(); + assert_eq!(t.rows.len(), 3); + let q = unit(&[1.0, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]); + let r = t.rank(&q, true, &|_| true); + assert_eq!(r.iter().map(|h| h.id.as_str()).collect::<Vec<_>>(), vec!["a", "b", "c"]); + let r = t.rank(&q, false, &|id| id != "a"); + assert_eq!(r[0].id, "b"); + } + + #[test] + fn rocchio_moves_away_from_the_negative() { + let pos = vec![unit(&[1.0, 1.0, 0.0])]; + let neg = vec![unit(&[0.0, 1.0, 0.0])]; + let q = rocchio(&pos, &neg).unwrap(); + assert!(q[0] > q[1], "{:?}", q); + let plain = rocchio(&pos, &[]).unwrap(); + assert!((plain[0] - plain[1]).abs() < 1e-6); + assert!(rocchio(&[], &neg).is_none()); + } + + #[test] + fn rrf_prefers_notes_high_in_both_lists() { + let l1 = vec!["a".to_string(), "b".into(), "c".into()]; + let l2 = vec!["b".to_string(), "c".into(), "d".into()]; + let r = rrf(&[l1, l2]); + assert_eq!(r[0].0, "b"); + assert_eq!(r.len(), 4); + } + + #[test] + fn retain_drops_stale_rows() { + let mut s = Store::default(); + let row = |id: &str, h: &str| Row { id: id.into(), hash: h.into(), chunk: 0, line: 0 }; + s.push(row("a", "1"), &unit(&[1.0, 0.0])); + s.push(row("b", "1"), &unit(&[0.0, 1.0])); + s.retain(|r| r.id == "a" && r.hash == "1"); + assert_eq!(s.rows.len(), 1); + assert!(s.has("a", "1") && !s.has("b", "1")); + assert_eq!(s.vec_of(0), &[1.0, 0.0]); + } +} |
