diff options
| author | JSDurand <mmemmew@gmail.com> | 2026-11-19 09:29:47 +0800 |
|---|---|---|
| committer | JSDurand <mmemmew@gmail.com> | 2026-11-19 09:29:47 +0800 |
| commit | 5b9a806af13d38163156302097e405a7b145b6cc (patch) | |
| tree | c079f3c4e027dc730892630f667b0e6401736043 /elel-import.el | |
Tantivy full text index, link graph, inline query filters, Ollama
embeddings with paragraph chunks, and an Elisp summary/article UI.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Diffstat (limited to 'elel-import.el')
| -rw-r--r-- | elel-import.el | 508 |
1 files changed, 508 insertions, 0 deletions
diff --git a/elel-import.el b/elel-import.el new file mode 100644 index 0000000..0dd2b9c --- /dev/null +++ b/elel-import.el @@ -0,0 +1,508 @@ +;;; elel-import.el --- Import old notes into elel -*- lexical-binding: t; -*- + +;; Copyright (C) 2026 Durand + +;; Author: Durand +;; Keywords: outlines, files + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Two importers, which only read their sources: +;; +;; `elel-import-drafts' -- notes kept as Gnus drafts, i.e. messages +;; whose Gcc is a nnmaildir+notes:GROUP group. Each GROUP becomes +;; one container file GROUP.org, and the Message-IDs become the +;; IDs, so the In-Reply-To threads become PARENT properties. +;; +;; `elel-import-roam' -- an org-roam (version 1, file based) directory. +;; Each file becomes one note, and the links to other roam files +;; become [[note:ID][TITLE]] links. +;; +;; Both take the source, the output directory, and an optional DRY-RUN. +;; With a nil output directory, they only report what they would do. +;; They never overwrite: if a target file exists, nothing is written. +;; +;; From a shell: +;; +;; emacs --batch -L DIR -l elel-import \ +;; --eval '(elel-import-drafts "~/News/drafts/drafts" "~/notes")' + +;;; Code: + +(require 'cl-lib) +(require 'subr-x) +(require 'rfc2047) +(require 'qp) +(require 'elel) + +;;; Common helpers + +(defun elel-import--read (path) + "The content of the file PATH, decoded as UTF-8." + (with-temp-buffer + (let ((coding-system-for-read 'utf-8)) + (insert-file-contents path)) + (buffer-string))) + +(defun elel-import--org-note (title tags props body) + "The text of a note in org format. +PROPS is an alist of (NAME . VALUE) for the property drawer." + (concat "* " title + (and tags (concat " :" (mapconcat #'identity tags ":") ":")) + "\n:PROPERTIES:\n" + (mapconcat (lambda (p) (format ":%s: %s\n" (car p) (cdr p))) + props "") + ":END:\n" + body "\n")) + +(defun elel-import--demote (body) + "Demote every heading in BODY by one level. +Return (NEW-BODY . NUMBER-OF-HEADINGS)." + (let ((count 0) + (pos 0)) + (while (string-match "^\\*+ " body pos) + (setq count (1+ count)) + (setq pos (match-end 0))) + (cons (replace-regexp-in-string "^\\(\\*+\\) " "*\\1 " body) + count))) + +(defun elel-import--finish (result out dry-run) + "Report RESULT, and write its files to OUT unless DRY-RUN or no OUT." + (let ((files (plist-get result :files))) + (message "%s" (mapconcat #'identity (plist-get result :report) "\n")) + (cond + ((or dry-run (null out)) + (message "elel import: dry run, nothing written.")) + (t + (let* ((out (expand-file-name out)) + (targets (mapcar (lambda (f) (expand-file-name (car f) out)) + files)) + (clash (cl-find-if #'file-exists-p targets))) + (cond + (clash + (user-error "elel import: %s already exists; nothing written" + clash)) + ((and (not noninteractive) + (not (y-or-n-p (format "Write %d files to %s? " + (length files) out)))) + (user-error "elel import: cancelled")) + (t + (cl-mapc + (lambda (f target) + (make-directory (file-name-directory target) t) + (let ((coding-system-for-write 'utf-8-unix)) + (write-region (cdr f) nil target nil 'silent))) + files targets) + (message "elel import: wrote %d files to %s" + (length files) out)))))))) + +;;; Gnus drafts + +(defun elel-import--header (headers name) + "The value of the header NAME in the string HEADERS, or nil." + (let ((case-fold-search t) + (flat (replace-regexp-in-string "\n[ \t]+" " " headers))) + (and (string-match + (concat "^" (regexp-quote name) ":[ \t]*\\(.*\\)$") flat) + (string-trim (match-string 1 flat))))) + +(defun elel-import--decode-body (body headers) + "Decode BODY according to the transfer encoding and charset in HEADERS." + (let* ((cte (downcase (or (elel-import--header + headers "Content-Transfer-Encoding") + ""))) + (type (or (elel-import--header headers "Content-Type") "")) + (charset (cond + ((string-match "charset=\"?\\([^\";[:space:]]+\\)" type) + (match-string 1 type)) + ("utf-8"))) + (coding (or (ignore-errors (coding-system-from-name charset)) + 'utf-8))) + (cond + ((string= cte "quoted-printable") + (decode-coding-string (quoted-printable-decode-string body) coding)) + ((string= cte "base64") + (decode-coding-string (base64-decode-string body) coding)) + (body)))) + +(defun elel-import--strip-signature (body) + "Remove a short signature (\"-- \" and up to three lines) from BODY." + (let ((pos nil) + (start 0)) + (while (string-match "\n-- \n" body start) + (setq pos (match-beginning 0)) + (setq start (match-end 0))) + (cond + ((and pos (<= (cl-count ?\n (substring body pos)) 4)) + (substring body 0 pos)) + (body)))) + +(defun elel-import--draft (path) + "Parse the draft file PATH. Return a plist, or nil if it is no note." + (let* ((text (elel-import--read path)) + (number (file-name-nondirectory path))) + (cond + ((not (string-match "^--text follows this line--\n" text)) + (list :skip (format "draft %s: no header separator" number))) + (t + (let* ((headers (substring text 0 (match-beginning 0))) + (body (substring text (match-end 0))) + (gcc (or (elel-import--header headers "Gcc") "")) + (subject (or (elel-import--header headers "Subject") "")) + (irt (elel-import--header headers "In-Reply-To")) + (mtime (file-attribute-modification-time + (file-attributes path)))) + (cond + ((not (string-match "\\`nnmaildir\\+notes:\\(.+\\)\\'" gcc)) + (list :skip (format "draft %s (%s): Gcc is %s, not a notes group" + number (rfc2047-decode-string subject) + (if (string-empty-p gcc) "empty" gcc)))) + (t + (list + :group (match-string 1 gcc) + :number number + :title (rfc2047-decode-string subject) + :id (or (elel-import--header headers "Message-ID") + (funcall elel-id-function)) + :parent (and irt (string-match "<[^>]+>" irt) + (match-string 0 irt)) + :time mtime + :body (string-trim + (elel-import--strip-signature + (string-trim + (elel-import--decode-body body headers)))))))))))) + +;;;###autoload +(defun elel-import-drafts (src out &optional dry-run) + "Import the notes kept in the Gnus draft directory SRC into OUT. +Only drafts whose Gcc is a nnmaildir+notes:GROUP group are notes; +each GROUP becomes the container file GROUP.org, and a tag. With a +nil OUT, or a non-nil DRY-RUN, only report." + (interactive + (list (read-directory-name "Draft directory: " "~/News/drafts/drafts/") + (read-directory-name "Write to: " elel-root) + current-prefix-arg)) + (let* ((paths (sort (directory-files (expand-file-name src) t "\\`[0-9]+\\'") + (lambda (a b) + (< (string-to-number (file-name-nondirectory a)) + (string-to-number (file-name-nondirectory b)))))) + (parsed (mapcar #'elel-import--draft paths)) + (notes (cl-remove-if (lambda (n) (plist-get n :skip)) parsed)) + (ids (mapcar (lambda (n) (plist-get n :id)) notes)) + (report (list (format "source : %s" (expand-file-name src)) + (format "drafts : %d, notes: %d" + (length paths) (length notes)))) + (groups nil) + (files nil) + (demoted 0)) + (dolist (n parsed) + (cond + ((plist-get n :skip) + (push (concat " skipped " (plist-get n :skip)) report)))) + (dolist (n notes) + (let ((cell (assoc (plist-get n :group) groups))) + (cond + (cell (setcdr cell (cons n (cdr cell)))) + ((push (list (plist-get n :group) n) groups))))) + (dolist (g (reverse groups)) + (let* ((group (car g)) + (sorted (sort (cdr g) + (lambda (a b) + (time-less-p (plist-get a :time) + (plist-get b :time))))) + (texts nil)) + (dolist (n sorted) + (let* ((demotion (elel-import--demote (plist-get n :body))) + (parent (plist-get n :parent)) + (known (and parent (member parent ids)))) + (setq demoted (+ demoted (cdr demotion))) + (cond + ((and parent (not known)) + (push (format " note %s: parent %s is not among the notes" + (plist-get n :number) parent) + report))) + (push (elel-import--org-note + (plist-get n :title) + (list group) + (append + (list (cons "ID" (plist-get n :id)) + (cons "DATE" (elel--now (plist-get n :time)))) + (and known (list (cons "PARENT" parent))) + (list (cons "DRAFT_FILE" (plist-get n :number)))) + (car demotion)) + texts))) + (push (cons (concat group ".org") + (mapconcat #'identity (nreverse texts) "\n")) + files) + (push (format " %s.org: %d notes" group (length sorted)) report))) + (push (format "headings demoted : %d" demoted) report) + (push "DATE is the modification time of the draft file (drafts have no Date header)." + report) + (elel-import--finish + (list :files (nreverse files) :report (reverse report)) + out dry-run))) + +;;; org-roam + +(defconst elel-import--file-link-regexp + "\\[\\[file:\\([^]\n]*\\)\\]\\(?:\\[\\([^]\n]*\\)\\]\\)?\\]" + "Matches an org file link. Group 1 is the path, group 2 the description.") + +(defun elel-import--keyword (text name) + "The value of the keyword #+NAME: in TEXT, or nil." + (let ((case-fold-search t)) + (and (string-match + (concat "^#\\+" name ":[ \t]*\\(.*?\\)[ \t]*$") text) + (not (string-empty-p (match-string 1 text))) + (match-string 1 text)))) + +(defun elel-import--stamp (path) + "A compact time stamp such as 20200514T232108 for the roam file PATH." + (let ((name (file-name-nondirectory path))) + (cond + ((or (string-match + (concat "\\`\\([0-9]\\{4\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)" + "\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)-") + name) + (string-match + (concat "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)" + "--\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-") + name)) + (format "%s%s%sT%s%s%s" + (match-string 1 name) (match-string 2 name) + (match-string 3 name) (match-string 4 name) + (match-string 5 name) (match-string 6 name))) + ((string-match + "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)\\.org\\'" + name) + (format "%s%s%sT000000" + (match-string 1 name) (match-string 2 name) + (match-string 3 name))) + ((format-time-string + "%Y%m%dT%H%M%S" + (file-attribute-modification-time (file-attributes path))))))) + +(defun elel-import--stamp-time (stamp) + "The time of STAMP, a string such as 20200514T232108, in local time." + (let ((n (lambda (a b) (string-to-number (substring stamp a b))))) + (encode-time (list (funcall n 13 15) (funcall n 11 13) (funcall n 9 11) + (funcall n 6 8) (funcall n 4 6) (funcall n 0 4) + nil -1 nil)))) + +(defun elel-import--cover (items min) + "Choose one tag for each of ITEMS, so that few tags are used. +ITEMS is a list of (KEY . TAGS). Return a hash table from KEY to the +chosen tag, or to nothing when no good tag exists. Tags are picked +greedily, each time the one that covers most of the notes left, until +no tag covers MIN notes; the notes left are not in the table." + (let ((left (cl-remove-if-not #'cdr items)) + (chosen (make-hash-table :test 'equal)) + (counts (make-hash-table :test 'equal)) + (continue t)) + (while (and left continue) + (clrhash counts) + (dolist (item left) + (dolist (tag (cdr item)) + (puthash tag (1+ (gethash tag counts 0)) counts))) + (let (best (best-count 0)) + (maphash (lambda (tag count) + (cond + ((or (> count best-count) + (and (= count best-count) best (string< tag best))) + (setq best tag best-count count)))) + counts) + (cond + ((or (null best) (< best-count min)) (setq continue nil)) + (t + (dolist (item left) + (cond ((member best (cdr item)) (puthash (car item) best chosen)))) + (setq left (cl-remove-if (lambda (item) (member best (cdr item))) + left)))))) + chosen)) + +(defun elel-import--target (rel _tags &optional chosen) + "The container file for the roam file REL, whose tags are TAGS. +A journal goes to journal.org; otherwise the tag CHOSEN names the +file; otherwise the directory does; otherwise the file is roam.org." + (let ((dir (file-name-directory rel))) + (cond + ((string-prefix-p "journal/" rel) "journal.org") + (chosen (concat chosen ".org")) + (dir (concat (replace-regexp-in-string + "[ \t]+" "-" (directory-file-name dir)) + ".org")) + ("roam.org")))) + +(defun elel-import--tags (text) + "The #+roam_tags of TEXT, made into valid org tags." + (let ((raw (elel-import--keyword text "roam_tags")) + (tags nil)) + (dolist (w (and raw (or (ignore-errors (split-string-and-unquote raw)) + (split-string raw)))) + (let ((tag (string-trim + (replace-regexp-in-string "[^[:alnum:]_@#%]" "_" w) + "_+" "_+"))) + (cond + ((and (not (string-empty-p tag)) (not (member tag tags))) + (push tag tags))))) + (nreverse tags))) + +(defcustom elel-import-roam-min-group 2 + "The fewest notes that a file made from a tag may hold. +A note whose tags would make smaller files goes to roam.org instead." + :type 'integer + :group 'elel) + +;;;###autoload +(defun elel-import-roam (src out &optional dry-run) + "Import the org-roam directory SRC into the directory OUT. +Each roam file becomes one note. The notes are collected into +container files, one per first tag (see `elel-import--target'), sorted +by date. With a nil OUT, or a non-nil DRY-RUN, only report." + (interactive + (list (read-directory-name "Roam directory: " "~/org/roam/") + (read-directory-name "Write to: " (expand-file-name "roam" elel-root)) + current-prefix-arg)) + (let* ((src (file-name-as-directory (expand-file-name src))) + (paths (cl-remove-if + (lambda (p) + (string-match-p "\\(\\`\\|/\\)\\." + (file-relative-name p src))) + (sort (directory-files-recursively src "\\.org\\'") + #'string<))) + (notes (make-hash-table :test 'equal)) + (links 0) + (demoted 0) + (unresolved nil) + (drawers nil) + (cover nil) + (groups nil) + (group-report nil) + (files nil)) + (dolist (path paths) + (let* ((rel (file-relative-name path src)) + (text (elel-import--read path)) + (stamp (elel-import--stamp path))) + (puthash path + (list :rel rel + :text text + :title (or (elel-import--keyword text "title") + (file-name-base path)) + :stamp stamp + :id (concat stamp "-" + (substring (md5 rel nil nil 'utf-8) 0 4)) + :tags (elel-import--tags text)) + notes))) + (setq cover + (elel-import--cover + (cl-loop for path in paths + for n = (gethash path notes) + unless (string-prefix-p "journal/" (plist-get n :rel)) + collect (cons (plist-get n :rel) (plist-get n :tags))) + elel-import-roam-min-group)) + (dolist (path paths) + (let* ((n (gethash path notes)) + (rel (plist-get n :rel)) + (body (let ((case-fold-search t)) + (string-trim + (replace-regexp-in-string + "^#\\+\\(?:title\\|roam_tags\\):.*\n?" "" + (plist-get n :text)) + "\n+" "\n+"))) + (rewritten + (replace-regexp-in-string + elel-import--file-link-regexp + (lambda (m) + (string-match elel-import--file-link-regexp m) + (let* ((target (match-string 1 m)) + (desc (match-string 2 m)) + (p (car (split-string target "::" t))) + (found (and p + (gethash + (expand-file-name + p (file-name-directory path)) + notes)))) + (cond + (found + (setq links (1+ links)) + (concat "[[note:" (plist-get found :id) "][" + (cond + ((or (null desc) (string-empty-p desc)) + (plist-get found :title)) + (desc)) + "]]")) + (t + (cond + ((and p (string-suffix-p ".org" p)) + (push (cons rel target) unresolved))) + m)))) + body t t)) + (demotion (elel-import--demote rewritten))) + (cond + ((string-match-p "\\`[ \t\n]*:PROPERTIES:" body) + (push rel drawers))) + (setq demoted (+ demoted (cdr demotion))) + (let* ((target (elel-import--target rel (plist-get n :tags) + (gethash rel cover))) + (cell (assoc target groups)) + (item (cons + (plist-get n :stamp) + (elel-import--org-note + (plist-get n :title) + (plist-get n :tags) + (append + (list (cons "ID" (plist-get n :id)) + (cons "DATE" + (elel--now (elel-import--stamp-time + (plist-get n :stamp))))) + (and (string-prefix-p "journal/" rel) + (list (cons "KIND" "journal"))) + (list (cons "ROAM_FILE" rel))) + (car demotion))))) + (cond + (cell (setcdr cell (cons item (cdr cell)))) + ((push (list target item) groups)))))) + (dolist (g (sort groups (lambda (a b) (string< (car a) (car b))))) + (let ((sorted (sort (cdr g) (lambda (a b) (string< (car a) (car b)))))) + (push (cons (car g) + (mapconcat #'cdr sorted "\n")) + files) + (push (format " %s: %d notes" (car g) (length sorted)) group-report))) + (elel-import--finish + (list + :files (nreverse files) + :report + (append + (list (format "source : %s" src) + (format "notes : %d" (length paths)) + (format "with tags : %d" + (cl-count-if (lambda (p) + (plist-get (gethash p notes) :tags)) + paths)) + (format "links to notes rewritten : %d" links) + (format "headings demoted : %d" demoted)) + (reverse group-report) + (mapcar (lambda (u) + (format " unresolved .org link in %s: %s" + (car u) (cdr u))) + (reverse unresolved)) + (mapcar (lambda (rel) + (format " warning: %s starts with a property drawer" rel)) + (reverse drawers)))) + out dry-run))) + +(provide 'elel-import) +;;; elel-import.el ends here |
