;;; elel-import.el --- Import old notes into elel -*- lexical-binding: t; -*- ;; Copyright (C) 2026 Durand ;; Author: Durand ;; Keywords: outlines, files ;; This program is free software; you can redistribute it and/or modify ;; it under the terms of the GNU General Public License as published by ;; the Free Software Foundation, either version 3 of the License, or ;; (at your option) any later version. ;; This program is distributed in the hope that it will be useful, ;; but WITHOUT ANY WARRANTY; without even the implied warranty of ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;; GNU General Public License for more details. ;; You should have received a copy of the GNU General Public License ;; along with this program. If not, see . ;;; Commentary: ;; Two importers, which only read their sources: ;; ;; `elel-import-drafts' -- notes kept as Gnus drafts, i.e. messages ;; whose Gcc is a nnmaildir+notes:GROUP group. Each GROUP becomes ;; one container file GROUP.org, and the Message-IDs become the ;; IDs, so the In-Reply-To threads become PARENT properties. ;; ;; `elel-import-roam' -- an org-roam (version 1, file based) directory. ;; Each file becomes one note, and the links to other roam files ;; become [[note:ID][TITLE]] links. ;; ;; Both take the source, the output directory, and an optional DRY-RUN. ;; With a nil output directory, they only report what they would do. ;; They never overwrite: if a target file exists, nothing is written. ;; ;; From a shell: ;; ;; emacs --batch -L DIR -l elel-import \ ;; --eval '(elel-import-drafts "~/News/drafts/drafts" "~/notes")' ;;; Code: (require 'cl-lib) (require 'subr-x) (require 'rfc2047) (require 'qp) (require 'elel) ;;; Common helpers (defun elel-import--read (path) "The content of the file PATH, decoded as UTF-8." (with-temp-buffer (let ((coding-system-for-read 'utf-8)) (insert-file-contents path)) (buffer-string))) (defun elel-import--org-note (title tags props body) "The text of a note in org format. PROPS is an alist of (NAME . VALUE) for the property drawer." (concat "* " title (and tags (concat " :" (mapconcat #'identity tags ":") ":")) "\n:PROPERTIES:\n" (mapconcat (lambda (p) (format ":%s: %s\n" (car p) (cdr p))) props "") ":END:\n" body "\n")) (defun elel-import--demote (body) "Demote every heading in BODY by one level. Return (NEW-BODY . NUMBER-OF-HEADINGS)." (let ((count 0) (pos 0)) (while (string-match "^\\*+ " body pos) (setq count (1+ count)) (setq pos (match-end 0))) (cons (replace-regexp-in-string "^\\(\\*+\\) " "*\\1 " body) count))) (defun elel-import--finish (result out dry-run) "Report RESULT, and write its files to OUT unless DRY-RUN or no OUT." (let ((files (plist-get result :files))) (message "%s" (mapconcat #'identity (plist-get result :report) "\n")) (cond ((or dry-run (null out)) (message "elel import: dry run, nothing written.")) (t (let* ((out (expand-file-name out)) (targets (mapcar (lambda (f) (expand-file-name (car f) out)) files)) (clash (cl-find-if #'file-exists-p targets))) (cond (clash (user-error "elel import: %s already exists; nothing written" clash)) ((and (not noninteractive) (not (y-or-n-p (format "Write %d files to %s? " (length files) out)))) (user-error "elel import: cancelled")) (t (cl-mapc (lambda (f target) (make-directory (file-name-directory target) t) (let ((coding-system-for-write 'utf-8-unix)) (write-region (cdr f) nil target nil 'silent))) files targets) (message "elel import: wrote %d files to %s" (length files) out)))))))) ;;; Gnus drafts (defun elel-import--header (headers name) "The value of the header NAME in the string HEADERS, or nil." (let ((case-fold-search t) (flat (replace-regexp-in-string "\n[ \t]+" " " headers))) (and (string-match (concat "^" (regexp-quote name) ":[ \t]*\\(.*\\)$") flat) (string-trim (match-string 1 flat))))) (defun elel-import--decode-body (body headers) "Decode BODY according to the transfer encoding and charset in HEADERS." (let* ((cte (downcase (or (elel-import--header headers "Content-Transfer-Encoding") ""))) (type (or (elel-import--header headers "Content-Type") "")) (charset (cond ((string-match "charset=\"?\\([^\";[:space:]]+\\)" type) (match-string 1 type)) ("utf-8"))) (coding (or (ignore-errors (coding-system-from-name charset)) 'utf-8))) (cond ((string= cte "quoted-printable") (decode-coding-string (quoted-printable-decode-string body) coding)) ((string= cte "base64") (decode-coding-string (base64-decode-string body) coding)) (body)))) (defun elel-import--strip-signature (body) "Remove a short signature (\"-- \" and up to three lines) from BODY." (let ((pos nil) (start 0)) (while (string-match "\n-- \n" body start) (setq pos (match-beginning 0)) (setq start (match-end 0))) (cond ((and pos (<= (cl-count ?\n (substring body pos)) 4)) (substring body 0 pos)) (body)))) (defun elel-import--draft (path) "Parse the draft file PATH. Return a plist, or nil if it is no note." (let* ((text (elel-import--read path)) (number (file-name-nondirectory path))) (cond ((not (string-match "^--text follows this line--\n" text)) (list :skip (format "draft %s: no header separator" number))) (t (let* ((headers (substring text 0 (match-beginning 0))) (body (substring text (match-end 0))) (gcc (or (elel-import--header headers "Gcc") "")) (subject (or (elel-import--header headers "Subject") "")) (irt (elel-import--header headers "In-Reply-To")) (mtime (file-attribute-modification-time (file-attributes path)))) (cond ((not (string-match "\\`nnmaildir\\+notes:\\(.+\\)\\'" gcc)) (list :skip (format "draft %s (%s): Gcc is %s, not a notes group" number (rfc2047-decode-string subject) (if (string-empty-p gcc) "empty" gcc)))) (t (list :group (match-string 1 gcc) :number number :title (rfc2047-decode-string subject) :id (or (elel-import--header headers "Message-ID") (funcall elel-id-function)) :parent (and irt (string-match "<[^>]+>" irt) (match-string 0 irt)) :time mtime :body (string-trim (elel-import--strip-signature (string-trim (elel-import--decode-body body headers)))))))))))) ;;;###autoload (defun elel-import-drafts (src out &optional dry-run) "Import the notes kept in the Gnus draft directory SRC into OUT. Only drafts whose Gcc is a nnmaildir+notes:GROUP group are notes; each GROUP becomes the container file GROUP.org, and a tag. With a nil OUT, or a non-nil DRY-RUN, only report." (interactive (list (read-directory-name "Draft directory: " "~/News/drafts/drafts/") (read-directory-name "Write to: " elel-root) current-prefix-arg)) (let* ((paths (sort (directory-files (expand-file-name src) t "\\`[0-9]+\\'") (lambda (a b) (< (string-to-number (file-name-nondirectory a)) (string-to-number (file-name-nondirectory b)))))) (parsed (mapcar #'elel-import--draft paths)) (notes (cl-remove-if (lambda (n) (plist-get n :skip)) parsed)) (ids (mapcar (lambda (n) (plist-get n :id)) notes)) (report (list (format "source : %s" (expand-file-name src)) (format "drafts : %d, notes: %d" (length paths) (length notes)))) (groups nil) (files nil) (demoted 0)) (dolist (n parsed) (cond ((plist-get n :skip) (push (concat " skipped " (plist-get n :skip)) report)))) (dolist (n notes) (let ((cell (assoc (plist-get n :group) groups))) (cond (cell (setcdr cell (cons n (cdr cell)))) ((push (list (plist-get n :group) n) groups))))) (dolist (g (reverse groups)) (let* ((group (car g)) (sorted (sort (cdr g) (lambda (a b) (time-less-p (plist-get a :time) (plist-get b :time))))) (texts nil)) (dolist (n sorted) (let* ((demotion (elel-import--demote (plist-get n :body))) (parent (plist-get n :parent)) (known (and parent (member parent ids)))) (setq demoted (+ demoted (cdr demotion))) (cond ((and parent (not known)) (push (format " note %s: parent %s is not among the notes" (plist-get n :number) parent) report))) (push (elel-import--org-note (plist-get n :title) (list group) (append (list (cons "ID" (plist-get n :id)) (cons "DATE" (elel--now (plist-get n :time)))) (and known (list (cons "PARENT" parent))) (list (cons "DRAFT_FILE" (plist-get n :number)))) (car demotion)) texts))) (push (cons (concat group ".org") (mapconcat #'identity (nreverse texts) "\n")) files) (push (format " %s.org: %d notes" group (length sorted)) report))) (push (format "headings demoted : %d" demoted) report) (push "DATE is the modification time of the draft file (drafts have no Date header)." report) (elel-import--finish (list :files (nreverse files) :report (reverse report)) out dry-run))) ;;; org-roam (defconst elel-import--file-link-regexp "\\[\\[file:\\([^]\n]*\\)\\]\\(?:\\[\\([^]\n]*\\)\\]\\)?\\]" "Matches an org file link. Group 1 is the path, group 2 the description.") (defun elel-import--keyword (text name) "The value of the keyword #+NAME: in TEXT, or nil." (let ((case-fold-search t)) (and (string-match (concat "^#\\+" name ":[ \t]*\\(.*?\\)[ \t]*$") text) (not (string-empty-p (match-string 1 text))) (match-string 1 text)))) (defun elel-import--stamp (path) "A compact time stamp such as 20200514T232108 for the roam file PATH." (let ((name (file-name-nondirectory path))) (cond ((or (string-match (concat "\\`\\([0-9]\\{4\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)" "\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)-") name) (string-match (concat "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)" "--\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-") name)) (format "%s%s%sT%s%s%s" (match-string 1 name) (match-string 2 name) (match-string 3 name) (match-string 4 name) (match-string 5 name) (match-string 6 name))) ((string-match "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)\\.org\\'" name) (format "%s%s%sT000000" (match-string 1 name) (match-string 2 name) (match-string 3 name))) ((format-time-string "%Y%m%dT%H%M%S" (file-attribute-modification-time (file-attributes path))))))) (defun elel-import--stamp-time (stamp) "The time of STAMP, a string such as 20200514T232108, in local time." (let ((n (lambda (a b) (string-to-number (substring stamp a b))))) (encode-time (list (funcall n 13 15) (funcall n 11 13) (funcall n 9 11) (funcall n 6 8) (funcall n 4 6) (funcall n 0 4) nil -1 nil)))) (defun elel-import--cover (items min) "Choose one tag for each of ITEMS, so that few tags are used. ITEMS is a list of (KEY . TAGS). Return a hash table from KEY to the chosen tag, or to nothing when no good tag exists. Tags are picked greedily, each time the one that covers most of the notes left, until no tag covers MIN notes; the notes left are not in the table." (let ((left (cl-remove-if-not #'cdr items)) (chosen (make-hash-table :test 'equal)) (counts (make-hash-table :test 'equal)) (continue t)) (while (and left continue) (clrhash counts) (dolist (item left) (dolist (tag (cdr item)) (puthash tag (1+ (gethash tag counts 0)) counts))) (let (best (best-count 0)) (maphash (lambda (tag count) (cond ((or (> count best-count) (and (= count best-count) best (string< tag best))) (setq best tag best-count count)))) counts) (cond ((or (null best) (< best-count min)) (setq continue nil)) (t (dolist (item left) (cond ((member best (cdr item)) (puthash (car item) best chosen)))) (setq left (cl-remove-if (lambda (item) (member best (cdr item))) left)))))) chosen)) (defun elel-import--target (rel _tags &optional chosen) "The container file for the roam file REL, whose tags are TAGS. A journal goes to journal.org; otherwise the tag CHOSEN names the file; otherwise the directory does; otherwise the file is roam.org." (let ((dir (file-name-directory rel))) (cond ((string-prefix-p "journal/" rel) "journal.org") (chosen (concat chosen ".org")) (dir (concat (replace-regexp-in-string "[ \t]+" "-" (directory-file-name dir)) ".org")) ("roam.org")))) (defun elel-import--tags (text) "The #+roam_tags of TEXT, made into valid org tags." (let ((raw (elel-import--keyword text "roam_tags")) (tags nil)) (dolist (w (and raw (or (ignore-errors (split-string-and-unquote raw)) (split-string raw)))) (let ((tag (string-trim (replace-regexp-in-string "[^[:alnum:]_@#%]" "_" w) "_+" "_+"))) (cond ((and (not (string-empty-p tag)) (not (member tag tags))) (push tag tags))))) (nreverse tags))) (defcustom elel-import-roam-min-group 2 "The fewest notes that a file made from a tag may hold. A note whose tags would make smaller files goes to roam.org instead." :type 'integer :group 'elel) ;;;###autoload (defun elel-import-roam (src out &optional dry-run) "Import the org-roam directory SRC into the directory OUT. Each roam file becomes one note. The notes are collected into container files, one per first tag (see `elel-import--target'), sorted by date. With a nil OUT, or a non-nil DRY-RUN, only report." (interactive (list (read-directory-name "Roam directory: " "~/org/roam/") (read-directory-name "Write to: " (expand-file-name "roam" elel-root)) current-prefix-arg)) (let* ((src (file-name-as-directory (expand-file-name src))) (paths (cl-remove-if (lambda (p) (string-match-p "\\(\\`\\|/\\)\\." (file-relative-name p src))) (sort (directory-files-recursively src "\\.org\\'") #'string<))) (notes (make-hash-table :test 'equal)) (links 0) (demoted 0) (unresolved nil) (drawers nil) (cover nil) (groups nil) (group-report nil) (files nil)) (dolist (path paths) (let* ((rel (file-relative-name path src)) (text (elel-import--read path)) (stamp (elel-import--stamp path))) (puthash path (list :rel rel :text text :title (or (elel-import--keyword text "title") (file-name-base path)) :stamp stamp :id (concat stamp "-" (substring (md5 rel nil nil 'utf-8) 0 4)) :tags (elel-import--tags text)) notes))) (setq cover (elel-import--cover (cl-loop for path in paths for n = (gethash path notes) unless (string-prefix-p "journal/" (plist-get n :rel)) collect (cons (plist-get n :rel) (plist-get n :tags))) elel-import-roam-min-group)) (dolist (path paths) (let* ((n (gethash path notes)) (rel (plist-get n :rel)) (body (let ((case-fold-search t)) (string-trim (replace-regexp-in-string "^#\\+\\(?:title\\|roam_tags\\):.*\n?" "" (plist-get n :text)) "\n+" "\n+"))) (rewritten (replace-regexp-in-string elel-import--file-link-regexp (lambda (m) (string-match elel-import--file-link-regexp m) (let* ((target (match-string 1 m)) (desc (match-string 2 m)) (p (car (split-string target "::" t))) (found (and p (gethash (expand-file-name p (file-name-directory path)) notes)))) (cond (found (setq links (1+ links)) (concat "[[note:" (plist-get found :id) "][" (cond ((or (null desc) (string-empty-p desc)) (plist-get found :title)) (desc)) "]]")) (t (cond ((and p (string-suffix-p ".org" p)) (push (cons rel target) unresolved))) m)))) body t t)) (demotion (elel-import--demote rewritten))) (cond ((string-match-p "\\`[ \t\n]*:PROPERTIES:" body) (push rel drawers))) (setq demoted (+ demoted (cdr demotion))) (let* ((target (elel-import--target rel (plist-get n :tags) (gethash rel cover))) (cell (assoc target groups)) (item (cons (plist-get n :stamp) (elel-import--org-note (plist-get n :title) (plist-get n :tags) (append (list (cons "ID" (plist-get n :id)) (cons "DATE" (elel--now (elel-import--stamp-time (plist-get n :stamp))))) (and (string-prefix-p "journal/" rel) (list (cons "KIND" "journal"))) (list (cons "ROAM_FILE" rel))) (car demotion))))) (cond (cell (setcdr cell (cons item (cdr cell)))) ((push (list target item) groups)))))) (dolist (g (sort groups (lambda (a b) (string< (car a) (car b))))) (let ((sorted (sort (cdr g) (lambda (a b) (string< (car a) (car b)))))) (push (cons (car g) (mapconcat #'cdr sorted "\n")) files) (push (format " %s: %d notes" (car g) (length sorted)) group-report))) (elel-import--finish (list :files (nreverse files) :report (append (list (format "source : %s" src) (format "notes : %d" (length paths)) (format "with tags : %d" (cl-count-if (lambda (p) (plist-get (gethash p notes) :tags)) paths)) (format "links to notes rewritten : %d" links) (format "headings demoted : %d" demoted)) (reverse group-report) (mapcar (lambda (u) (format " unresolved .org link in %s: %s" (car u) (cdr u))) (reverse unresolved)) (mapcar (lambda (rel) (format " warning: %s starts with a property drawer" rel)) (reverse drawers)))) out dry-run))) (provide 'elel-import) ;;; elel-import.el ends here