aboutsummaryrefslogtreecommitdiff
path: root/elel-import.el
diff options
context:
space:
mode:
Diffstat (limited to 'elel-import.el')
-rw-r--r--elel-import.el508
1 files changed, 508 insertions, 0 deletions
diff --git a/elel-import.el b/elel-import.el
new file mode 100644
index 0000000..0dd2b9c
--- /dev/null
+++ b/elel-import.el
@@ -0,0 +1,508 @@
+;;; elel-import.el --- Import old notes into elel -*- lexical-binding: t; -*-
+
+;; Copyright (C) 2026 Durand
+
+;; Author: Durand
+;; Keywords: outlines, files
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation, either version 3 of the License, or
+;; (at your option) any later version.
+
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+;;; Commentary:
+
+;; Two importers, which only read their sources:
+;;
+;; `elel-import-drafts' -- notes kept as Gnus drafts, i.e. messages
+;; whose Gcc is a nnmaildir+notes:GROUP group. Each GROUP becomes
+;; one container file GROUP.org, and the Message-IDs become the
+;; IDs, so the In-Reply-To threads become PARENT properties.
+;;
+;; `elel-import-roam' -- an org-roam (version 1, file based) directory.
+;; Each file becomes one note, and the links to other roam files
+;; become [[note:ID][TITLE]] links.
+;;
+;; Both take the source, the output directory, and an optional DRY-RUN.
+;; With a nil output directory, they only report what they would do.
+;; They never overwrite: if a target file exists, nothing is written.
+;;
+;; From a shell:
+;;
+;; emacs --batch -L DIR -l elel-import \
+;; --eval '(elel-import-drafts "~/News/drafts/drafts" "~/notes")'
+
+;;; Code:
+
+(require 'cl-lib)
+(require 'subr-x)
+(require 'rfc2047)
+(require 'qp)
+(require 'elel)
+
+;;; Common helpers
+
+(defun elel-import--read (path)
+ "The content of the file PATH, decoded as UTF-8."
+ (with-temp-buffer
+ (let ((coding-system-for-read 'utf-8))
+ (insert-file-contents path))
+ (buffer-string)))
+
+(defun elel-import--org-note (title tags props body)
+ "The text of a note in org format.
+PROPS is an alist of (NAME . VALUE) for the property drawer."
+ (concat "* " title
+ (and tags (concat " :" (mapconcat #'identity tags ":") ":"))
+ "\n:PROPERTIES:\n"
+ (mapconcat (lambda (p) (format ":%s: %s\n" (car p) (cdr p)))
+ props "")
+ ":END:\n"
+ body "\n"))
+
+(defun elel-import--demote (body)
+ "Demote every heading in BODY by one level.
+Return (NEW-BODY . NUMBER-OF-HEADINGS)."
+ (let ((count 0)
+ (pos 0))
+ (while (string-match "^\\*+ " body pos)
+ (setq count (1+ count))
+ (setq pos (match-end 0)))
+ (cons (replace-regexp-in-string "^\\(\\*+\\) " "*\\1 " body)
+ count)))
+
+(defun elel-import--finish (result out dry-run)
+ "Report RESULT, and write its files to OUT unless DRY-RUN or no OUT."
+ (let ((files (plist-get result :files)))
+ (message "%s" (mapconcat #'identity (plist-get result :report) "\n"))
+ (cond
+ ((or dry-run (null out))
+ (message "elel import: dry run, nothing written."))
+ (t
+ (let* ((out (expand-file-name out))
+ (targets (mapcar (lambda (f) (expand-file-name (car f) out))
+ files))
+ (clash (cl-find-if #'file-exists-p targets)))
+ (cond
+ (clash
+ (user-error "elel import: %s already exists; nothing written"
+ clash))
+ ((and (not noninteractive)
+ (not (y-or-n-p (format "Write %d files to %s? "
+ (length files) out))))
+ (user-error "elel import: cancelled"))
+ (t
+ (cl-mapc
+ (lambda (f target)
+ (make-directory (file-name-directory target) t)
+ (let ((coding-system-for-write 'utf-8-unix))
+ (write-region (cdr f) nil target nil 'silent)))
+ files targets)
+ (message "elel import: wrote %d files to %s"
+ (length files) out))))))))
+
+;;; Gnus drafts
+
+(defun elel-import--header (headers name)
+ "The value of the header NAME in the string HEADERS, or nil."
+ (let ((case-fold-search t)
+ (flat (replace-regexp-in-string "\n[ \t]+" " " headers)))
+ (and (string-match
+ (concat "^" (regexp-quote name) ":[ \t]*\\(.*\\)$") flat)
+ (string-trim (match-string 1 flat)))))
+
+(defun elel-import--decode-body (body headers)
+ "Decode BODY according to the transfer encoding and charset in HEADERS."
+ (let* ((cte (downcase (or (elel-import--header
+ headers "Content-Transfer-Encoding")
+ "")))
+ (type (or (elel-import--header headers "Content-Type") ""))
+ (charset (cond
+ ((string-match "charset=\"?\\([^\";[:space:]]+\\)" type)
+ (match-string 1 type))
+ ("utf-8")))
+ (coding (or (ignore-errors (coding-system-from-name charset))
+ 'utf-8)))
+ (cond
+ ((string= cte "quoted-printable")
+ (decode-coding-string (quoted-printable-decode-string body) coding))
+ ((string= cte "base64")
+ (decode-coding-string (base64-decode-string body) coding))
+ (body))))
+
+(defun elel-import--strip-signature (body)
+ "Remove a short signature (\"-- \" and up to three lines) from BODY."
+ (let ((pos nil)
+ (start 0))
+ (while (string-match "\n-- \n" body start)
+ (setq pos (match-beginning 0))
+ (setq start (match-end 0)))
+ (cond
+ ((and pos (<= (cl-count ?\n (substring body pos)) 4))
+ (substring body 0 pos))
+ (body))))
+
+(defun elel-import--draft (path)
+ "Parse the draft file PATH. Return a plist, or nil if it is no note."
+ (let* ((text (elel-import--read path))
+ (number (file-name-nondirectory path)))
+ (cond
+ ((not (string-match "^--text follows this line--\n" text))
+ (list :skip (format "draft %s: no header separator" number)))
+ (t
+ (let* ((headers (substring text 0 (match-beginning 0)))
+ (body (substring text (match-end 0)))
+ (gcc (or (elel-import--header headers "Gcc") ""))
+ (subject (or (elel-import--header headers "Subject") ""))
+ (irt (elel-import--header headers "In-Reply-To"))
+ (mtime (file-attribute-modification-time
+ (file-attributes path))))
+ (cond
+ ((not (string-match "\\`nnmaildir\\+notes:\\(.+\\)\\'" gcc))
+ (list :skip (format "draft %s (%s): Gcc is %s, not a notes group"
+ number (rfc2047-decode-string subject)
+ (if (string-empty-p gcc) "empty" gcc))))
+ (t
+ (list
+ :group (match-string 1 gcc)
+ :number number
+ :title (rfc2047-decode-string subject)
+ :id (or (elel-import--header headers "Message-ID")
+ (funcall elel-id-function))
+ :parent (and irt (string-match "<[^>]+>" irt)
+ (match-string 0 irt))
+ :time mtime
+ :body (string-trim
+ (elel-import--strip-signature
+ (string-trim
+ (elel-import--decode-body body headers))))))))))))
+
+;;;###autoload
+(defun elel-import-drafts (src out &optional dry-run)
+ "Import the notes kept in the Gnus draft directory SRC into OUT.
+Only drafts whose Gcc is a nnmaildir+notes:GROUP group are notes;
+each GROUP becomes the container file GROUP.org, and a tag. With a
+nil OUT, or a non-nil DRY-RUN, only report."
+ (interactive
+ (list (read-directory-name "Draft directory: " "~/News/drafts/drafts/")
+ (read-directory-name "Write to: " elel-root)
+ current-prefix-arg))
+ (let* ((paths (sort (directory-files (expand-file-name src) t "\\`[0-9]+\\'")
+ (lambda (a b)
+ (< (string-to-number (file-name-nondirectory a))
+ (string-to-number (file-name-nondirectory b))))))
+ (parsed (mapcar #'elel-import--draft paths))
+ (notes (cl-remove-if (lambda (n) (plist-get n :skip)) parsed))
+ (ids (mapcar (lambda (n) (plist-get n :id)) notes))
+ (report (list (format "source : %s" (expand-file-name src))
+ (format "drafts : %d, notes: %d"
+ (length paths) (length notes))))
+ (groups nil)
+ (files nil)
+ (demoted 0))
+ (dolist (n parsed)
+ (cond
+ ((plist-get n :skip)
+ (push (concat " skipped " (plist-get n :skip)) report))))
+ (dolist (n notes)
+ (let ((cell (assoc (plist-get n :group) groups)))
+ (cond
+ (cell (setcdr cell (cons n (cdr cell))))
+ ((push (list (plist-get n :group) n) groups)))))
+ (dolist (g (reverse groups))
+ (let* ((group (car g))
+ (sorted (sort (cdr g)
+ (lambda (a b)
+ (time-less-p (plist-get a :time)
+ (plist-get b :time)))))
+ (texts nil))
+ (dolist (n sorted)
+ (let* ((demotion (elel-import--demote (plist-get n :body)))
+ (parent (plist-get n :parent))
+ (known (and parent (member parent ids))))
+ (setq demoted (+ demoted (cdr demotion)))
+ (cond
+ ((and parent (not known))
+ (push (format " note %s: parent %s is not among the notes"
+ (plist-get n :number) parent)
+ report)))
+ (push (elel-import--org-note
+ (plist-get n :title)
+ (list group)
+ (append
+ (list (cons "ID" (plist-get n :id))
+ (cons "DATE" (elel--now (plist-get n :time))))
+ (and known (list (cons "PARENT" parent)))
+ (list (cons "DRAFT_FILE" (plist-get n :number))))
+ (car demotion))
+ texts)))
+ (push (cons (concat group ".org")
+ (mapconcat #'identity (nreverse texts) "\n"))
+ files)
+ (push (format " %s.org: %d notes" group (length sorted)) report)))
+ (push (format "headings demoted : %d" demoted) report)
+ (push "DATE is the modification time of the draft file (drafts have no Date header)."
+ report)
+ (elel-import--finish
+ (list :files (nreverse files) :report (reverse report))
+ out dry-run)))
+
+;;; org-roam
+
+(defconst elel-import--file-link-regexp
+ "\\[\\[file:\\([^]\n]*\\)\\]\\(?:\\[\\([^]\n]*\\)\\]\\)?\\]"
+ "Matches an org file link. Group 1 is the path, group 2 the description.")
+
+(defun elel-import--keyword (text name)
+ "The value of the keyword #+NAME: in TEXT, or nil."
+ (let ((case-fold-search t))
+ (and (string-match
+ (concat "^#\\+" name ":[ \t]*\\(.*?\\)[ \t]*$") text)
+ (not (string-empty-p (match-string 1 text)))
+ (match-string 1 text))))
+
+(defun elel-import--stamp (path)
+ "A compact time stamp such as 20200514T232108 for the roam file PATH."
+ (let ((name (file-name-nondirectory path)))
+ (cond
+ ((or (string-match
+ (concat "\\`\\([0-9]\\{4\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)"
+ "\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)\\([0-9]\\{2\\}\\)-")
+ name)
+ (string-match
+ (concat "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)"
+ "--\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)-")
+ name))
+ (format "%s%s%sT%s%s%s"
+ (match-string 1 name) (match-string 2 name)
+ (match-string 3 name) (match-string 4 name)
+ (match-string 5 name) (match-string 6 name)))
+ ((string-match
+ "\\`\\([0-9]\\{4\\}\\)-\\([0-9]\\{2\\}\\)-\\([0-9]\\{2\\}\\)\\.org\\'"
+ name)
+ (format "%s%s%sT000000"
+ (match-string 1 name) (match-string 2 name)
+ (match-string 3 name)))
+ ((format-time-string
+ "%Y%m%dT%H%M%S"
+ (file-attribute-modification-time (file-attributes path)))))))
+
+(defun elel-import--stamp-time (stamp)
+ "The time of STAMP, a string such as 20200514T232108, in local time."
+ (let ((n (lambda (a b) (string-to-number (substring stamp a b)))))
+ (encode-time (list (funcall n 13 15) (funcall n 11 13) (funcall n 9 11)
+ (funcall n 6 8) (funcall n 4 6) (funcall n 0 4)
+ nil -1 nil))))
+
+(defun elel-import--cover (items min)
+ "Choose one tag for each of ITEMS, so that few tags are used.
+ITEMS is a list of (KEY . TAGS). Return a hash table from KEY to the
+chosen tag, or to nothing when no good tag exists. Tags are picked
+greedily, each time the one that covers most of the notes left, until
+no tag covers MIN notes; the notes left are not in the table."
+ (let ((left (cl-remove-if-not #'cdr items))
+ (chosen (make-hash-table :test 'equal))
+ (counts (make-hash-table :test 'equal))
+ (continue t))
+ (while (and left continue)
+ (clrhash counts)
+ (dolist (item left)
+ (dolist (tag (cdr item))
+ (puthash tag (1+ (gethash tag counts 0)) counts)))
+ (let (best (best-count 0))
+ (maphash (lambda (tag count)
+ (cond
+ ((or (> count best-count)
+ (and (= count best-count) best (string< tag best)))
+ (setq best tag best-count count))))
+ counts)
+ (cond
+ ((or (null best) (< best-count min)) (setq continue nil))
+ (t
+ (dolist (item left)
+ (cond ((member best (cdr item)) (puthash (car item) best chosen))))
+ (setq left (cl-remove-if (lambda (item) (member best (cdr item)))
+ left))))))
+ chosen))
+
+(defun elel-import--target (rel _tags &optional chosen)
+ "The container file for the roam file REL, whose tags are TAGS.
+A journal goes to journal.org; otherwise the tag CHOSEN names the
+file; otherwise the directory does; otherwise the file is roam.org."
+ (let ((dir (file-name-directory rel)))
+ (cond
+ ((string-prefix-p "journal/" rel) "journal.org")
+ (chosen (concat chosen ".org"))
+ (dir (concat (replace-regexp-in-string
+ "[ \t]+" "-" (directory-file-name dir))
+ ".org"))
+ ("roam.org"))))
+
+(defun elel-import--tags (text)
+ "The #+roam_tags of TEXT, made into valid org tags."
+ (let ((raw (elel-import--keyword text "roam_tags"))
+ (tags nil))
+ (dolist (w (and raw (or (ignore-errors (split-string-and-unquote raw))
+ (split-string raw))))
+ (let ((tag (string-trim
+ (replace-regexp-in-string "[^[:alnum:]_@#%]" "_" w)
+ "_+" "_+")))
+ (cond
+ ((and (not (string-empty-p tag)) (not (member tag tags)))
+ (push tag tags)))))
+ (nreverse tags)))
+
+(defcustom elel-import-roam-min-group 2
+ "The fewest notes that a file made from a tag may hold.
+A note whose tags would make smaller files goes to roam.org instead."
+ :type 'integer
+ :group 'elel)
+
+;;;###autoload
+(defun elel-import-roam (src out &optional dry-run)
+ "Import the org-roam directory SRC into the directory OUT.
+Each roam file becomes one note. The notes are collected into
+container files, one per first tag (see `elel-import--target'), sorted
+by date. With a nil OUT, or a non-nil DRY-RUN, only report."
+ (interactive
+ (list (read-directory-name "Roam directory: " "~/org/roam/")
+ (read-directory-name "Write to: " (expand-file-name "roam" elel-root))
+ current-prefix-arg))
+ (let* ((src (file-name-as-directory (expand-file-name src)))
+ (paths (cl-remove-if
+ (lambda (p)
+ (string-match-p "\\(\\`\\|/\\)\\."
+ (file-relative-name p src)))
+ (sort (directory-files-recursively src "\\.org\\'")
+ #'string<)))
+ (notes (make-hash-table :test 'equal))
+ (links 0)
+ (demoted 0)
+ (unresolved nil)
+ (drawers nil)
+ (cover nil)
+ (groups nil)
+ (group-report nil)
+ (files nil))
+ (dolist (path paths)
+ (let* ((rel (file-relative-name path src))
+ (text (elel-import--read path))
+ (stamp (elel-import--stamp path)))
+ (puthash path
+ (list :rel rel
+ :text text
+ :title (or (elel-import--keyword text "title")
+ (file-name-base path))
+ :stamp stamp
+ :id (concat stamp "-"
+ (substring (md5 rel nil nil 'utf-8) 0 4))
+ :tags (elel-import--tags text))
+ notes)))
+ (setq cover
+ (elel-import--cover
+ (cl-loop for path in paths
+ for n = (gethash path notes)
+ unless (string-prefix-p "journal/" (plist-get n :rel))
+ collect (cons (plist-get n :rel) (plist-get n :tags)))
+ elel-import-roam-min-group))
+ (dolist (path paths)
+ (let* ((n (gethash path notes))
+ (rel (plist-get n :rel))
+ (body (let ((case-fold-search t))
+ (string-trim
+ (replace-regexp-in-string
+ "^#\\+\\(?:title\\|roam_tags\\):.*\n?" ""
+ (plist-get n :text))
+ "\n+" "\n+")))
+ (rewritten
+ (replace-regexp-in-string
+ elel-import--file-link-regexp
+ (lambda (m)
+ (string-match elel-import--file-link-regexp m)
+ (let* ((target (match-string 1 m))
+ (desc (match-string 2 m))
+ (p (car (split-string target "::" t)))
+ (found (and p
+ (gethash
+ (expand-file-name
+ p (file-name-directory path))
+ notes))))
+ (cond
+ (found
+ (setq links (1+ links))
+ (concat "[[note:" (plist-get found :id) "]["
+ (cond
+ ((or (null desc) (string-empty-p desc))
+ (plist-get found :title))
+ (desc))
+ "]]"))
+ (t
+ (cond
+ ((and p (string-suffix-p ".org" p))
+ (push (cons rel target) unresolved)))
+ m))))
+ body t t))
+ (demotion (elel-import--demote rewritten)))
+ (cond
+ ((string-match-p "\\`[ \t\n]*:PROPERTIES:" body)
+ (push rel drawers)))
+ (setq demoted (+ demoted (cdr demotion)))
+ (let* ((target (elel-import--target rel (plist-get n :tags)
+ (gethash rel cover)))
+ (cell (assoc target groups))
+ (item (cons
+ (plist-get n :stamp)
+ (elel-import--org-note
+ (plist-get n :title)
+ (plist-get n :tags)
+ (append
+ (list (cons "ID" (plist-get n :id))
+ (cons "DATE"
+ (elel--now (elel-import--stamp-time
+ (plist-get n :stamp)))))
+ (and (string-prefix-p "journal/" rel)
+ (list (cons "KIND" "journal")))
+ (list (cons "ROAM_FILE" rel)))
+ (car demotion)))))
+ (cond
+ (cell (setcdr cell (cons item (cdr cell))))
+ ((push (list target item) groups))))))
+ (dolist (g (sort groups (lambda (a b) (string< (car a) (car b)))))
+ (let ((sorted (sort (cdr g) (lambda (a b) (string< (car a) (car b))))))
+ (push (cons (car g)
+ (mapconcat #'cdr sorted "\n"))
+ files)
+ (push (format " %s: %d notes" (car g) (length sorted)) group-report)))
+ (elel-import--finish
+ (list
+ :files (nreverse files)
+ :report
+ (append
+ (list (format "source : %s" src)
+ (format "notes : %d" (length paths))
+ (format "with tags : %d"
+ (cl-count-if (lambda (p)
+ (plist-get (gethash p notes) :tags))
+ paths))
+ (format "links to notes rewritten : %d" links)
+ (format "headings demoted : %d" demoted))
+ (reverse group-report)
+ (mapcar (lambda (u)
+ (format " unresolved .org link in %s: %s"
+ (car u) (cdr u)))
+ (reverse unresolved))
+ (mapcar (lambda (rel)
+ (format " warning: %s starts with a property drawer" rel))
+ (reverse drawers))))
+ out dry-run)))
+
+(provide 'elel-import)
+;;; elel-import.el ends here