add VPS snapshot pipeline, run.sh, and data-repair fixesHEADmaster

- fetch_tw_daily.sh: --snapshot mode (one file per trading day, merged by
  symbol), retries, validation of numeric fields
- cron_daily.sh (VPS): lock, pruning, e-mail alerts on failure
- sync_from_vps.sh (Mac): rsync snapshots and merge into per-stock CSVs
- run.sh: pick data files for a symbol, build, analyse, open the page
- lib.sh: LC_ALL=C; fetch.sh: small fixes
- data.rs: repair rows whose open/close fall outside high/low
- template: horizontal hover lines in the RSI and KD panels

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

1 files changed, 60 insertions, 0 deletions

diff --git a/scripts/sync_from_vps.sh b/scripts/sync_from_vps.sh
new file mode 100755
index 0000000..20ba9d1
--- /dev/null
+++ b/scripts/sync_from_vps.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Mac side: pull the VPS's daily snapshot files and merge them into the per-stock CSVs.
+#
+#   scripts/sync_from_vps.sh [--dry-run] [-r user@host:stock/updates/] [-d data/tw] [-u data/updates]
+#
+# The remote can also be given as $VPS_UPDATES. The VPS keeps only recent daily files
+# (Symbol,Date,Open,High,Low,Close,Volume). They are copied to the local cache (-u), which mirrors
+# the remote folder, and all cached snapshots are merged by date into the CSVs in -d. Merging is
+# idempotent (re-applying a day changes nothing), which also repairs a day that was published in
+# two parts. Stocks without a local CSV are ignored. Days you missed entirely (the VPS only keeps
+# a few weeks) can be filled from Yahoo with fetch.sh.
+set -u
+source "$(dirname "$0")/lib.sh"
+remote=${VPS_UPDATES:-}; outdir=data/tw; cache=data/updates; dry=0
+args=(); for a in "$@"; do [ "$a" = "--dry-run" ] && a=-n; args+=("$a"); done   # getopts has no long options
+set -- ${args[@]+"${args[@]}"}
+while getopts "r:d:u:n" o; do
+  case $o in r) remote=$OPTARG;; d) outdir=$OPTARG;; u) cache=$OPTARG;; n) dry=1;; *) sed -n '2,13p' "$0"; exit 2;; esac
+done
+[ -n "$remote" ] || { echo "no remote given: use -r user@host:stock/updates/ or set VPS_UPDATES" >&2; exit 2; }
+[ -d "$outdir" ] || { echo "$outdir does not exist" >&2; exit 1; }
+for cmd in rsync awk sort; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
+mkdir -p "$cache"
+
+latest_local() {
+  for f in "$outdir"/*.csv; do tail -n 1 "$f" | cut -d, -f1; done | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' | sort | tail -n 1
+}
+echo "latest date in $outdir before sync: $(latest_local)"
+
+remote=${remote%/}/
+if [ $dry -eq 1 ]; then
+  echo "--- files that would be downloaded:"
+  rsync -avn --delete "$remote" "$cache/"
+  exit 0
+fi
+rsync -av --partial --delete "$remote" "$cache/" || { echo "rsync failed" >&2; exit 1; }
+
+shopt -s nullglob
+snaps=("$cache"/????-??-??.csv)
+[ ${#snaps[@]} -gt 0 ] || { echo "no snapshot files in $cache"; exit 0; }
+
+work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; }
+trap 'rm -rf "$work"' EXIT
+
+# All cached rows sorted by symbol then date, restricted to stocks we have, split into one file each.
+for f in "${snaps[@]}"; do tail -n +2 "$f"; done | sort -t, -k1,1 -k2,2 > "$work/all.csv"
+ls "$outdir" | sed -n 's/\.csv$//p' > "$work/have.txt"
+mkdir "$work/split"
+awk -F, -v dir="$work/split" '
+  FNR == NR { want[$1]; next }
+  !($1 in want) { next }
+  $1 != prev { if (out != "") close(out); prev = $1; out = dir "/" $1 ".csv" }
+  { print $2 "," $3 "," $4 "," $5 "," $6 "," $7 > out }' "$work/have.txt" "$work/all.csv"
+
+n=0
+for f in "$work"/split/*.csv; do
+  merge_into "$outdir/$(basename "$f")" "$f" && n=$((n + 1))
+done
+echo "merged ${#snaps[@]} snapshot days into $n stock files"
+echo "latest date in $outdir after sync: $(latest_local)"