add VPS snapshot pipeline, run.sh, and data-repair fixesHEADmaster

- fetch_tw_daily.sh: --snapshot mode (one file per trading day, merged by
  symbol), retries, validation of numeric fields
- cron_daily.sh (VPS): lock, pruning, e-mail alerts on failure
- sync_from_vps.sh (Mac): rsync snapshots and merge into per-stock CSVs
- run.sh: pick data files for a symbol, build, analyse, open the page
- lib.sh: LC_ALL=C; fetch.sh: small fixes
- data.rs: repair rows whose open/close fall outside high/low
- template: horizontal hover lines in the RSI and KD panels

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

1 files changed, 46 insertions, 16 deletions

diff --git a/scripts/fetch_tw_daily.sh b/scripts/fetch_tw_daily.sh
index 3055f01..68792cd 100755
--- a/scripts/fetch_tw_daily.sh
+++ b/scripts/fetch_tw_daily.sh
@@ -1,19 +1,25 @@
 #!/usr/bin/env bash
 # Daily update of Taiwan stock CSVs from the exchanges' official open data (2 requests in total).
 #
-#   scripts/fetch_tw_daily.sh [-d outdir]        (default outdir: data/tw)
+#   scripts/fetch_tw_daily.sh [-d outdir]              merge the latest day into each CODE.TW.csv
+#   scripts/fetch_tw_daily.sh --snapshot DIR          write DIR/YYYY-MM-DD.csv (all stocks, one day)
 #
-# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security and merges the row
-# into each existing CODE.TW.csv / CODE.TWO.csv in outdir (create those first with fetch.sh).
-# Only the latest day is available from these endpoints: run it at least once per trading day, and
-# use fetch.sh (which repairs the last ~10 days from Yahoo) to fill any days you missed.
+# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security (2 requests).
+# Merge mode (default outdir: data/tw) adds the rows to each existing CODE.TW.csv / CODE.TWO.csv
+# (create those first with fetch.sh). Snapshot mode needs no per-stock files: it writes one
+# "Symbol,Date,Open,High,Low,Close,Volume" file per trading day, which the VPS publishes for
+# scripts/sync_from_vps.sh to apply. Only the latest day is available from these endpoints.
 # Prices are raw, volume is in shares.
 set -u
 source "$(dirname "$0")/lib.sh"
-outdir=data/tw
-while getopts "d:" o; do case $o in d) outdir=$OPTARG;; *) sed -n '2,11p' "$0"; exit 2;; esac; done
+outdir=data/tw; snapshot=
+args=(); for a in "$@"; do [ "$a" = "--snapshot" ] && a=-s; args+=("$a"); done   # getopts has no long options
+set -- ${args[@]+"${args[@]}"}
+while getopts "d:s:" o; do
+  case $o in d) outdir=$OPTARG;; s) snapshot=$OPTARG;; *) sed -n '2,12p' "$0"; exit 2;; esac
+done
 for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
-[ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; }
+[ -n "$snapshot" ] || [ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; }
 
 # Private scratch folder (always a fresh mktemp directory); removed again when the script ends.
 work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; }
@@ -21,32 +27,56 @@ trap 'rm -rf "$work"' EXIT
 
 get_json() {  # URL OUTFILE; retries until the response parses as a JSON array
   local try
-  for try in 1 2 3 4 5; do
-    curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1 && return 0
-    echo "  $1: bad response (try $try), retrying" >&2; sleep $((try * 3))
+  # The exchanges sometimes cut a large response short (curl exit 56) or return an empty array
+  # before the day is published; wait longer between tries (2, 4, ... 30 s).
+  for try in 1 2 3 4 5 6 7 8; do
+    rc=0; curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" || rc=$?
+    if [ $rc -eq 0 ] && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1; then return 0; fi
+    echo "  $1: bad response (try $try, curl exit $rc), retrying" >&2; sleep $(( (try * 2 > 30) ? 30 : try * 2 ))
   done
   echo "failed to download $1" >&2; return 1
 }
 
 # The jq programs below turn ROC dates (1151008 -> 2026-10-08) into ISO dates and keep only rows with
-# positive numeric OHLC, emitting "SYMBOL,Date,O,H,L,C,V".
+# positive numeric OHLC and volume (a missing value becomes null, never a dropped field), emitting "SYMBOL,Date,O,H,L,C,V".
 rows="$work/rows.csv"; : > "$rows"
 get_json "https://openapi.twse.com.tw/v1/exchangeReport/STOCK_DAY_ALL" "$work/twse.json" || exit 1
 get_json "https://www.tpex.org.tw/openapi/v1/tpex_mainboard_daily_close_quotes" "$work/tpex.json" || exit 1
 
 jq -r '
-  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
+  def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end;
   .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
   | [(.Code + ".TW"), $d, (.OpeningPrice|num), (.HighestPrice|num), (.LowestPrice|num), (.ClosingPrice|num), (.TradeVolume|num)]
-  | select(all(.[2:6][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows"
+  | select(all(.[2:7][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows"
 jq -r '
-  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
+  def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end;
   .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
   | [(.SecuritiesCompanyCode + ".TWO"), $d, (.Open|num), (.High|num), (.Low|num), (.Close|num), (.TradingShares|num)]
-  | select(all(.[2:6][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows"
+  | select(all(.[2:7][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows"
 
 echo "TWSE date: $(jq -r '.[0].Date' "$work/twse.json")  TPEx date: $(jq -r '.[0].Date' "$work/tpex.json") (ROC calendar)"
 echo "usable rows: $(wc -l < "$rows" | tr -d ' ')"
+[ -s "$rows" ] || { echo "no usable rows in either response" >&2; exit 1; }
+
+if [ -n "$snapshot" ]; then
+  mkdir -p "$snapshot"
+  # TWSE and TPEx publish at different times, so one date can arrive in two runs: merge by symbol
+  # into an existing snapshot instead of overwriting it.
+  for d in $(cut -d, -f2 "$rows" | sort -u); do
+    f="$snapshot/$d.csv"
+    if [ -s "$f" ]; then state="updated"; else state="new"; fi
+    awk -F, -v d="$d" '
+      FNR == 1 && FILENAME != ARGV[ARGC-1] { next }              # header of the existing snapshot
+      FILENAME == ARGV[ARGC-1] { if ($2 == d) r[$1] = $0; next }  # new rows win
+      { if (!($1 in r)) r[$1] = $0 }
+      END { for (k in r) print r[k] }' "$f" "$rows" 2>/dev/null | sort > "$work/snap.body"
+    if [ ! -s "$f" ]; then awk -F, -v d="$d" '$2 == d' "$rows" | sort > "$work/snap.body"; fi
+    { echo "Symbol,Date,Open,High,Low,Close,Volume"; cat "$work/snap.body"; } > "$f.tmp"
+    if [ -s "$f" ] && cmp -s "$f" "$f.tmp"; then state="unchanged"; rm -f "$f.tmp"; else mv "$f.tmp" "$f"; fi
+    echo "$state: $f ($(wc -l < "$work/snap.body" | tr -d ' ') rows)"
+  done
+  exit 0
+fi
 
 updated=0; skipped=0
 while IFS= read -r sym; do