From 75dd524fadc022d9e6845f41f19203c3d5a951cd Mon Sep 17 00:00:00 2001 From: JSDurand Date: Wed, 21 Oct 2026 15:28:20 +0800 Subject: add VPS snapshot pipeline, run.sh, and data-repair fixes - fetch_tw_daily.sh: --snapshot mode (one file per trading day, merged by symbol), retries, validation of numeric fields - cron_daily.sh (VPS): lock, pruning, e-mail alerts on failure - sync_from_vps.sh (Mac): rsync snapshots and merge into per-stock CSVs - run.sh: pick data files for a symbol, build, analyse, open the page - lib.sh: LC_ALL=C; fetch.sh: small fixes - data.rs: repair rows whose open/close fall outside high/low - template: horizontal hover lines in the RSI and KD panels Co-Authored-By: Claude Sonnet 5.5 --- scripts/fetch_tw_daily.sh | 62 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 16 deletions(-) (limited to 'scripts/fetch_tw_daily.sh') diff --git a/scripts/fetch_tw_daily.sh b/scripts/fetch_tw_daily.sh index 3055f01..68792cd 100755 --- a/scripts/fetch_tw_daily.sh +++ b/scripts/fetch_tw_daily.sh @@ -1,19 +1,25 @@ #!/usr/bin/env bash # Daily update of Taiwan stock CSVs from the exchanges' official open data (2 requests in total). # -# scripts/fetch_tw_daily.sh [-d outdir] (default outdir: data/tw) +# scripts/fetch_tw_daily.sh [-d outdir] merge the latest day into each CODE.TW.csv +# scripts/fetch_tw_daily.sh --snapshot DIR write DIR/YYYY-MM-DD.csv (all stocks, one day) # -# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security and merges the row -# into each existing CODE.TW.csv / CODE.TWO.csv in outdir (create those first with fetch.sh). -# Only the latest day is available from these endpoints: run it at least once per trading day, and -# use fetch.sh (which repairs the last ~10 days from Yahoo) to fill any days you missed. +# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security (2 requests). +# Merge mode (default outdir: data/tw) adds the rows to each existing CODE.TW.csv / CODE.TWO.csv +# (create those first with fetch.sh). Snapshot mode needs no per-stock files: it writes one +# "Symbol,Date,Open,High,Low,Close,Volume" file per trading day, which the VPS publishes for +# scripts/sync_from_vps.sh to apply. Only the latest day is available from these endpoints. # Prices are raw, volume is in shares. set -u source "$(dirname "$0")/lib.sh" -outdir=data/tw -while getopts "d:" o; do case $o in d) outdir=$OPTARG;; *) sed -n '2,11p' "$0"; exit 2;; esac; done +outdir=data/tw; snapshot= +args=(); for a in "$@"; do [ "$a" = "--snapshot" ] && a=-s; args+=("$a"); done # getopts has no long options +set -- ${args[@]+"${args[@]}"} +while getopts "d:s:" o; do + case $o in d) outdir=$OPTARG;; s) snapshot=$OPTARG;; *) sed -n '2,12p' "$0"; exit 2;; esac +done for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done -[ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; } +[ -n "$snapshot" ] || [ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; } # Private scratch folder (always a fresh mktemp directory); removed again when the script ends. work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; } @@ -21,32 +27,56 @@ trap 'rm -rf "$work"' EXIT get_json() { # URL OUTFILE; retries until the response parses as a JSON array local try - for try in 1 2 3 4 5; do - curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1 && return 0 - echo " $1: bad response (try $try), retrying" >&2; sleep $((try * 3)) + # The exchanges sometimes cut a large response short (curl exit 56) or return an empty array + # before the day is published; wait longer between tries (2, 4, ... 30 s). + for try in 1 2 3 4 5 6 7 8; do + rc=0; curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" || rc=$? + if [ $rc -eq 0 ] && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1; then return 0; fi + echo " $1: bad response (try $try, curl exit $rc), retrying" >&2; sleep $(( (try * 2 > 30) ? 30 : try * 2 )) done echo "failed to download $1" >&2; return 1 } # The jq programs below turn ROC dates (1151008 -> 2026-10-08) into ISO dates and keep only rows with -# positive numeric OHLC, emitting "SYMBOL,Date,O,H,L,C,V". +# positive numeric OHLC and volume (a missing value becomes null, never a dropped field), emitting "SYMBOL,Date,O,H,L,C,V". rows="$work/rows.csv"; : > "$rows" get_json "https://openapi.twse.com.tw/v1/exchangeReport/STOCK_DAY_ALL" "$work/twse.json" || exit 1 get_json "https://www.tpex.org.tw/openapi/v1/tpex_mainboard_daily_close_quotes" "$work/tpex.json" || exit 1 jq -r ' - def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end; + def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end; .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d | [(.Code + ".TW"), $d, (.OpeningPrice|num), (.HighestPrice|num), (.LowestPrice|num), (.ClosingPrice|num), (.TradeVolume|num)] - | select(all(.[2:6][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows" + | select(all(.[2:7][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows" jq -r ' - def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end; + def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end; .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d | [(.SecuritiesCompanyCode + ".TWO"), $d, (.Open|num), (.High|num), (.Low|num), (.Close|num), (.TradingShares|num)] - | select(all(.[2:6][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows" + | select(all(.[2:7][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows" echo "TWSE date: $(jq -r '.[0].Date' "$work/twse.json") TPEx date: $(jq -r '.[0].Date' "$work/tpex.json") (ROC calendar)" echo "usable rows: $(wc -l < "$rows" | tr -d ' ')" +[ -s "$rows" ] || { echo "no usable rows in either response" >&2; exit 1; } + +if [ -n "$snapshot" ]; then + mkdir -p "$snapshot" + # TWSE and TPEx publish at different times, so one date can arrive in two runs: merge by symbol + # into an existing snapshot instead of overwriting it. + for d in $(cut -d, -f2 "$rows" | sort -u); do + f="$snapshot/$d.csv" + if [ -s "$f" ]; then state="updated"; else state="new"; fi + awk -F, -v d="$d" ' + FNR == 1 && FILENAME != ARGV[ARGC-1] { next } # header of the existing snapshot + FILENAME == ARGV[ARGC-1] { if ($2 == d) r[$1] = $0; next } # new rows win + { if (!($1 in r)) r[$1] = $0 } + END { for (k in r) print r[k] }' "$f" "$rows" 2>/dev/null | sort > "$work/snap.body" + if [ ! -s "$f" ]; then awk -F, -v d="$d" '$2 == d' "$rows" | sort > "$work/snap.body"; fi + { echo "Symbol,Date,Open,High,Low,Close,Volume"; cat "$work/snap.body"; } > "$f.tmp" + if [ -s "$f" ] && cmp -s "$f" "$f.tmp"; then state="unchanged"; rm -f "$f.tmp"; else mv "$f.tmp" "$f"; fi + echo "$state: $f ($(wc -l < "$work/snap.body" | tr -d ' ') rows)" + done + exit 0 +fi updated=0; skipped=0 while IFS= read -r sym; do -- cgit v1.2.3-18-g5258