From 75dd524fadc022d9e6845f41f19203c3d5a951cd Mon Sep 17 00:00:00 2001 From: JSDurand Date: Wed, 21 Oct 2026 15:28:20 +0800 Subject: add VPS snapshot pipeline, run.sh, and data-repair fixes - fetch_tw_daily.sh: --snapshot mode (one file per trading day, merged by symbol), retries, validation of numeric fields - cron_daily.sh (VPS): lock, pruning, e-mail alerts on failure - sync_from_vps.sh (Mac): rsync snapshots and merge into per-stock CSVs - run.sh: pick data files for a symbol, build, analyse, open the page - lib.sh: LC_ALL=C; fetch.sh: small fixes - data.rs: repair rows whose open/close fall outside high/low - template: horizontal hover lines in the RSI and KD panels Co-Authored-By: Claude Sonnet 5.5 --- scripts/sync_from_vps.sh | 60 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 scripts/sync_from_vps.sh (limited to 'scripts/sync_from_vps.sh') diff --git a/scripts/sync_from_vps.sh b/scripts/sync_from_vps.sh new file mode 100755 index 0000000..20ba9d1 --- /dev/null +++ b/scripts/sync_from_vps.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Mac side: pull the VPS's daily snapshot files and merge them into the per-stock CSVs. +# +# scripts/sync_from_vps.sh [--dry-run] [-r user@host:stock/updates/] [-d data/tw] [-u data/updates] +# +# The remote can also be given as $VPS_UPDATES. The VPS keeps only recent daily files +# (Symbol,Date,Open,High,Low,Close,Volume). They are copied to the local cache (-u), which mirrors +# the remote folder, and all cached snapshots are merged by date into the CSVs in -d. Merging is +# idempotent (re-applying a day changes nothing), which also repairs a day that was published in +# two parts. Stocks without a local CSV are ignored. Days you missed entirely (the VPS only keeps +# a few weeks) can be filled from Yahoo with fetch.sh. +set -u +source "$(dirname "$0")/lib.sh" +remote=${VPS_UPDATES:-}; outdir=data/tw; cache=data/updates; dry=0 +args=(); for a in "$@"; do [ "$a" = "--dry-run" ] && a=-n; args+=("$a"); done # getopts has no long options +set -- ${args[@]+"${args[@]}"} +while getopts "r:d:u:n" o; do + case $o in r) remote=$OPTARG;; d) outdir=$OPTARG;; u) cache=$OPTARG;; n) dry=1;; *) sed -n '2,13p' "$0"; exit 2;; esac +done +[ -n "$remote" ] || { echo "no remote given: use -r user@host:stock/updates/ or set VPS_UPDATES" >&2; exit 2; } +[ -d "$outdir" ] || { echo "$outdir does not exist" >&2; exit 1; } +for cmd in rsync awk sort; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done +mkdir -p "$cache" + +latest_local() { + for f in "$outdir"/*.csv; do tail -n 1 "$f" | cut -d, -f1; done | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' | sort | tail -n 1 +} +echo "latest date in $outdir before sync: $(latest_local)" + +remote=${remote%/}/ +if [ $dry -eq 1 ]; then + echo "--- files that would be downloaded:" + rsync -avn --delete "$remote" "$cache/" + exit 0 +fi +rsync -av --partial --delete "$remote" "$cache/" || { echo "rsync failed" >&2; exit 1; } + +shopt -s nullglob +snaps=("$cache"/????-??-??.csv) +[ ${#snaps[@]} -gt 0 ] || { echo "no snapshot files in $cache"; exit 0; } + +work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; } +trap 'rm -rf "$work"' EXIT + +# All cached rows sorted by symbol then date, restricted to stocks we have, split into one file each. +for f in "${snaps[@]}"; do tail -n +2 "$f"; done | sort -t, -k1,1 -k2,2 > "$work/all.csv" +ls "$outdir" | sed -n 's/\.csv$//p' > "$work/have.txt" +mkdir "$work/split" +awk -F, -v dir="$work/split" ' + FNR == NR { want[$1]; next } + !($1 in want) { next } + $1 != prev { if (out != "") close(out); prev = $1; out = dir "/" $1 ".csv" } + { print $2 "," $3 "," $4 "," $5 "," $6 "," $7 > out }' "$work/have.txt" "$work/all.csv" + +n=0 +for f in "$work"/split/*.csv; do + merge_into "$outdir/$(basename "$f")" "$f" && n=$((n + 1)) +done +echo "merged ${#snaps[@]} snapshot days into $n stock files" +echo "latest date in $outdir after sync: $(latest_local)" -- cgit v1.2.3-18-g5258