diff options
| -rwxr-xr-x | scripts/cron_daily.sh | 62 | ||||
| -rwxr-xr-x | scripts/fetch.sh | 3 | ||||
| -rwxr-xr-x | scripts/fetch_tw_daily.sh | 62 | ||||
| -rw-r--r-- | scripts/lib.sh | 1 | ||||
| -rwxr-xr-x | scripts/run.sh | 53 | ||||
| -rwxr-xr-x | scripts/sync_from_vps.sh | 60 | ||||
| -rw-r--r-- | src/data.rs | 49 | ||||
| -rw-r--r-- | web/template.html | 8 |
8 files changed, 278 insertions, 20 deletions
diff --git a/scripts/cron_daily.sh b/scripts/cron_daily.sh new file mode 100755 index 0000000..9fb0b44 --- /dev/null +++ b/scripts/cron_daily.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# VPS wrapper for cron: download the latest trading day into a daily snapshot file, keep only +# recent snapshots, and send an e-mail when something goes wrong. +# +# crontab (UTC; Taipei = UTC+8): +# ALERT_TO=you@example.com +# 30 6 * * 1-5 $HOME/stock/scripts/cron_daily.sh +# 0 11 * * 1-5 $HOME/stock/scripts/cron_daily.sh +# +# Environment: STOCK_DIR (default ~/stock), KEEP_DAYS (default 30), ALERT_TO (e-mail address; +# needs msmtp or mail). Snapshots go to $STOCK_DIR/updates/YYYY-MM-DD.csv, the log to +# $STOCK_DIR/logs/daily.log. Failures: download failed after all retries, or no new data for 10 days. +set -u +root=${STOCK_DIR:-$HOME/stock} +keep_days=${KEEP_DAYS:-30} +updates="$root/updates"; logs="$root/logs"; log="$logs/daily.log" +mkdir -p "$updates" "$logs" + +notify() { # SUBJECT BODY + if [ -z "${ALERT_TO:-}" ]; then echo "ALERT_TO is not set; cannot send e-mail: $1" >> "$log"; return; fi + if command -v msmtp >/dev/null; then + printf 'To: %s\nSubject: %s\n\n%s\n' "$ALERT_TO" "$1" "$2" | msmtp "$ALERT_TO" || echo "msmtp failed" >> "$log" + elif command -v mail >/dev/null; then + printf '%s\n' "$2" | mail -s "$1" "$ALERT_TO" || echo "mail failed" >> "$log" + else + echo "no msmtp/mail command; cannot send e-mail: $1" >> "$log" + fi +} + +# One run at a time (mkdir is atomic); a lock older than 3 hours is treated as stale. +lock="$root/.daily.lock" +if ! mkdir "$lock" 2>/dev/null; then + if [ -n "$(find "$lock" -maxdepth 0 -mmin +180 2>/dev/null)" ]; then + rmdir "$lock" 2>/dev/null; mkdir "$lock" 2>/dev/null || exit 0 + else + echo "$(date -u +%FT%TZ) another run is in progress, skipping" >> "$log"; exit 0 + fi +fi +out=$(mktemp) +trap 'rmdir "$lock" 2>/dev/null; rm -f "$out"' EXIT + +"$(dirname "$0")/fetch_tw_daily.sh" --snapshot "$updates" > "$out" 2>&1 +rc=$? +{ echo "== $(date -u +%FT%TZ) exit $rc"; cat "$out"; } >> "$log" +find "$updates" -name '????-??-??.csv' -mtime +"$keep_days" -delete +tail -n 3000 "$log" > "$log.tmp" && mv "$log.tmp" "$log" # keep the log small + +if [ $rc -ne 0 ]; then + notify "stock daily update FAILED on $(hostname) ($(date -u +%F))" "$(tail -n 40 "$out")" + exit $rc +fi +if grep -qE '^(new|updated):' "$out"; then + echo " -> new data written" >> "$log" +else + echo " -> no new trading day" >> "$log" + # A long silence (beyond the longest market holiday) means the exchange feed has probably changed. + if ! find "$updates" -name '????-??-??.csv' -mtime -10 2>/dev/null | grep -q .; then + notify "stock daily update: no new data for 10 days on $(hostname)" "No snapshot file has changed in $updates for 10 days. +Last log lines: +$(tail -n 20 "$log")" + fi +fi diff --git a/scripts/fetch.sh b/scripts/fetch.sh index 33b1da2..1ad05a3 100755 --- a/scripts/fetch.sh +++ b/scripts/fetch.sh @@ -17,6 +17,9 @@ # trade on one of those days (halted) is fetched again, which is harmless. Skipped stocks are # not backfilled when START is moved earlier: use -F for that. # A >1% close mismatch on overlapping days is reported (probable split / capital reduction). + + + set -u source "$(dirname "$0")/lib.sh" diff --git a/scripts/fetch_tw_daily.sh b/scripts/fetch_tw_daily.sh index 3055f01..68792cd 100755 --- a/scripts/fetch_tw_daily.sh +++ b/scripts/fetch_tw_daily.sh @@ -1,19 +1,25 @@ #!/usr/bin/env bash # Daily update of Taiwan stock CSVs from the exchanges' official open data (2 requests in total). # -# scripts/fetch_tw_daily.sh [-d outdir] (default outdir: data/tw) +# scripts/fetch_tw_daily.sh [-d outdir] merge the latest day into each CODE.TW.csv +# scripts/fetch_tw_daily.sh --snapshot DIR write DIR/YYYY-MM-DD.csv (all stocks, one day) # -# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security and merges the row -# into each existing CODE.TW.csv / CODE.TWO.csv in outdir (create those first with fetch.sh). -# Only the latest day is available from these endpoints: run it at least once per trading day, and -# use fetch.sh (which repairs the last ~10 days from Yahoo) to fill any days you missed. +# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security (2 requests). +# Merge mode (default outdir: data/tw) adds the rows to each existing CODE.TW.csv / CODE.TWO.csv +# (create those first with fetch.sh). Snapshot mode needs no per-stock files: it writes one +# "Symbol,Date,Open,High,Low,Close,Volume" file per trading day, which the VPS publishes for +# scripts/sync_from_vps.sh to apply. Only the latest day is available from these endpoints. # Prices are raw, volume is in shares. set -u source "$(dirname "$0")/lib.sh" -outdir=data/tw -while getopts "d:" o; do case $o in d) outdir=$OPTARG;; *) sed -n '2,11p' "$0"; exit 2;; esac; done +outdir=data/tw; snapshot= +args=(); for a in "$@"; do [ "$a" = "--snapshot" ] && a=-s; args+=("$a"); done # getopts has no long options +set -- ${args[@]+"${args[@]}"} +while getopts "d:s:" o; do + case $o in d) outdir=$OPTARG;; s) snapshot=$OPTARG;; *) sed -n '2,12p' "$0"; exit 2;; esac +done for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done -[ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; } +[ -n "$snapshot" ] || [ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; } # Private scratch folder (always a fresh mktemp directory); removed again when the script ends. work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; } @@ -21,32 +27,56 @@ trap 'rm -rf "$work"' EXIT get_json() { # URL OUTFILE; retries until the response parses as a JSON array local try - for try in 1 2 3 4 5; do - curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1 && return 0 - echo " $1: bad response (try $try), retrying" >&2; sleep $((try * 3)) + # The exchanges sometimes cut a large response short (curl exit 56) or return an empty array + # before the day is published; wait longer between tries (2, 4, ... 30 s). + for try in 1 2 3 4 5 6 7 8; do + rc=0; curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" || rc=$? + if [ $rc -eq 0 ] && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1; then return 0; fi + echo " $1: bad response (try $try, curl exit $rc), retrying" >&2; sleep $(( (try * 2 > 30) ? 30 : try * 2 )) done echo "failed to download $1" >&2; return 1 } # The jq programs below turn ROC dates (1151008 -> 2026-10-08) into ISO dates and keep only rows with -# positive numeric OHLC, emitting "SYMBOL,Date,O,H,L,C,V". +# positive numeric OHLC and volume (a missing value becomes null, never a dropped field), emitting "SYMBOL,Date,O,H,L,C,V". rows="$work/rows.csv"; : > "$rows" get_json "https://openapi.twse.com.tw/v1/exchangeReport/STOCK_DAY_ALL" "$work/twse.json" || exit 1 get_json "https://www.tpex.org.tw/openapi/v1/tpex_mainboard_daily_close_quotes" "$work/tpex.json" || exit 1 jq -r ' - def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end; + def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end; .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d | [(.Code + ".TW"), $d, (.OpeningPrice|num), (.HighestPrice|num), (.LowestPrice|num), (.ClosingPrice|num), (.TradeVolume|num)] - | select(all(.[2:6][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows" + | select(all(.[2:7][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows" jq -r ' - def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end; + def num: if type == "string" then (gsub(",";"") | if test("^[0-9]+(\\.[0-9]+)?$") then tonumber else null end) else null end | if . != null and . > 0 then . else null end; .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d | [(.SecuritiesCompanyCode + ".TWO"), $d, (.Open|num), (.High|num), (.Low|num), (.Close|num), (.TradingShares|num)] - | select(all(.[2:6][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows" + | select(all(.[2:7][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows" echo "TWSE date: $(jq -r '.[0].Date' "$work/twse.json") TPEx date: $(jq -r '.[0].Date' "$work/tpex.json") (ROC calendar)" echo "usable rows: $(wc -l < "$rows" | tr -d ' ')" +[ -s "$rows" ] || { echo "no usable rows in either response" >&2; exit 1; } + +if [ -n "$snapshot" ]; then + mkdir -p "$snapshot" + # TWSE and TPEx publish at different times, so one date can arrive in two runs: merge by symbol + # into an existing snapshot instead of overwriting it. + for d in $(cut -d, -f2 "$rows" | sort -u); do + f="$snapshot/$d.csv" + if [ -s "$f" ]; then state="updated"; else state="new"; fi + awk -F, -v d="$d" ' + FNR == 1 && FILENAME != ARGV[ARGC-1] { next } # header of the existing snapshot + FILENAME == ARGV[ARGC-1] { if ($2 == d) r[$1] = $0; next } # new rows win + { if (!($1 in r)) r[$1] = $0 } + END { for (k in r) print r[k] }' "$f" "$rows" 2>/dev/null | sort > "$work/snap.body" + if [ ! -s "$f" ]; then awk -F, -v d="$d" '$2 == d' "$rows" | sort > "$work/snap.body"; fi + { echo "Symbol,Date,Open,High,Low,Close,Volume"; cat "$work/snap.body"; } > "$f.tmp" + if [ -s "$f" ] && cmp -s "$f" "$f.tmp"; then state="unchanged"; rm -f "$f.tmp"; else mv "$f.tmp" "$f"; fi + echo "$state: $f ($(wc -l < "$work/snap.body" | tr -d ' ') rows)" + done + exit 0 +fi updated=0; skipped=0 while IFS= read -r sym; do diff --git a/scripts/lib.sh b/scripts/lib.sh index 3a28552..efe5556 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,4 +1,5 @@ # Shared helpers for fetch.sh and fetch_tw_daily.sh (source this file). +export LC_ALL=C # awk/sort numeric and ordering behaviour must not depend on the user locale (e.g. comma decimals) header="Date,Open,High,Low,Close,Volume" # merge_into FILE NEWROWS: merge by date (new wins), sorted, warn on >1% close mismatch. diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..463740b --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Input a number and the script picks the right data and runs the +# executable to produce the HTML web page, and then opens the page +# with the default web browser. +# +# scripts/run.sh SYMBOL +# +# The SYMBOL does not have to contain TW or TWO; the script finds the +# correct one. If both options exist, do both. +# +# Extra options are passed on to the analyser, e.g. +# scripts/run.sh 2330 --interval week +# +# Environment: DATA_DIR (default data/tw), HTML_DIR (default data/html), OPEN_CMD (default `open` +# on macOS, `xdg-open` elsewhere). The page is written to $HTML_DIR/CODE.TW.html (or .TWO.html). +set -u +cd "$(dirname "$0")/.." || exit 1 # project root, so it works from any directory +data=${DATA_DIR:-data/tw}; htmldir=${HTML_DIR:-data/html} + +[ $# -ge 1 ] || { sed -n '2,15p' "$0"; exit 2; } +sym=$1; shift + +# An explicit suffix (2330.TW / 6488.TWO) selects that market only; otherwise try both. +case $sym in + *.[Tt][Ww][Oo]) code=${sym%.*}; candidates=("$data/$code.TWO.csv");; + *.[Tt][Ww]) code=${sym%.*}; candidates=("$data/$code.TW.csv");; + *) code=$sym; candidates=("$data/$code.TW.csv" "$data/$code.TWO.csv");; +esac + +files=(); for f in "${candidates[@]}"; do [ -s "$f" ] && files+=("$f"); done +if [ ${#files[@]} -eq 0 ]; then + echo "no data for '$sym' in $data (looked for: ${candidates[*]})" >&2 + echo "download it first, e.g.: scripts/fetch.sh -d $data 2010-01-01 \$(date +%F) $code" >&2 + exit 1 +fi + +cargo build --release -q || { echo "build failed" >&2; exit 1; } +bin=target/release/stock +mkdir -p "$htmldir" + +pages=() +for f in "${files[@]}"; do + page="$htmldir/$(basename "$f" .csv).html" + echo "== $(basename "$f" .csv)" + "$bin" "$f" -o "$page" "$@" || { echo "analysis failed for $f" >&2; exit 1; } + pages+=("$page") +done + +if [ -n "${OPEN_CMD:-}" ]; then opener=$OPEN_CMD +elif command -v open >/dev/null; then opener=open +elif command -v xdg-open >/dev/null; then opener=xdg-open +else echo "no program to open pages with; open them yourself: ${pages[*]}"; exit 0; fi +for page in "${pages[@]}"; do $opener "$page"; done diff --git a/scripts/sync_from_vps.sh b/scripts/sync_from_vps.sh new file mode 100755 index 0000000..20ba9d1 --- /dev/null +++ b/scripts/sync_from_vps.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Mac side: pull the VPS's daily snapshot files and merge them into the per-stock CSVs. +# +# scripts/sync_from_vps.sh [--dry-run] [-r user@host:stock/updates/] [-d data/tw] [-u data/updates] +# +# The remote can also be given as $VPS_UPDATES. The VPS keeps only recent daily files +# (Symbol,Date,Open,High,Low,Close,Volume). They are copied to the local cache (-u), which mirrors +# the remote folder, and all cached snapshots are merged by date into the CSVs in -d. Merging is +# idempotent (re-applying a day changes nothing), which also repairs a day that was published in +# two parts. Stocks without a local CSV are ignored. Days you missed entirely (the VPS only keeps +# a few weeks) can be filled from Yahoo with fetch.sh. +set -u +source "$(dirname "$0")/lib.sh" +remote=${VPS_UPDATES:-}; outdir=data/tw; cache=data/updates; dry=0 +args=(); for a in "$@"; do [ "$a" = "--dry-run" ] && a=-n; args+=("$a"); done # getopts has no long options +set -- ${args[@]+"${args[@]}"} +while getopts "r:d:u:n" o; do + case $o in r) remote=$OPTARG;; d) outdir=$OPTARG;; u) cache=$OPTARG;; n) dry=1;; *) sed -n '2,13p' "$0"; exit 2;; esac +done +[ -n "$remote" ] || { echo "no remote given: use -r user@host:stock/updates/ or set VPS_UPDATES" >&2; exit 2; } +[ -d "$outdir" ] || { echo "$outdir does not exist" >&2; exit 1; } +for cmd in rsync awk sort; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done +mkdir -p "$cache" + +latest_local() { + for f in "$outdir"/*.csv; do tail -n 1 "$f" | cut -d, -f1; done | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' | sort | tail -n 1 +} +echo "latest date in $outdir before sync: $(latest_local)" + +remote=${remote%/}/ +if [ $dry -eq 1 ]; then + echo "--- files that would be downloaded:" + rsync -avn --delete "$remote" "$cache/" + exit 0 +fi +rsync -av --partial --delete "$remote" "$cache/" || { echo "rsync failed" >&2; exit 1; } + +shopt -s nullglob +snaps=("$cache"/????-??-??.csv) +[ ${#snaps[@]} -gt 0 ] || { echo "no snapshot files in $cache"; exit 0; } + +work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; } +trap 'rm -rf "$work"' EXIT + +# All cached rows sorted by symbol then date, restricted to stocks we have, split into one file each. +for f in "${snaps[@]}"; do tail -n +2 "$f"; done | sort -t, -k1,1 -k2,2 > "$work/all.csv" +ls "$outdir" | sed -n 's/\.csv$//p' > "$work/have.txt" +mkdir "$work/split" +awk -F, -v dir="$work/split" ' + FNR == NR { want[$1]; next } + !($1 in want) { next } + $1 != prev { if (out != "") close(out); prev = $1; out = dir "/" $1 ".csv" } + { print $2 "," $3 "," $4 "," $5 "," $6 "," $7 > out }' "$work/have.txt" "$work/all.csv" + +n=0 +for f in "$work"/split/*.csv; do + merge_into "$outdir/$(basename "$f")" "$f" && n=$((n + 1)) +done +echo "merged ${#snaps[@]} snapshot days into $n stock files" +echo "latest date in $outdir after sync: $(latest_local)" diff --git a/src/data.rs b/src/data.rs index c5fc18c..24a80ef 100644 --- a/src/data.rs +++ b/src/data.rs @@ -18,21 +18,35 @@ pub struct Candle { } /// Read `Date,Open,High,Low,Close,Volume` rows; dates must be `YYYY-MM-DD`. -/// Rows are sorted ascending by date; duplicate dates are an error. +/// Rows are sorted ascending by date; duplicate dates are an error. Rows whose Open/Close lie +/// outside High/Low are repaired (High/Low widened) rather than rejected. pub fn load_csv(path: &Path) -> Result<Vec<Candle>, String> { let mut reader = csv::Reader::from_path(path).map_err(|e| format!("{}: {e}", path.display()))?; let mut candles = Vec::new(); + let mut repaired = 0usize; for (i, row) in reader.deserialize::<Candle>().enumerate() { let line = i + 2; // header is line 1 - let c = row.map_err(|e| format!("{}: line {line}: {e}", path.display()))?; + let mut c = row.map_err(|e| format!("{}: line {line}: {e}", path.display()))?; if !valid_date(&c.date) { return Err(format!("{}: line {line}: bad date '{}', expected YYYY-MM-DD", path.display(), c.date)); } - if c.high < c.low || !(c.low..=c.high).contains(&c.close) { - return Err(format!("{}: line {line}: inconsistent High/Low/Close", path.display())); + let prices = [c.open, c.high, c.low, c.close]; + if prices.iter().any(|p| !p.is_finite() || *p <= 0.0) { + return Err(format!("{}: line {line}: prices must be positive numbers", path.display())); + } + // Yahoo history has stray rows (e.g. the close outside the day's range). Keep the day but + // widen High/Low so they include Open and Close; the count is reported below. + let (hi, lo) = (prices.iter().cloned().fold(f64::MIN, f64::max), prices.iter().cloned().fold(f64::MAX, f64::min)); + if hi != c.high || lo != c.low { + repaired += 1; + c.high = hi; + c.low = lo; } candles.push(c); } + if repaired > 0 { + eprintln!("note: {}: {repaired} rows had Open/Close outside High/Low; High/Low widened to include them", path.display()); + } candles.sort_by(|a, b| a.date.cmp(&b.date)); if let Some(w) = candles.windows(2).find(|w| w[0].date == w[1].date) { return Err(format!("{}: duplicate date {}", path.display(), w[0].date)); @@ -50,3 +64,30 @@ fn valid_date(s: &str) -> bool { && b[7] == b'-' && b.iter().enumerate().all(|(i, c)| i == 4 || i == 7 || c.is_ascii_digit()) } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn load(body: &str) -> Result<Vec<Candle>, String> { + let path = std::env::temp_dir().join(format!("stock-test-{}-{}.csv", std::process::id(), body.len())); + std::fs::File::create(&path).unwrap().write_all(body.as_bytes()).unwrap(); + let r = load_csv(&path); + let _ = std::fs::remove_file(&path); + r + } + + #[test] + fn widens_high_low_instead_of_rejecting() { + // close 67 below low 71.6, as in real Yahoo data + let c = load("Date,Open,High,Low,Close,Volume\n2016-07-12,71.9,72.4,71.6,67,100\n").unwrap(); + assert_eq!((c[0].high, c[0].low, c[0].close), (72.4, 67.0, 67.0)); + } + + #[test] + fn rejects_non_positive_prices_and_bad_dates() { + assert!(load("Date,Open,High,Low,Close,Volume\n2016-07-12,0,1,1,1,100\n").is_err()); + assert!(load("Date,Open,High,Low,Close,Volume\n12/07/2016,1,1,1,1,100\n").is_err()); + } +} diff --git a/web/template.html b/web/template.html index 45fa874..966a584 100644 --- a/web/template.html +++ b/web/template.html @@ -206,6 +206,14 @@ function render() { const c = D.candles[hover]; s += `<line class="guide" x1="${M.l}" x2="${W-M.r}" y1="${py(c.Close)}" y2="${py(c.Close)}"/>` + `<text x="${W-M.r+4}" y="${py(c.Close)+4}" style="fill:${css('--fg')};font-weight:600">${fmt(c.Close)}</text>`; + // same horizontal crosshair for the indicator panels, at the hovered bar's values + for (const [p, arr, color] of [[R, D.rsi, css('--accent')], [K, D.k, css('--line1')], [K, D.d, css('--line2')]]) { + const v = arr[hover]; + if (v == null) continue; + const y = ind(p, v); + s += `<line x1="${M.l}" x2="${W-M.r}" y1="${y}" y2="${y}" stroke="${color}" stroke-width="1" stroke-dasharray="4 3"/>` + + `<text x="${W-M.r+4}" y="${y+4}" style="fill:${color};font-weight:600">${fmt(v, 1)}</text>`; + } } svg.innerHTML = s; updateInfo(); |