added daily fetchers

* scripts/fetch_tw_daily.sh
* scripts/lib.sh
* scripts/symbols.sh: daily fetchers quickly fetch data for today.

3 files changed, 108 insertions, 0 deletions

diff --git a/scripts/fetch_tw_daily.sh b/scripts/fetch_tw_daily.sh
new file mode 100755
index 0000000..3055f01
--- /dev/null
+++ b/scripts/fetch_tw_daily.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Daily update of Taiwan stock CSVs from the exchanges' official open data (2 requests in total).
+#
+#   scripts/fetch_tw_daily.sh [-d outdir]        (default outdir: data/tw)
+#
+# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security and merges the row
+# into each existing CODE.TW.csv / CODE.TWO.csv in outdir (create those first with fetch.sh).
+# Only the latest day is available from these endpoints: run it at least once per trading day, and
+# use fetch.sh (which repairs the last ~10 days from Yahoo) to fill any days you missed.
+# Prices are raw, volume is in shares.
+set -u
+source "$(dirname "$0")/lib.sh"
+outdir=data/tw
+while getopts "d:" o; do case $o in d) outdir=$OPTARG;; *) sed -n '2,11p' "$0"; exit 2;; esac; done
+for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
+[ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; }
+
+# Private scratch folder (always a fresh mktemp directory); removed again when the script ends.
+work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; }
+trap 'rm -rf "$work"' EXIT
+
+get_json() {  # URL OUTFILE; retries until the response parses as a JSON array
+  local try
+  for try in 1 2 3 4 5; do
+    curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1 && return 0
+    echo "  $1: bad response (try $try), retrying" >&2; sleep $((try * 3))
+  done
+  echo "failed to download $1" >&2; return 1
+}
+
+# The jq programs below turn ROC dates (1151008 -> 2026-10-08) into ISO dates and keep only rows with
+# positive numeric OHLC, emitting "SYMBOL,Date,O,H,L,C,V".
+rows="$work/rows.csv"; : > "$rows"
+get_json "https://openapi.twse.com.tw/v1/exchangeReport/STOCK_DAY_ALL" "$work/twse.json" || exit 1
+get_json "https://www.tpex.org.tw/openapi/v1/tpex_mainboard_daily_close_quotes" "$work/tpex.json" || exit 1
+
+jq -r '
+  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
+  .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
+  | [(.Code + ".TW"), $d, (.OpeningPrice|num), (.HighestPrice|num), (.LowestPrice|num), (.ClosingPrice|num), (.TradeVolume|num)]
+  | select(all(.[2:6][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows"
+jq -r '
+  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
+  .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
+  | [(.SecuritiesCompanyCode + ".TWO"), $d, (.Open|num), (.High|num), (.Low|num), (.Close|num), (.TradingShares|num)]
+  | select(all(.[2:6][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows"
+
+echo "TWSE date: $(jq -r '.[0].Date' "$work/twse.json")  TPEx date: $(jq -r '.[0].Date' "$work/tpex.json") (ROC calendar)"
+echo "usable rows: $(wc -l < "$rows" | tr -d ' ')"
+
+updated=0; skipped=0
+while IFS= read -r sym; do
+  file="$outdir/$sym.csv"
+  [ -s "$file" ] || { skipped=$((skipped + 1)); continue; }
+  new="$work/new.csv"; grep "^$sym," "$rows" | cut -d, -f2- > "$new"
+  [ -s "$new" ] || continue
+  merge_into "$file" "$new" && updated=$((updated + 1))
+done < <(cut -d, -f1 "$rows" | sort -u)
+echo "updated $updated files ($skipped securities without a CSV in $outdir ignored)"
diff --git a/scripts/lib.sh b/scripts/lib.sh
new file mode 100644
index 0000000..3a28552
--- /dev/null
+++ b/scripts/lib.sh
@@ -0,0 +1,18 @@
+# Shared helpers for fetch.sh and fetch_tw_daily.sh (source this file).
+header="Date,Open,High,Low,Close,Volume"
+
+# merge_into FILE NEWROWS: merge by date (new wins), sorted, warn on >1% close mismatch.
+merge_into() {
+  local file=$1 new=$2 tmp; tmp=$(mktemp)
+  { echo "$header"
+    awk -F, -v sym="$(basename "$file" .csv)" '
+      FNR == 1 { fileno++ }
+      $1 !~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}$/ { next }
+      fileno == 1 { r[$1] = $0; oldclose[$1] = $5; next }
+      { if (($1 in oldclose) && oldclose[$1] > 0) {
+          d = ($5 - oldclose[$1]) / oldclose[$1]; if (d < 0) d = -d
+          if (d > 0.01) printf "  warning: %s %s close %s -> %s (>1%% change; split or capital reduction? re-download from scratch)\n", sym, $1, oldclose[$1], $5 > "/dev/stderr" }
+        r[$1] = $0 }
+      END { for (d in r) print r[d] }' "$file" "$new" | sort
+  } > "$tmp" && mv "$tmp" "$file"
+}
diff --git a/scripts/symbols.sh b/scripts/symbols.sh
new file mode 100755
index 0000000..5d6ac3f
--- /dev/null
+++ b/scripts/symbols.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Build a list of Yahoo symbols for all ordinary Taiwan stocks (4-digit codes).
+#
+#   scripts/symbols.sh [-o symbols_tw.txt]
+#
+# TWSE-listed companies -> CODE.TW, TPEx (OTC) companies -> CODE.TWO, from the exchanges' open
+# company lists. ETFs, warrants and other products are not in these lists.
+set -u
+out=symbols_tw.txt
+while getopts "o:" o; do case $o in o) out=$OPTARG;; *) sed -n '2,8p' "$0"; exit 2;; esac; done
+for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
+
+get_codes() {  # URL JQ-FIELD SUFFIX; retries until the response parses as JSON
+  local url=$1 field=$2 suffix=$3 try body
+  for try in 1 2 3 4 5; do
+    body=$(curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$url")
+    if codes=$(printf '%s' "$body" | jq -r --arg f "$field" '.[][$f]' 2>/dev/null) && [ -n "$codes" ]; then
+      printf '%s\n' "$codes" | grep -E '^[0-9]{4}$' | sort -u | sed "s/\$/.$suffix/"
+      return 0
+    fi
+    echo "  $url: bad response (try $try), retrying" >&2; sleep $((try * 3))
+  done
+  echo "failed to download $url" >&2; return 1
+}
+
+tmp=$(mktemp)
+{ get_codes "https://openapi.twse.com.tw/v1/opendata/t187ap03_L" "公司代號" TW &&
+  get_codes "https://www.tpex.org.tw/openapi/v1/mopsfin_t187ap03_O" "SecuritiesCompanyCode" TWO; } > "$tmp" \
+  || { rm -f "$tmp"; exit 1; }
+mv "$tmp" "$out"
+echo "wrote $(wc -l < "$out" | tr -d ' ') symbols to $out ($(grep -c '\.TW$' "$out") TWSE, $(grep -c '\.TWO$' "$out") TPEx)"