#!/usr/bin/env bash
# Daily update of Taiwan stock CSVs from the exchanges' official open data (2 requests in total).
#
#   scripts/fetch_tw_daily.sh [-d outdir]        (default outdir: data/tw)
#
# Downloads the latest trading day of every TWSE-listed and TPEx (OTC) security and merges the row
# into each existing CODE.TW.csv / CODE.TWO.csv in outdir (create those first with fetch.sh).
# Only the latest day is available from these endpoints: run it at least once per trading day, and
# use fetch.sh (which repairs the last ~10 days from Yahoo) to fill any days you missed.
# Prices are raw, volume is in shares.
set -u
source "$(dirname "$0")/lib.sh"
outdir=data/tw
while getopts "d:" o; do case $o in d) outdir=$OPTARG;; *) sed -n '2,11p' "$0"; exit 2;; esac; done
for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
[ -d "$outdir" ] || { echo "$outdir does not exist; run fetch.sh first" >&2; exit 1; }

# Private scratch folder (always a fresh mktemp directory); removed again when the script ends.
work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; }
trap 'rm -rf "$work"' EXIT

get_json() {  # URL OUTFILE; retries until the response parses as a JSON array
  local try
  for try in 1 2 3 4 5; do
    curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$1" -o "$2" && jq -e 'type == "array" and length > 0' "$2" >/dev/null 2>&1 && return 0
    echo "  $1: bad response (try $try), retrying" >&2; sleep $((try * 3))
  done
  echo "failed to download $1" >&2; return 1
}

# The jq programs below turn ROC dates (1151008 -> 2026-10-08) into ISO dates and keep only rows with
# positive numeric OHLC, emitting "SYMBOL,Date,O,H,L,C,V".
rows="$work/rows.csv"; : > "$rows"
get_json "https://openapi.twse.com.tw/v1/exchangeReport/STOCK_DAY_ALL" "$work/twse.json" || exit 1
get_json "https://www.tpex.org.tw/openapi/v1/tpex_mainboard_daily_close_quotes" "$work/tpex.json" || exit 1

jq -r '
  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
  .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
  | [(.Code + ".TW"), $d, (.OpeningPrice|num), (.HighestPrice|num), (.LowestPrice|num), (.ClosingPrice|num), (.TradeVolume|num)]
  | select(all(.[2:6][]; . != null)) | @csv' "$work/twse.json" | tr -d '"' >> "$rows"
jq -r '
  def num: if type == "string" then (gsub(",";"") | select(test("^[0-9]+(\\.[0-9]+)?$")) | tonumber | select(. > 0)) else empty end;
  .[] | ((.Date[0:3] | tonumber + 1911 | tostring) + "-" + .Date[3:5] + "-" + .Date[5:7]) as $d
  | [(.SecuritiesCompanyCode + ".TWO"), $d, (.Open|num), (.High|num), (.Low|num), (.Close|num), (.TradingShares|num)]
  | select(all(.[2:6][]; . != null)) | @csv' "$work/tpex.json" | tr -d '"' >> "$rows"

echo "TWSE date: $(jq -r '.[0].Date' "$work/twse.json")  TPEx date: $(jq -r '.[0].Date' "$work/tpex.json") (ROC calendar)"
echo "usable rows: $(wc -l < "$rows" | tr -d ' ')"

updated=0; skipped=0
while IFS= read -r sym; do
  file="$outdir/$sym.csv"
  [ -s "$file" ] || { skipped=$((skipped + 1)); continue; }
  new="$work/new.csv"; grep "^$sym," "$rows" | cut -d, -f2- > "$new"
  [ -s "$new" ] || continue
  merge_into "$file" "$new" && updated=$((updated + 1))
done < <(cut -d, -f1 "$rows" | sort -u)
echo "updated $updated files ($skipped securities without a CSV in $outdir ignored)"