#!/usr/bin/env bash
# Download daily OHLCV history from Yahoo Finance into CSV files readable by `stock`.
#
#   scripts/fetch.sh [-d outdir] [-f symbols.txt] START END SYMBOL...
#
# START/END are YYYY-MM-DD (END inclusive). Symbols use Yahoo naming:
#   2330.TW  TWSE-listed     6488.TWO  TPEx/OTC     ^TWII  Taiwan index     AAPL  US stock
# A bare 4-6 digit code (e.g. 2330) is treated as 2330.TW.
# Existing CSVs are extended incrementally (only newer days are fetched).
set -u

outdir=data; symfile=
while getopts "d:f:" o; do
  case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; *) sed -n '2,10p' "$0"; exit 2;; esac
done
shift $((OPTIND - 1))
[ $# -ge 2 ] || { sed -n '2,10p' "$0"; exit 2; }
start=$1; end=$2; shift 2
symbols=("$@")
[ -n "$symfile" ] && while IFS= read -r l; do l=${l%%#*}; l=${l//[[:space:]]/}; [ -n "$l" ] && symbols+=("$l"); done < "$symfile"
[ ${#symbols[@]} -gt 0 ] || { echo "no symbols given" >&2; exit 2; }
for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done

epoch() {  # YYYY-MM-DD -> epoch seconds (UTC); BSD date, then GNU date
  date -j -u -f "%Y-%m-%d %H:%M:%S" "$1 00:00:00" +%s 2>/dev/null || date -u -d "$1" +%s
}
next_day() { date -j -v+1d -f "%Y-%m-%d" "$1" +%Y-%m-%d 2>/dev/null || date -d "$1 +1 day" +%Y-%m-%d; }

mkdir -p "$outdir"
: > "$outdir/failed.txt"
header="Date,Open,High,Low,Close,Volume"
p2=$(( $(epoch "$end") + 86400 ))     # make END inclusive

fetch_one() {  # prints CSV rows (no header) for $1 between $2 and $3(epoch)
  local sym=$1 from=$2 to=$3 body code tries=0
  local url="https://query1.finance.yahoo.com/v8/finance/chart/${sym}?period1=${from}&period2=${to}&interval=1d&events=history"
  while :; do
    body=$(curl -s -A "Mozilla/5.0" -w '\n%{http_code}' "$url"); code=${body##*$'\n'}; body=${body%$'\n'*}
    case $code in
      200) break;;
      404) echo "  $sym: not found (404)" >&2; return 1;;
      429|5??) tries=$((tries + 1)); [ $tries -gt 5 ] && { echo "  $sym: giving up (HTTP $code)" >&2; return 1; }
               echo "  $sym: HTTP $code, retry $tries in $((5 * 2 ** (tries - 1)))s" >&2; sleep $((5 * 2 ** (tries - 1)));;
      *) echo "  $sym: HTTP $code" >&2; return 1;;
    esac
  done
  printf '%s' "$body" | jq -r '
    if .chart.error != null then error(.chart.error.description) else . end
    | .chart.result[0] as $r
    | if ($r.timestamp // null) == null then empty else
      ($r.meta.gmtoffset // 0) as $off
      | $r.indicators.quote[0] as $q
      | range(0; $r.timestamp | length) as $i
      | [($r.timestamp[$i] + $off | strftime("%Y-%m-%d")), $q.open[$i], $q.high[$i], $q.low[$i], $q.close[$i], $q.volume[$i]]
      | select(all(.[]; . != null))
      | map(if type == "number" then (. * 10000 | round / 10000) else . end)
      | @csv | gsub("\""; "") end'
}

n=0
for raw in "${symbols[@]}"; do
  sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
  file="$outdir/$sym.csv"
  from=$start
  if [ -s "$file" ]; then
    last=$(tail -n 1 "$file" | cut -d, -f1)
    [[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] && from=$(next_day "$last")
  fi
  [ $n -gt 0 ] && sleep $((1 + RANDOM % 2))
  n=$((n + 1))
  p1=$(epoch "$from")
  if [ "$p1" -ge "$p2" ]; then echo "$sym: up to date"; continue; fi
  echo "$sym: $from .. $end"
  if rows=$(fetch_one "$sym" "$p1" "$p2"); then
    [ -s "$file" ] || echo "$header" > "$file"
    [ -n "$rows" ] && printf '%s\n' "$rows" >> "$file"
    echo "  $(printf '%s' "$rows" | grep -c .) new rows -> $file"
  else
    echo "$sym" >> "$outdir/failed.txt"
  fi
done
[ -s "$outdir/failed.txt" ] && { echo "failed symbols listed in $outdir/failed.txt" >&2; exit 1; }
rm -f "$outdir/failed.txt"