#!/usr/bin/env bash
# Build a list of Yahoo symbols for all ordinary Taiwan stocks (4-digit codes).
#
#   scripts/symbols.sh [-o symbols_tw.txt]
#
# TWSE-listed companies -> CODE.TW, TPEx (OTC) companies -> CODE.TWO, from the exchanges' open
# company lists. ETFs, warrants and other products are not in these lists.
set -u
out=symbols_tw.txt
while getopts "o:" o; do case $o in o) out=$OPTARG;; *) sed -n '2,8p' "$0"; exit 2;; esac; done
for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done

get_codes() {  # URL JQ-FIELD SUFFIX; retries until the response parses as JSON
  local url=$1 field=$2 suffix=$3 try body
  for try in 1 2 3 4 5; do
    body=$(curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$url")
    if codes=$(printf '%s' "$body" | jq -r --arg f "$field" '.[][$f]' 2>/dev/null) && [ -n "$codes" ]; then
      printf '%s\n' "$codes" | grep -E '^[0-9]{4}$' | sort -u | sed "s/\$/.$suffix/"
      return 0
    fi
    echo "  $url: bad response (try $try), retrying" >&2; sleep $((try * 3))
  done
  echo "failed to download $url" >&2; return 1
}

tmp=$(mktemp)
{ get_codes "https://openapi.twse.com.tw/v1/opendata/t187ap03_L" "公司代號" TW &&
  get_codes "https://www.tpex.org.tw/openapi/v1/mopsfin_t187ap03_O" "SecuritiesCompanyCode" TWO; } > "$tmp" \
  || { rm -f "$tmp"; exit 1; }
mv "$tmp" "$out"
echo "wrote $(wc -l < "$out" | tr -d ' ') symbols to $out ($(grep -c '\.TW$' "$out") TWSE, $(grep -c '\.TWO$' "$out") TPEx)"