added daily fetchers

* scripts/fetch_tw_daily.sh
* scripts/lib.sh
* scripts/symbols.sh: daily fetchers quickly fetch data for today.

1 files changed, 31 insertions, 0 deletions

diff --git a/scripts/symbols.sh b/scripts/symbols.sh
new file mode 100755
index 0000000..5d6ac3f
--- /dev/null
+++ b/scripts/symbols.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Build a list of Yahoo symbols for all ordinary Taiwan stocks (4-digit codes).
+#
+#   scripts/symbols.sh [-o symbols_tw.txt]
+#
+# TWSE-listed companies -> CODE.TW, TPEx (OTC) companies -> CODE.TWO, from the exchanges' open
+# company lists. ETFs, warrants and other products are not in these lists.
+set -u
+out=symbols_tw.txt
+while getopts "o:" o; do case $o in o) out=$OPTARG;; *) sed -n '2,8p' "$0"; exit 2;; esac; done
+for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
+
+get_codes() {  # URL JQ-FIELD SUFFIX; retries until the response parses as JSON
+  local url=$1 field=$2 suffix=$3 try body
+  for try in 1 2 3 4 5; do
+    body=$(curl -s --http1.1 -m 120 -A "Mozilla/5.0" "$url")
+    if codes=$(printf '%s' "$body" | jq -r --arg f "$field" '.[][$f]' 2>/dev/null) && [ -n "$codes" ]; then
+      printf '%s\n' "$codes" | grep -E '^[0-9]{4}$' | sort -u | sed "s/\$/.$suffix/"
+      return 0
+    fi
+    echo "  $url: bad response (try $try), retrying" >&2; sleep $((try * 3))
+  done
+  echo "failed to download $url" >&2; return 1
+}
+
+tmp=$(mktemp)
+{ get_codes "https://openapi.twse.com.tw/v1/opendata/t187ap03_L" "公司代號" TW &&
+  get_codes "https://www.tpex.org.tw/openapi/v1/mopsfin_t187ap03_O" "SecuritiesCompanyCode" TWO; } > "$tmp" \
+  || { rm -f "$tmp"; exit 1; }
+mv "$tmp" "$out"
+echo "wrote $(wc -l < "$out" | tr -d ' ') symbols to $out ($(grep -c '\.TW$' "$out") TWSE, $(grep -c '\.TWO$' "$out") TPEx)"