#!/usr/bin/env bash # Download daily OHLCV history from Yahoo Finance into CSV files readable by `stock`. # # scripts/fetch.sh [-F] [-d outdir] [-f symbols.txt] START END SYMBOL... # scripts/fetch.sh -c [-d outdir] SYMBOL... (list weekdays with no row; holidays show up too) # # START/END are YYYY-MM-DD (END inclusive). Symbols use Yahoo naming: # 2330.TW TWSE-listed 6488.TWO TPEx/OTC ^TWII Taiwan index AAPL US stock # A bare 4-6 digit code (e.g. 2330) is treated as 2330.TW. # Existing CSVs are updated incrementally: the last ~10 days are re-fetched and merged by date # (new values win, file stays sorted, holes inside that window are repaired). If START is # earlier than the first stored date, the whole range is fetched again. # Skipping (so interrupted or frequent runs stay cheap; -F disables all of it): # - a symbol whose CSV was modified in the last 12 hours is skipped; # - for *.TW / *.TWO symbols, a CSV that already contains every one of the last 10 trading days of # 2330.TW (fetched once at the start as a reference) is skipped. A stock that really did not # trade on one of those days (halted) is fetched again, which is harmless. Skipped stocks are # not backfilled when START is moved earlier: use -F for that. # A >1% close mismatch on overlapping days is reported (probable split / capital reduction). set -u source "$(dirname "$0")/lib.sh" outdir=data; symfile=; check=0; force=0 while getopts "d:f:cF" o; do case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; c) check=1;; F) force=1;; *) sed -n '2,22p' "$0"; exit 2;; esac done shift $((OPTIND - 1)) if [ $check -eq 0 ]; then [ $# -ge 2 ] || { sed -n '2,22p' "$0"; exit 2; } start=$1; end=$2; shift 2 fi symbols=("$@") [ -n "$symfile" ] && while IFS= read -r l; do l=${l%%#*}; l=${l//[[:space:]]/}; [ -n "$l" ] && symbols+=("$l"); done < "$symfile" [ ${#symbols[@]} -gt 0 ] || { echo "no symbols given" >&2; exit 2; } for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done epoch() { # YYYY-MM-DD -> epoch seconds (UTC); BSD date, then GNU date date -j -u -f "%Y-%m-%d %H:%M:%S" "$1 00:00:00" +%s 2>/dev/null || date -u -d "$1" +%s } next_day() { date -j -v+1d -f "%Y-%m-%d" "$1" +%Y-%m-%d 2>/dev/null || date -d "$1 +1 day" +%Y-%m-%d; } if [ $check -eq 1 ]; then for raw in "${symbols[@]}"; do sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW file="$outdir/$sym.csv"; [ -s "$file" ] || { echo "$sym: no file"; continue; } echo "$sym: weekdays without a row" tail -n +2 "$file" | cut -d, -f1 | jq -Rn ' [inputs | strptime("%Y-%m-%d") | mktime] as $e | ($e | map({(tostring): true}) | add) as $have | range($e[0]; $e[-1]; 86400) | select(($have[tostring] | not) and (gmtime | .[6] | . >= 1 and . <= 5)) | strftime(" %Y-%m-%d")' -r done exit 0 fi mkdir -p "$outdir" : > "$outdir/failed.txt" p2=$(( $(epoch "$end") + 86400 )) # make END inclusive fetch_one() { # prints CSV rows (no header) for $1 between $2 and $3(epoch) local sym=$1 from=$2 to=$3 body code tries=0 local url="https://query1.finance.yahoo.com/v8/finance/chart/${sym}?period1=${from}&period2=${to}&interval=1d&events=history" while :; do body=$(curl -s -A "Mozilla/5.0" -w '\n%{http_code}' "$url"); code=${body##*$'\n'}; body=${body%$'\n'*} case $code in 200) break;; 404) echo " $sym: not found (404)" >&2; return 1;; 429|5??) tries=$((tries + 1)); [ $tries -gt 7 ] && { echo " $sym: giving up (HTTP $code)" >&2; return 1; } echo " $sym: HTTP $code, retry $tries in $((5 * 2 ** (tries - 1)))s" >&2; sleep $((5 * 2 ** (tries - 1)));; *) echo " $sym: HTTP $code" >&2; return 1;; esac done printf '%s' "$body" | jq -r ' if .chart.error != null then error(.chart.error.description) else . end | .chart.result[0] as $r | if ($r.timestamp // null) == null then empty else ($r.meta.gmtoffset // 0) as $off | $r.indicators.quote[0] as $q | range(0; $r.timestamp | length) as $i | [($r.timestamp[$i] + $off | strftime("%Y-%m-%d")), $q.open[$i], $q.high[$i], $q.low[$i], $q.close[$i], $q.volume[$i]] | select(all(.[]; . != null)) | map(if type == "number" then (. * 10000 | round / 10000) else . end) | @csv | gsub("\""; "") end' } # Reference trading days for the smart skip: last 10 trading days of 2330.TW. ref_file= if [ $force -eq 0 ]; then probe_from=$(epoch "$(date -j -v-20d -f "%Y-%m-%d" "$end" +%Y-%m-%d 2>/dev/null || date -d "$end -20 day" +%Y-%m-%d)") if probe=$(fetch_one 2330.TW "$probe_from" "$p2") && [ -n "$probe" ]; then ref_file=$(mktemp) printf '%s\n' "$probe" | cut -d, -f1 | tail -n 10 > "$ref_file" echo "reference trading days (2330.TW): $(head -n 1 "$ref_file") .. $(tail -n 1 "$ref_file")" else echo "warning: could not fetch the reference stock; skipping only by file age" >&2 fi fi n=0 for raw in "${symbols[@]}"; do sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW file="$outdir/$sym.csv" if [ $force -eq 0 ] && [ -n "$(find "$file" -mmin -720 2>/dev/null)" ]; then echo "$sym: updated within 12h, skipped"; continue fi if [ -n "$ref_file" ] && [[ $sym == *.TW || $sym == *.TWO ]] && [ -s "$file" ] && awk -F, 'NR == FNR { need[$1]; n++; next } $1 in need { c++ } END { exit c != n }' "$ref_file" "$file"; then echo "$sym: has the latest trading days, skipped"; continue fi from=$start if [ -s "$file" ]; then first=$(sed -n 2p "$file" | cut -d, -f1); last=$(tail -n 1 "$file" | cut -d, -f1) if [[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ && ! ( $start < $first ) ]]; then # re-fetch the last ~10 days so holes and revised values are repaired from=$(date -j -v-10d -f "%Y-%m-%d" "$last" +%Y-%m-%d 2>/dev/null || date -d "$last -10 day" +%Y-%m-%d) [[ $from < $start ]] && from=$start fi fi [ $n -gt 0 ] && sleep $((1 + RANDOM % 2)) n=$((n + 1)) p1=$(epoch "$from") echo "$sym: $from .. $end" if rows=$(fetch_one "$sym" "$p1" "$p2"); then if [ -s "$file" ]; then newf=$(mktemp); printf '%s\n' "$rows" > "$newf"; merge_into "$file" "$newf"; rm -f "$newf" else { echo "$header"; [ -n "$rows" ] && printf '%s\n' "$rows"; } > "$file" fi echo " fetched $(printf '%s' "$rows" | grep -c .) rows -> $file ($(($(wc -l < "$file") - 1)) total)" else echo "$sym" >> "$outdir/failed.txt" fi done [ -n "$ref_file" ] && rm -f "$ref_file" [ -s "$outdir/failed.txt" ] && { echo "failed symbols listed in $outdir/failed.txt" >&2; exit 1; } rm -f "$outdir/failed.txt"