1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
#!/usr/bin/env bash
# Download daily OHLCV history from Yahoo Finance into CSV files readable by `stock`.
#
# scripts/fetch.sh [-d outdir] [-f symbols.txt] START END SYMBOL...
#
# START/END are YYYY-MM-DD (END inclusive). Symbols use Yahoo naming:
# 2330.TW TWSE-listed 6488.TWO TPEx/OTC ^TWII Taiwan index AAPL US stock
# A bare 4-6 digit code (e.g. 2330) is treated as 2330.TW.
# Existing CSVs are extended incrementally (only newer days are fetched).
set -u
outdir=data; symfile=
while getopts "d:f:" o; do
case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; *) sed -n '2,10p' "$0"; exit 2;; esac
done
shift $((OPTIND - 1))
[ $# -ge 2 ] || { sed -n '2,10p' "$0"; exit 2; }
start=$1; end=$2; shift 2
symbols=("$@")
[ -n "$symfile" ] && while IFS= read -r l; do l=${l%%#*}; l=${l//[[:space:]]/}; [ -n "$l" ] && symbols+=("$l"); done < "$symfile"
[ ${#symbols[@]} -gt 0 ] || { echo "no symbols given" >&2; exit 2; }
for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
epoch() { # YYYY-MM-DD -> epoch seconds (UTC); BSD date, then GNU date
date -j -u -f "%Y-%m-%d %H:%M:%S" "$1 00:00:00" +%s 2>/dev/null || date -u -d "$1" +%s
}
next_day() { date -j -v+1d -f "%Y-%m-%d" "$1" +%Y-%m-%d 2>/dev/null || date -d "$1 +1 day" +%Y-%m-%d; }
mkdir -p "$outdir"
: > "$outdir/failed.txt"
header="Date,Open,High,Low,Close,Volume"
p2=$(( $(epoch "$end") + 86400 )) # make END inclusive
fetch_one() { # prints CSV rows (no header) for $1 between $2 and $3(epoch)
local sym=$1 from=$2 to=$3 body code tries=0
local url="https://query1.finance.yahoo.com/v8/finance/chart/${sym}?period1=${from}&period2=${to}&interval=1d&events=history"
while :; do
body=$(curl -s -A "Mozilla/5.0" -w '\n%{http_code}' "$url"); code=${body##*$'\n'}; body=${body%$'\n'*}
case $code in
200) break;;
404) echo " $sym: not found (404)" >&2; return 1;;
429|5??) tries=$((tries + 1)); [ $tries -gt 5 ] && { echo " $sym: giving up (HTTP $code)" >&2; return 1; }
echo " $sym: HTTP $code, retry $tries in $((5 * 2 ** (tries - 1)))s" >&2; sleep $((5 * 2 ** (tries - 1)));;
*) echo " $sym: HTTP $code" >&2; return 1;;
esac
done
printf '%s' "$body" | jq -r '
if .chart.error != null then error(.chart.error.description) else . end
| .chart.result[0] as $r
| if ($r.timestamp // null) == null then empty else
($r.meta.gmtoffset // 0) as $off
| $r.indicators.quote[0] as $q
| range(0; $r.timestamp | length) as $i
| [($r.timestamp[$i] + $off | strftime("%Y-%m-%d")), $q.open[$i], $q.high[$i], $q.low[$i], $q.close[$i], $q.volume[$i]]
| select(all(.[]; . != null))
| map(if type == "number" then (. * 10000 | round / 10000) else . end)
| @csv | gsub("\""; "") end'
}
n=0
for raw in "${symbols[@]}"; do
sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
file="$outdir/$sym.csv"
from=$start
if [ -s "$file" ]; then
last=$(tail -n 1 "$file" | cut -d, -f1)
[[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] && from=$(next_day "$last")
fi
[ $n -gt 0 ] && sleep $((1 + RANDOM % 2))
n=$((n + 1))
p1=$(epoch "$from")
if [ "$p1" -ge "$p2" ]; then echo "$sym: up to date"; continue; fi
echo "$sym: $from .. $end"
if rows=$(fetch_one "$sym" "$p1" "$p2"); then
[ -s "$file" ] || echo "$header" > "$file"
[ -n "$rows" ] && printf '%s\n' "$rows" >> "$file"
echo " $(printf '%s' "$rows" | grep -c .) new rows -> $file"
else
echo "$sym" >> "$outdir/failed.txt"
fi
done
[ -s "$outdir/failed.txt" ] && { echo "failed symbols listed in $outdir/failed.txt" >&2; exit 1; }
rm -f "$outdir/failed.txt"
|
