1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
|
#!/usr/bin/env bash
# Download daily OHLCV history from Yahoo Finance into CSV files readable by `stock`.
#
# scripts/fetch.sh [-F] [-d outdir] [-f symbols.txt] START END SYMBOL...
# scripts/fetch.sh -c [-d outdir] SYMBOL... (list weekdays with no row; holidays show up too)
#
# START/END are YYYY-MM-DD (END inclusive). Symbols use Yahoo naming:
# 2330.TW TWSE-listed 6488.TWO TPEx/OTC ^TWII Taiwan index AAPL US stock
# A bare 4-6 digit code (e.g. 2330) is treated as 2330.TW.
# Existing CSVs are updated incrementally: the last ~10 days are re-fetched and merged by date
# (new values win, file stays sorted, holes inside that window are repaired). If START is
# earlier than the first stored date, the whole range is fetched again.
# Skipping (so interrupted or frequent runs stay cheap; -F disables all of it):
# - a symbol whose CSV was modified in the last 12 hours is skipped;
# - for *.TW / *.TWO symbols, a CSV that already contains every one of the last 10 trading days of
# 2330.TW (fetched once at the start as a reference) is skipped. A stock that really did not
# trade on one of those days (halted) is fetched again, which is harmless. Skipped stocks are
# not backfilled when START is moved earlier: use -F for that.
# A >1% close mismatch on overlapping days is reported (probable split / capital reduction).
set -u
source "$(dirname "$0")/lib.sh"
outdir=data; symfile=; check=0; force=0
while getopts "d:f:cF" o; do
case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; c) check=1;; F) force=1;; *) sed -n '2,22p' "$0"; exit 2;; esac
done
shift $((OPTIND - 1))
if [ $check -eq 0 ]; then
[ $# -ge 2 ] || { sed -n '2,22p' "$0"; exit 2; }
start=$1; end=$2; shift 2
fi
symbols=("$@")
[ -n "$symfile" ] && while IFS= read -r l; do l=${l%%#*}; l=${l//[[:space:]]/}; [ -n "$l" ] && symbols+=("$l"); done < "$symfile"
[ ${#symbols[@]} -gt 0 ] || { echo "no symbols given" >&2; exit 2; }
for cmd in curl jq; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
epoch() { # YYYY-MM-DD -> epoch seconds (UTC); BSD date, then GNU date
date -j -u -f "%Y-%m-%d %H:%M:%S" "$1 00:00:00" +%s 2>/dev/null || date -u -d "$1" +%s
}
next_day() { date -j -v+1d -f "%Y-%m-%d" "$1" +%Y-%m-%d 2>/dev/null || date -d "$1 +1 day" +%Y-%m-%d; }
if [ $check -eq 1 ]; then
for raw in "${symbols[@]}"; do
sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
file="$outdir/$sym.csv"; [ -s "$file" ] || { echo "$sym: no file"; continue; }
echo "$sym: weekdays without a row"
tail -n +2 "$file" | cut -d, -f1 | jq -Rn '
[inputs | strptime("%Y-%m-%d") | mktime] as $e
| ($e | map({(tostring): true}) | add) as $have
| range($e[0]; $e[-1]; 86400) | select(($have[tostring] | not) and (gmtime | .[6] | . >= 1 and . <= 5))
| strftime(" %Y-%m-%d")' -r
done
exit 0
fi
mkdir -p "$outdir"
: > "$outdir/failed.txt"
p2=$(( $(epoch "$end") + 86400 )) # make END inclusive
fetch_one() { # prints CSV rows (no header) for $1 between $2 and $3(epoch)
local sym=$1 from=$2 to=$3 body code tries=0
local url="https://query1.finance.yahoo.com/v8/finance/chart/${sym}?period1=${from}&period2=${to}&interval=1d&events=history"
while :; do
body=$(curl -s -A "Mozilla/5.0" -w '\n%{http_code}' "$url"); code=${body##*$'\n'}; body=${body%$'\n'*}
case $code in
200) break;;
404) echo " $sym: not found (404)" >&2; return 1;;
429|5??) tries=$((tries + 1)); [ $tries -gt 7 ] && { echo " $sym: giving up (HTTP $code)" >&2; return 1; }
echo " $sym: HTTP $code, retry $tries in $((5 * 2 ** (tries - 1)))s" >&2; sleep $((5 * 2 ** (tries - 1)));;
*) echo " $sym: HTTP $code" >&2; return 1;;
esac
done
printf '%s' "$body" | jq -r '
if .chart.error != null then error(.chart.error.description) else . end
| .chart.result[0] as $r
| if ($r.timestamp // null) == null then empty else
($r.meta.gmtoffset // 0) as $off
| $r.indicators.quote[0] as $q
| range(0; $r.timestamp | length) as $i
| [($r.timestamp[$i] + $off | strftime("%Y-%m-%d")), $q.open[$i], $q.high[$i], $q.low[$i], $q.close[$i], $q.volume[$i]]
| select(all(.[]; . != null))
| map(if type == "number" then (. * 10000 | round / 10000) else . end)
| @csv | gsub("\""; "") end'
}
# Reference trading days for the smart skip: last 10 trading days of 2330.TW.
ref_file=
if [ $force -eq 0 ]; then
probe_from=$(epoch "$(date -j -v-20d -f "%Y-%m-%d" "$end" +%Y-%m-%d 2>/dev/null || date -d "$end -20 day" +%Y-%m-%d)")
if probe=$(fetch_one 2330.TW "$probe_from" "$p2") && [ -n "$probe" ]; then
ref_file=$(mktemp)
printf '%s\n' "$probe" | cut -d, -f1 | tail -n 10 > "$ref_file"
echo "reference trading days (2330.TW): $(head -n 1 "$ref_file") .. $(tail -n 1 "$ref_file")"
else
echo "warning: could not fetch the reference stock; skipping only by file age" >&2
fi
fi
n=0
for raw in "${symbols[@]}"; do
sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
file="$outdir/$sym.csv"
if [ $force -eq 0 ] && [ -n "$(find "$file" -mmin -720 2>/dev/null)" ]; then
echo "$sym: updated within 12h, skipped"; continue
fi
if [ -n "$ref_file" ] && [[ $sym == *.TW || $sym == *.TWO ]] && [ -s "$file" ] &&
awk -F, 'NR == FNR { need[$1]; n++; next } $1 in need { c++ } END { exit c != n }' "$ref_file" "$file"; then
echo "$sym: has the latest trading days, skipped"; continue
fi
from=$start
if [ -s "$file" ]; then
first=$(sed -n 2p "$file" | cut -d, -f1); last=$(tail -n 1 "$file" | cut -d, -f1)
if [[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ && ! ( $start < $first ) ]]; then
# re-fetch the last ~10 days so holes and revised values are repaired
from=$(date -j -v-10d -f "%Y-%m-%d" "$last" +%Y-%m-%d 2>/dev/null || date -d "$last -10 day" +%Y-%m-%d)
[[ $from < $start ]] && from=$start
fi
fi
[ $n -gt 0 ] && sleep $((1 + RANDOM % 2))
n=$((n + 1))
p1=$(epoch "$from")
echo "$sym: $from .. $end"
if rows=$(fetch_one "$sym" "$p1" "$p2"); then
if [ -s "$file" ]; then
newf=$(mktemp); printf '%s\n' "$rows" > "$newf"; merge_into "$file" "$newf"; rm -f "$newf"
else
{ echo "$header"; [ -n "$rows" ] && printf '%s\n' "$rows"; } > "$file"
fi
echo " fetched $(printf '%s' "$rows" | grep -c .) rows -> $file ($(($(wc -l < "$file") - 1)) total)"
else
echo "$sym" >> "$outdir/failed.txt"
fi
done
[ -n "$ref_file" ] && rm -f "$ref_file"
[ -s "$outdir/failed.txt" ] && { echo "failed symbols listed in $outdir/failed.txt" >&2; exit 1; }
rm -f "$outdir/failed.txt"
|
