fetch.sh update: more advanced skipping and handling

* scripts/fetch.sh

1 files changed, 68 insertions, 15 deletions

diff --git a/scripts/fetch.sh b/scripts/fetch.sh
index 5ed4901..33b1da2 100755
--- a/scripts/fetch.sh
+++ b/scripts/fetch.sh
@@ -1,21 +1,34 @@
 #!/usr/bin/env bash
 # Download daily OHLCV history from Yahoo Finance into CSV files readable by `stock`.
 #
-#   scripts/fetch.sh [-d outdir] [-f symbols.txt] START END SYMBOL...
+#   scripts/fetch.sh [-F] [-d outdir] [-f symbols.txt] START END SYMBOL...
+#   scripts/fetch.sh -c [-d outdir] SYMBOL...        (list weekdays with no row; holidays show up too)
 #
 # START/END are YYYY-MM-DD (END inclusive). Symbols use Yahoo naming:
 #   2330.TW  TWSE-listed     6488.TWO  TPEx/OTC     ^TWII  Taiwan index     AAPL  US stock
 # A bare 4-6 digit code (e.g. 2330) is treated as 2330.TW.
-# Existing CSVs are extended incrementally (only newer days are fetched).
+# Existing CSVs are updated incrementally: the last ~10 days are re-fetched and merged by date
+# (new values win, file stays sorted, holes inside that window are repaired). If START is
+# earlier than the first stored date, the whole range is fetched again.
+# Skipping (so interrupted or frequent runs stay cheap; -F disables all of it):
+#  - a symbol whose CSV was modified in the last 12 hours is skipped;
+#  - for *.TW / *.TWO symbols, a CSV that already contains every one of the last 10 trading days of
+#    2330.TW (fetched once at the start as a reference) is skipped. A stock that really did not
+#    trade on one of those days (halted) is fetched again, which is harmless. Skipped stocks are
+#    not backfilled when START is moved earlier: use -F for that.
+# A >1% close mismatch on overlapping days is reported (probable split / capital reduction).
 set -u
+source "$(dirname "$0")/lib.sh"
 
-outdir=data; symfile=
-while getopts "d:f:" o; do
-  case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; *) sed -n '2,10p' "$0"; exit 2;; esac
+outdir=data; symfile=; check=0; force=0
+while getopts "d:f:cF" o; do
+  case $o in d) outdir=$OPTARG;; f) symfile=$OPTARG;; c) check=1;; F) force=1;; *) sed -n '2,22p' "$0"; exit 2;; esac
 done
 shift $((OPTIND - 1))
-[ $# -ge 2 ] || { sed -n '2,10p' "$0"; exit 2; }
-start=$1; end=$2; shift 2
+if [ $check -eq 0 ]; then
+  [ $# -ge 2 ] || { sed -n '2,22p' "$0"; exit 2; }
+  start=$1; end=$2; shift 2
+fi
 symbols=("$@")
 [ -n "$symfile" ] && while IFS= read -r l; do l=${l%%#*}; l=${l//[[:space:]]/}; [ -n "$l" ] && symbols+=("$l"); done < "$symfile"
 [ ${#symbols[@]} -gt 0 ] || { echo "no symbols given" >&2; exit 2; }
@@ -26,9 +39,22 @@ epoch() {  # YYYY-MM-DD -> epoch seconds (UTC); BSD date, then GNU date
 }
 next_day() { date -j -v+1d -f "%Y-%m-%d" "$1" +%Y-%m-%d 2>/dev/null || date -d "$1 +1 day" +%Y-%m-%d; }
 
+if [ $check -eq 1 ]; then
+  for raw in "${symbols[@]}"; do
+    sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
+    file="$outdir/$sym.csv"; [ -s "$file" ] || { echo "$sym: no file"; continue; }
+    echo "$sym: weekdays without a row"
+    tail -n +2 "$file" | cut -d, -f1 | jq -Rn '
+      [inputs | strptime("%Y-%m-%d") | mktime] as $e
+      | ($e | map({(tostring): true}) | add) as $have
+      | range($e[0]; $e[-1]; 86400) | select(($have[tostring] | not) and (gmtime | .[6] | . >= 1 and . <= 5))
+      | strftime("  %Y-%m-%d")' -r
+  done
+  exit 0
+fi
+
 mkdir -p "$outdir"
 : > "$outdir/failed.txt"
-header="Date,Open,High,Low,Close,Volume"
 p2=$(( $(epoch "$end") + 86400 ))     # make END inclusive
 
 fetch_one() {  # prints CSV rows (no header) for $1 between $2 and $3(epoch)
@@ -39,7 +65,7 @@ fetch_one() {  # prints CSV rows (no header) for $1 between $2 and $3(epoch)
     case $code in
       200) break;;
       404) echo "  $sym: not found (404)" >&2; return 1;;
-      429|5??) tries=$((tries + 1)); [ $tries -gt 5 ] && { echo "  $sym: giving up (HTTP $code)" >&2; return 1; }
+      429|5??) tries=$((tries + 1)); [ $tries -gt 7 ] && { echo "  $sym: giving up (HTTP $code)" >&2; return 1; }
                echo "  $sym: HTTP $code, retry $tries in $((5 * 2 ** (tries - 1)))s" >&2; sleep $((5 * 2 ** (tries - 1)));;
       *) echo "  $sym: HTTP $code" >&2; return 1;;
     esac
@@ -57,27 +83,54 @@ fetch_one() {  # prints CSV rows (no header) for $1 between $2 and $3(epoch)
       | @csv | gsub("\""; "") end'
 }
 
+# Reference trading days for the smart skip: last 10 trading days of 2330.TW.
+ref_file=
+if [ $force -eq 0 ]; then
+  probe_from=$(epoch "$(date -j -v-20d -f "%Y-%m-%d" "$end" +%Y-%m-%d 2>/dev/null || date -d "$end -20 day" +%Y-%m-%d)")
+  if probe=$(fetch_one 2330.TW "$probe_from" "$p2") && [ -n "$probe" ]; then
+    ref_file=$(mktemp)
+    printf '%s\n' "$probe" | cut -d, -f1 | tail -n 10 > "$ref_file"
+    echo "reference trading days (2330.TW): $(head -n 1 "$ref_file") .. $(tail -n 1 "$ref_file")"
+  else
+    echo "warning: could not fetch the reference stock; skipping only by file age" >&2
+  fi
+fi
+
 n=0
 for raw in "${symbols[@]}"; do
   sym=$raw; [[ $sym =~ ^[0-9]{4,6}$ ]] && sym=$sym.TW
   file="$outdir/$sym.csv"
+  if [ $force -eq 0 ] && [ -n "$(find "$file" -mmin -720 2>/dev/null)" ]; then
+    echo "$sym: updated within 12h, skipped"; continue
+  fi
+  if [ -n "$ref_file" ] && [[ $sym == *.TW || $sym == *.TWO ]] && [ -s "$file" ] &&
+     awk -F, 'NR == FNR { need[$1]; n++; next } $1 in need { c++ } END { exit c != n }' "$ref_file" "$file"; then
+    echo "$sym: has the latest trading days, skipped"; continue
+  fi
   from=$start
   if [ -s "$file" ]; then
-    last=$(tail -n 1 "$file" | cut -d, -f1)
-    [[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ ]] && from=$(next_day "$last")
+    first=$(sed -n 2p "$file" | cut -d, -f1); last=$(tail -n 1 "$file" | cut -d, -f1)
+    if [[ $last =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ && ! ( $start < $first ) ]]; then
+      # re-fetch the last ~10 days so holes and revised values are repaired
+      from=$(date -j -v-10d -f "%Y-%m-%d" "$last" +%Y-%m-%d 2>/dev/null || date -d "$last -10 day" +%Y-%m-%d)
+      [[ $from < $start ]] && from=$start
+    fi
   fi
   [ $n -gt 0 ] && sleep $((1 + RANDOM % 2))
   n=$((n + 1))
   p1=$(epoch "$from")
-  if [ "$p1" -ge "$p2" ]; then echo "$sym: up to date"; continue; fi
   echo "$sym: $from .. $end"
   if rows=$(fetch_one "$sym" "$p1" "$p2"); then
-    [ -s "$file" ] || echo "$header" > "$file"
-    [ -n "$rows" ] && printf '%s\n' "$rows" >> "$file"
-    echo "  $(printf '%s' "$rows" | grep -c .) new rows -> $file"
+    if [ -s "$file" ]; then
+      newf=$(mktemp); printf '%s\n' "$rows" > "$newf"; merge_into "$file" "$newf"; rm -f "$newf"
+    else
+      { echo "$header"; [ -n "$rows" ] && printf '%s\n' "$rows"; } > "$file"
+    fi
+    echo "  fetched $(printf '%s' "$rows" | grep -c .) rows -> $file ($(($(wc -l < "$file") - 1)) total)"
   else
     echo "$sym" >> "$outdir/failed.txt"
   fi
 done
+[ -n "$ref_file" ] && rm -f "$ref_file"
 [ -s "$outdir/failed.txt" ] && { echo "failed symbols listed in $outdir/failed.txt" >&2; exit 1; }
 rm -f "$outdir/failed.txt"