1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
|
#!/usr/bin/env bash
# Mac side: pull the VPS's daily snapshot files and merge them into the per-stock CSVs.
#
# scripts/sync_from_vps.sh [--dry-run] [-r user@host:stock/updates/] [-d data/tw] [-u data/updates]
#
# The remote can also be given as $VPS_UPDATES. The VPS keeps only recent daily files
# (Symbol,Date,Open,High,Low,Close,Volume). They are copied to the local cache (-u), which mirrors
# the remote folder, and all cached snapshots are merged by date into the CSVs in -d. Merging is
# idempotent (re-applying a day changes nothing), which also repairs a day that was published in
# two parts. Stocks without a local CSV are ignored. Days you missed entirely (the VPS only keeps
# a few weeks) can be filled from Yahoo with fetch.sh.
set -u
source "$(dirname "$0")/lib.sh"
remote=${VPS_UPDATES:-}; outdir=data/tw; cache=data/updates; dry=0
args=(); for a in "$@"; do [ "$a" = "--dry-run" ] && a=-n; args+=("$a"); done # getopts has no long options
set -- ${args[@]+"${args[@]}"}
while getopts "r:d:u:n" o; do
case $o in r) remote=$OPTARG;; d) outdir=$OPTARG;; u) cache=$OPTARG;; n) dry=1;; *) sed -n '2,13p' "$0"; exit 2;; esac
done
[ -n "$remote" ] || { echo "no remote given: use -r user@host:stock/updates/ or set VPS_UPDATES" >&2; exit 2; }
[ -d "$outdir" ] || { echo "$outdir does not exist" >&2; exit 1; }
for cmd in rsync awk sort; do command -v $cmd >/dev/null || { echo "$cmd is required" >&2; exit 1; }; done
mkdir -p "$cache"
latest_local() {
for f in "$outdir"/*.csv; do tail -n 1 "$f" | cut -d, -f1; done | grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' | sort | tail -n 1
}
echo "latest date in $outdir before sync: $(latest_local)"
remote=${remote%/}/
if [ $dry -eq 1 ]; then
echo "--- files that would be downloaded:"
rsync -avn --delete "$remote" "$cache/"
exit 0
fi
rsync -av --partial --delete "$remote" "$cache/" || { echo "rsync failed" >&2; exit 1; }
shopt -s nullglob
snaps=("$cache"/????-??-??.csv)
[ ${#snaps[@]} -gt 0 ] || { echo "no snapshot files in $cache"; exit 0; }
work=$(mktemp -d) && [ -n "$work" ] && [ -d "$work" ] || { echo "cannot create a temporary directory" >&2; exit 1; }
trap 'rm -rf "$work"' EXIT
# All cached rows sorted by symbol then date, restricted to stocks we have, split into one file each.
for f in "${snaps[@]}"; do tail -n +2 "$f"; done | sort -t, -k1,1 -k2,2 > "$work/all.csv"
ls "$outdir" | sed -n 's/\.csv$//p' > "$work/have.txt"
mkdir "$work/split"
awk -F, -v dir="$work/split" '
FNR == NR { want[$1]; next }
!($1 in want) { next }
$1 != prev { if (out != "") close(out); prev = $1; out = dir "/" $1 ".csv" }
{ print $2 "," $3 "," $4 "," $5 "," $6 "," $7 > out }' "$work/have.txt" "$work/all.csv"
n=0
for f in "$work"/split/*.csv; do
merge_into "$outdir/$(basename "$f")" "$f" && n=$((n + 1))
done
echo "merged ${#snaps[@]} snapshot days into $n stock files"
echo "latest date in $outdir after sync: $(latest_local)"
|
