This repository has been archived by the owner on May 26, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
test
executable file
·119 lines (112 loc) · 4.01 KB
/
test
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
#!/bin/bash -ex
header() {
echo -e "\n\033[32m$@\033[0m"
}
header basic
manifest=$(./run-flow flow/basic/tac.unit)
# difficult way to get the output
result=$(./kgit cat-file blob $manifest | tail -n1 | cut -d' ' -f3)
diff - <(./kgit cat-file blob $result) <<EOF
3
2
1
EOF
header reuse
manifest=$(./run-flow flow/reuse/abc.unit)
# difficult way to get the output
result=$(./kgit cat-file blob $manifest | tail -n1 | cut -d' ' -f3)
diff - <(./kgit cat-file blob $result) <<EOF
a
b
c
EOF
header storage
manifest=$(./run-flow flow/storage/tac.unit)
# difficult way to get the output
result=$(./kgit cat-file blob $manifest | tail -n1 | cut -d' ' -f3)
diff - <(./kgit cat-file blob $result) <<EOF
format=text/plain
path=/var/tmp/knit/9941e2c2869be9f6fef2250f3cd85527d059dadd
store=fs
EOF
diff - /var/tmp/knit/9941e2c2869be9f6fef2250f3cd85527d059dadd <<EOF
3
2
1
EOF
header build
rm -rf gen/flow
build=$(./build-unit start)
./build-unit --build $build copy /dev/stdin 123 <<EOF
1
2
3
EOF
./build-unit --build $build run 'cat $input_123'
./build-unit --build $build commit gen/flow/123.unit
build=$(./build-unit start)
./build-unit --build $build set in abc
./build-unit --build $build run 'cat $input_in'
./build-unit --build $build commit gen/flow/abc.unit
build=$(./build-unit start)
./build-unit --build $build after gen/flow/123.unit 123
./build-unit --build $build after gen/flow/abc.unit abc
./build-unit --build $build run 'tac $input_123 $input_abc'
./build-unit --build $build commit gen/flow/tac.unit
manifest=$(./run-flow gen/flow/tac.unit)
# difficult way to get the output
result=$(./kgit cat-file blob $manifest | tail -n1 | cut -d' ' -f3)
diff - <(./kgit cat-file blob $result) <<EOF
3
2
1
abc
EOF
header build taxi
rm -rf gen/flow
build=$(./build-unit start)
./build-unit --build $build run 'curl -fs https://s3.amazonaws.com/nyc-tlc/misc/taxi+_zone_lookup.csv'
./build-unit --build $build commit gen/flow/taxi-zone-lookup.unit
build=$(./build-unit start)
./build-unit --build $build after gen/flow/taxi-zone-lookup.unit csv
./build-unit --build $build copy helpers/sqlite sqlite
./build-unit --build $build copy /dev/stdin sql <<EOF
SELECT LocationID, Borough
FROM zones;
EOF
./build-unit --build $build run 'bash -e $input_sqlite zones=$input_csv < $input_sql'
./build-unit --build $build commit gen/flow/zone-borough.unit
build=$(./build-unit start)
./build-unit --build $build run 'curl -fs https://s3.amazonaws.com/nyc-tlc/trip+data/yellow_tripdata_2018-12.csv | head -n 10000'
./build-unit --build $build commit gen/flow/yellow-tripdata.unit
build=$(./build-unit start)
./build-unit --build $build after gen/flow/yellow-tripdata.unit tripdata
./build-unit --build $build copy helpers/sqlite sqlite
./build-unit --build $build copy /dev/stdin sql <<EOF
SELECT STRFTIME('%s', tpep_dropoff_datetime) - STRFTIME('%s', tpep_pickup_datetime) AS duration_s, trip_distance, total_amount
FROM tripdata;
EOF
./build-unit --build $build run 'bash -e $input_sqlite tripdata=$input_tripdata < $input_sql'
./build-unit --build $build commit gen/flow/duration-fare.unit
build=$(./build-unit start)
./build-unit --build $build after gen/flow/duration-fare.unit duration_fare
./build-unit --build $build copy helpers/sqlite sqlite
# TODO parameterize
./build-unit --build $build copy /dev/stdin sql <<EOF
SELECT duration_s / 60 AS duration_m, AVG(total_amount) AS total_amount
FROM input
GROUP BY duration_s / 60
HAVING duration_m > 0
AND duration_m < 60
ORDER BY duration_m;
EOF
./build-unit --build $build run 'bash -e $input_sqlite input=$input_duration_fare < $input_sql'
./build-unit --build $build commit gen/flow/duration-amount.unit
build=$(./build-unit start)
./build-unit --build $build after gen/flow/duration-amount.unit duration_amount
./build-unit --build $build copy helpers/bash-template bt
# TODO non-test dependency
tail -n+2 flow/taxi/taxi.html | head -n-1 | ./build-unit --build $build copy /dev/stdin html
./build-unit --build $build run 'export input_duration_amount; bash -e $input_bt $input_html /dev/null'
./build-unit --build $build commit gen/flow/taxi.unit
header PASS