|
1 #!/usr/bin/env bash |
|
2 |
|
3 |
|
4 stderr() { |
|
5 echo "$@" 1>&2 |
|
6 } |
|
7 |
|
8 usage() { |
|
9 b=$(basename "$0") |
|
10 echo $b: ERROR: "$@" 1>&2 |
|
11 |
|
12 cat 1>&2 <<EOF |
|
13 |
|
14 DESCRIPTION |
|
15 |
|
16 $(basename "$0") is the script to run continuous integration commands for |
|
17 go-toml on unix. |
|
18 |
|
19 Requires Go and Git to be available in the PATH. Expects to be ran from the |
|
20 root of go-toml's Git repository. |
|
21 |
|
22 USAGE |
|
23 |
|
24 $b COMMAND [OPTIONS...] |
|
25 |
|
26 COMMANDS |
|
27 |
|
28 benchmark [OPTIONS...] [BRANCH] |
|
29 |
|
30 Run benchmarks. |
|
31 |
|
32 ARGUMENTS |
|
33 |
|
34 BRANCH Optional. Defines which Git branch to use when running |
|
35 benchmarks. |
|
36 |
|
37 OPTIONS |
|
38 |
|
39 -d Compare benchmarks of HEAD with BRANCH using benchstats. In |
|
40 this form the BRANCH argument is required. |
|
41 |
|
42 -a Compare benchmarks of HEAD against go-toml v1 and |
|
43 BurntSushi/toml. |
|
44 |
|
45 -html When used with -a, emits the output as HTML, ready to be |
|
46 embedded in the README. |
|
47 |
|
48 coverage [OPTIONS...] [BRANCH] |
|
49 |
|
50 Generates code coverage. |
|
51 |
|
52 ARGUMENTS |
|
53 |
|
54 BRANCH Optional. Defines which Git branch to use when reporting |
|
55 coverage. Defaults to HEAD. |
|
56 |
|
57 OPTIONS |
|
58 |
|
59 -d Compare coverage of HEAD with the one of BRANCH. In this form, |
|
60 the BRANCH argument is required. Exit code is non-zero when |
|
61 coverage percentage decreased. |
|
62 EOF |
|
63 exit 1 |
|
64 } |
|
65 |
|
66 cover() { |
|
67 branch="${1}" |
|
68 dir="$(mktemp -d)" |
|
69 |
|
70 stderr "Executing coverage for ${branch} at ${dir}" |
|
71 |
|
72 if [ "${branch}" = "HEAD" ]; then |
|
73 cp -r . "${dir}/" |
|
74 else |
|
75 git worktree add "$dir" "$branch" |
|
76 fi |
|
77 |
|
78 pushd "$dir" |
|
79 go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./... |
|
80 cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out |
|
81 go tool cover -func=coverage.out |
|
82 popd |
|
83 |
|
84 if [ "${branch}" != "HEAD" ]; then |
|
85 git worktree remove --force "$dir" |
|
86 fi |
|
87 } |
|
88 |
|
89 coverage() { |
|
90 case "$1" in |
|
91 -d) |
|
92 shift |
|
93 target="${1?Need to provide a target branch argument}" |
|
94 |
|
95 output_dir="$(mktemp -d)" |
|
96 target_out="${output_dir}/target.txt" |
|
97 head_out="${output_dir}/head.txt" |
|
98 |
|
99 cover "${target}" > "${target_out}" |
|
100 cover "HEAD" > "${head_out}" |
|
101 |
|
102 cat "${target_out}" |
|
103 cat "${head_out}" |
|
104 |
|
105 echo "" |
|
106 |
|
107 target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')" |
|
108 head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')" |
|
109 echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" |
|
110 |
|
111 delta_pct=$(echo "$head_pct - $target_pct" | bc -l) |
|
112 echo "Delta: ${delta_pct}" |
|
113 |
|
114 if [[ $delta_pct = \-* ]]; then |
|
115 echo "Regression!"; |
|
116 |
|
117 target_diff="${output_dir}/target.diff.txt" |
|
118 head_diff="${output_dir}/head.diff.txt" |
|
119 cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}" |
|
120 cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}" |
|
121 |
|
122 diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}" |
|
123 return 1 |
|
124 fi |
|
125 return 0 |
|
126 ;; |
|
127 esac |
|
128 |
|
129 cover "${1-HEAD}" |
|
130 } |
|
131 |
|
132 bench() { |
|
133 branch="${1}" |
|
134 out="${2}" |
|
135 replace="${3}" |
|
136 dir="$(mktemp -d)" |
|
137 |
|
138 stderr "Executing benchmark for ${branch} at ${dir}" |
|
139 |
|
140 if [ "${branch}" = "HEAD" ]; then |
|
141 cp -r . "${dir}/" |
|
142 else |
|
143 git worktree add "$dir" "$branch" |
|
144 fi |
|
145 |
|
146 pushd "$dir" |
|
147 |
|
148 if [ "${replace}" != "" ]; then |
|
149 find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \; |
|
150 go get "${replace}" |
|
151 fi |
|
152 |
|
153 export GOMAXPROCS=2 |
|
154 nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}" |
|
155 popd |
|
156 |
|
157 if [ "${branch}" != "HEAD" ]; then |
|
158 git worktree remove --force "$dir" |
|
159 fi |
|
160 } |
|
161 |
|
162 fmktemp() { |
|
163 if mktemp --version|grep GNU >/dev/null; then |
|
164 mktemp --suffix=-$1; |
|
165 else |
|
166 mktemp -t $1; |
|
167 fi |
|
168 } |
|
169 |
|
170 benchstathtml() { |
|
171 python3 - $1 <<'EOF' |
|
172 import sys |
|
173 |
|
174 lines = [] |
|
175 stop = False |
|
176 |
|
177 with open(sys.argv[1]) as f: |
|
178 for line in f.readlines(): |
|
179 line = line.strip() |
|
180 if line == "": |
|
181 stop = True |
|
182 if not stop: |
|
183 lines.append(line.split(',')) |
|
184 |
|
185 results = [] |
|
186 for line in reversed(lines[1:]): |
|
187 v2 = float(line[1]) |
|
188 results.append([ |
|
189 line[0].replace("-32", ""), |
|
190 "%.1fx" % (float(line[3])/v2), # v1 |
|
191 "%.1fx" % (float(line[5])/v2), # bs |
|
192 ]) |
|
193 # move geomean to the end |
|
194 results.append(results[0]) |
|
195 del results[0] |
|
196 |
|
197 |
|
198 def printtable(data): |
|
199 print(""" |
|
200 <table> |
|
201 <thead> |
|
202 <tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr> |
|
203 </thead> |
|
204 <tbody>""") |
|
205 |
|
206 for r in data: |
|
207 print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r)) |
|
208 |
|
209 print(""" </tbody> |
|
210 </table>""") |
|
211 |
|
212 |
|
213 def match(x): |
|
214 return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0] |
|
215 |
|
216 above = [x for x in results if match(x)] |
|
217 below = [x for x in results if not match(x)] |
|
218 |
|
219 printtable(above) |
|
220 print("<details><summary>See more</summary>") |
|
221 print("""<p>The table above has the results of the most common use-cases. The table below |
|
222 contains the results of all benchmarks, including unrealistic ones. It is |
|
223 provided for completeness.</p>""") |
|
224 printtable(below) |
|
225 print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>') |
|
226 print("</details>") |
|
227 |
|
228 EOF |
|
229 } |
|
230 |
|
231 benchmark() { |
|
232 case "$1" in |
|
233 -d) |
|
234 shift |
|
235 target="${1?Need to provide a target branch argument}" |
|
236 |
|
237 old=`fmktemp ${target}` |
|
238 bench "${target}" "${old}" |
|
239 |
|
240 new=`fmktemp HEAD` |
|
241 bench HEAD "${new}" |
|
242 |
|
243 benchstat "${old}" "${new}" |
|
244 return 0 |
|
245 ;; |
|
246 -a) |
|
247 shift |
|
248 |
|
249 v2stats=`fmktemp go-toml-v2` |
|
250 bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2" |
|
251 v1stats=`fmktemp go-toml-v1` |
|
252 bench HEAD "${v1stats}" "github.com/pelletier/go-toml" |
|
253 bsstats=`fmktemp bs-toml` |
|
254 bench HEAD "${bsstats}" "github.com/BurntSushi/toml" |
|
255 |
|
256 cp "${v2stats}" go-toml-v2.txt |
|
257 cp "${v1stats}" go-toml-v1.txt |
|
258 cp "${bsstats}" bs-toml.txt |
|
259 |
|
260 if [ "$1" = "-html" ]; then |
|
261 tmpcsv=`fmktemp csv` |
|
262 benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv |
|
263 benchstathtml $tmpcsv |
|
264 else |
|
265 benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt |
|
266 fi |
|
267 |
|
268 rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt |
|
269 return $? |
|
270 esac |
|
271 |
|
272 bench "${1-HEAD}" `mktemp` |
|
273 } |
|
274 |
|
275 case "$1" in |
|
276 coverage) shift; coverage $@;; |
|
277 benchmark) shift; benchmark $@;; |
|
278 *) usage "bad argument $1";; |
|
279 esac |