260
|
1 |
#!/usr/bin/env bash |
|
2 |
|
|
3 |
|
|
4 |
stderr() { |
|
5 |
echo "$@" 1>&2 |
|
6 |
} |
|
7 |
|
|
8 |
usage() { |
|
9 |
b=$(basename "$0") |
|
10 |
echo $b: ERROR: "$@" 1>&2 |
|
11 |
|
|
12 |
cat 1>&2 <<EOF |
|
13 |
|
|
14 |
DESCRIPTION |
|
15 |
|
|
16 |
$(basename "$0") is the script to run continuous integration commands for |
|
17 |
go-toml on unix. |
|
18 |
|
|
19 |
Requires Go and Git to be available in the PATH. Expects to be ran from the |
|
20 |
root of go-toml's Git repository. |
|
21 |
|
|
22 |
USAGE |
|
23 |
|
|
24 |
$b COMMAND [OPTIONS...] |
|
25 |
|
|
26 |
COMMANDS |
|
27 |
|
|
28 |
benchmark [OPTIONS...] [BRANCH] |
|
29 |
|
|
30 |
Run benchmarks. |
|
31 |
|
|
32 |
ARGUMENTS |
|
33 |
|
|
34 |
BRANCH Optional. Defines which Git branch to use when running |
|
35 |
benchmarks. |
|
36 |
|
|
37 |
OPTIONS |
|
38 |
|
|
39 |
-d Compare benchmarks of HEAD with BRANCH using benchstats. In |
|
40 |
this form the BRANCH argument is required. |
|
41 |
|
|
42 |
-a Compare benchmarks of HEAD against go-toml v1 and |
|
43 |
BurntSushi/toml. |
|
44 |
|
|
45 |
-html When used with -a, emits the output as HTML, ready to be |
|
46 |
embedded in the README. |
|
47 |
|
|
48 |
coverage [OPTIONS...] [BRANCH] |
|
49 |
|
|
50 |
Generates code coverage. |
|
51 |
|
|
52 |
ARGUMENTS |
|
53 |
|
|
54 |
BRANCH Optional. Defines which Git branch to use when reporting |
|
55 |
coverage. Defaults to HEAD. |
|
56 |
|
|
57 |
OPTIONS |
|
58 |
|
|
59 |
-d Compare coverage of HEAD with the one of BRANCH. In this form, |
|
60 |
the BRANCH argument is required. Exit code is non-zero when |
|
61 |
coverage percentage decreased. |
|
62 |
EOF |
|
63 |
exit 1 |
|
64 |
} |
|
65 |
|
|
66 |
cover() { |
|
67 |
branch="${1}" |
|
68 |
dir="$(mktemp -d)" |
|
69 |
|
|
70 |
stderr "Executing coverage for ${branch} at ${dir}" |
|
71 |
|
|
72 |
if [ "${branch}" = "HEAD" ]; then |
|
73 |
cp -r . "${dir}/" |
|
74 |
else |
|
75 |
git worktree add "$dir" "$branch" |
|
76 |
fi |
|
77 |
|
|
78 |
pushd "$dir" |
|
79 |
go test -covermode=atomic -coverpkg=./... -coverprofile=coverage.out.tmp ./... |
|
80 |
cat coverage.out.tmp | grep -v testsuite | grep -v tomltestgen | grep -v gotoml-test-decoder > coverage.out |
|
81 |
go tool cover -func=coverage.out |
|
82 |
popd |
|
83 |
|
|
84 |
if [ "${branch}" != "HEAD" ]; then |
|
85 |
git worktree remove --force "$dir" |
|
86 |
fi |
|
87 |
} |
|
88 |
|
|
89 |
coverage() { |
|
90 |
case "$1" in |
|
91 |
-d) |
|
92 |
shift |
|
93 |
target="${1?Need to provide a target branch argument}" |
|
94 |
|
|
95 |
output_dir="$(mktemp -d)" |
|
96 |
target_out="${output_dir}/target.txt" |
|
97 |
head_out="${output_dir}/head.txt" |
|
98 |
|
|
99 |
cover "${target}" > "${target_out}" |
|
100 |
cover "HEAD" > "${head_out}" |
|
101 |
|
|
102 |
cat "${target_out}" |
|
103 |
cat "${head_out}" |
|
104 |
|
|
105 |
echo "" |
|
106 |
|
|
107 |
target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')" |
|
108 |
head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')" |
|
109 |
echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" |
|
110 |
|
|
111 |
delta_pct=$(echo "$head_pct - $target_pct" | bc -l) |
|
112 |
echo "Delta: ${delta_pct}" |
|
113 |
|
|
114 |
if [[ $delta_pct = \-* ]]; then |
|
115 |
echo "Regression!"; |
|
116 |
|
|
117 |
target_diff="${output_dir}/target.diff.txt" |
|
118 |
head_diff="${output_dir}/head.diff.txt" |
|
119 |
cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}" |
|
120 |
cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}" |
|
121 |
|
|
122 |
diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}" |
|
123 |
return 1 |
|
124 |
fi |
|
125 |
return 0 |
|
126 |
;; |
|
127 |
esac |
|
128 |
|
|
129 |
cover "${1-HEAD}" |
|
130 |
} |
|
131 |
|
|
132 |
bench() { |
|
133 |
branch="${1}" |
|
134 |
out="${2}" |
|
135 |
replace="${3}" |
|
136 |
dir="$(mktemp -d)" |
|
137 |
|
|
138 |
stderr "Executing benchmark for ${branch} at ${dir}" |
|
139 |
|
|
140 |
if [ "${branch}" = "HEAD" ]; then |
|
141 |
cp -r . "${dir}/" |
|
142 |
else |
|
143 |
git worktree add "$dir" "$branch" |
|
144 |
fi |
|
145 |
|
|
146 |
pushd "$dir" |
|
147 |
|
|
148 |
if [ "${replace}" != "" ]; then |
|
149 |
find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \; |
|
150 |
go get "${replace}" |
|
151 |
fi |
|
152 |
|
|
153 |
export GOMAXPROCS=2 |
|
154 |
nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}" |
|
155 |
popd |
|
156 |
|
|
157 |
if [ "${branch}" != "HEAD" ]; then |
|
158 |
git worktree remove --force "$dir" |
|
159 |
fi |
|
160 |
} |
|
161 |
|
|
162 |
fmktemp() { |
|
163 |
if mktemp --version|grep GNU >/dev/null; then |
|
164 |
mktemp --suffix=-$1; |
|
165 |
else |
|
166 |
mktemp -t $1; |
|
167 |
fi |
|
168 |
} |
|
169 |
|
|
170 |
benchstathtml() { |
|
171 |
python3 - $1 <<'EOF' |
|
172 |
import sys |
|
173 |
|
|
174 |
lines = [] |
|
175 |
stop = False |
|
176 |
|
|
177 |
with open(sys.argv[1]) as f: |
|
178 |
for line in f.readlines(): |
|
179 |
line = line.strip() |
|
180 |
if line == "": |
|
181 |
stop = True |
|
182 |
if not stop: |
|
183 |
lines.append(line.split(',')) |
|
184 |
|
|
185 |
results = [] |
|
186 |
for line in reversed(lines[1:]): |
|
187 |
v2 = float(line[1]) |
|
188 |
results.append([ |
|
189 |
line[0].replace("-32", ""), |
|
190 |
"%.1fx" % (float(line[3])/v2), # v1 |
|
191 |
"%.1fx" % (float(line[5])/v2), # bs |
|
192 |
]) |
|
193 |
# move geomean to the end |
|
194 |
results.append(results[0]) |
|
195 |
del results[0] |
|
196 |
|
|
197 |
|
|
198 |
def printtable(data): |
|
199 |
print(""" |
|
200 |
<table> |
|
201 |
<thead> |
|
202 |
<tr><th>Benchmark</th><th>go-toml v1</th><th>BurntSushi/toml</th></tr> |
|
203 |
</thead> |
|
204 |
<tbody>""") |
|
205 |
|
|
206 |
for r in data: |
|
207 |
print(" <tr><td>{}</td><td>{}</td><td>{}</td></tr>".format(*r)) |
|
208 |
|
|
209 |
print(""" </tbody> |
|
210 |
</table>""") |
|
211 |
|
|
212 |
|
|
213 |
def match(x): |
|
214 |
return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0] |
|
215 |
|
|
216 |
above = [x for x in results if match(x)] |
|
217 |
below = [x for x in results if not match(x)] |
|
218 |
|
|
219 |
printtable(above) |
|
220 |
print("<details><summary>See more</summary>") |
|
221 |
print("""<p>The table above has the results of the most common use-cases. The table below |
|
222 |
contains the results of all benchmarks, including unrealistic ones. It is |
|
223 |
provided for completeness.</p>""") |
|
224 |
printtable(below) |
|
225 |
print('<p>This table can be generated with <code>./ci.sh benchmark -a -html</code>.</p>') |
|
226 |
print("</details>") |
|
227 |
|
|
228 |
EOF |
|
229 |
} |
|
230 |
|
|
231 |
benchmark() { |
|
232 |
case "$1" in |
|
233 |
-d) |
|
234 |
shift |
|
235 |
target="${1?Need to provide a target branch argument}" |
|
236 |
|
|
237 |
old=`fmktemp ${target}` |
|
238 |
bench "${target}" "${old}" |
|
239 |
|
|
240 |
new=`fmktemp HEAD` |
|
241 |
bench HEAD "${new}" |
|
242 |
|
|
243 |
benchstat "${old}" "${new}" |
|
244 |
return 0 |
|
245 |
;; |
|
246 |
-a) |
|
247 |
shift |
|
248 |
|
|
249 |
v2stats=`fmktemp go-toml-v2` |
|
250 |
bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2" |
|
251 |
v1stats=`fmktemp go-toml-v1` |
|
252 |
bench HEAD "${v1stats}" "github.com/pelletier/go-toml" |
|
253 |
bsstats=`fmktemp bs-toml` |
|
254 |
bench HEAD "${bsstats}" "github.com/BurntSushi/toml" |
|
255 |
|
|
256 |
cp "${v2stats}" go-toml-v2.txt |
|
257 |
cp "${v1stats}" go-toml-v1.txt |
|
258 |
cp "${bsstats}" bs-toml.txt |
|
259 |
|
|
260 |
if [ "$1" = "-html" ]; then |
|
261 |
tmpcsv=`fmktemp csv` |
|
262 |
benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv |
|
263 |
benchstathtml $tmpcsv |
|
264 |
else |
|
265 |
benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt |
|
266 |
fi |
|
267 |
|
|
268 |
rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt |
|
269 |
return $? |
|
270 |
esac |
|
271 |
|
|
272 |
bench "${1-HEAD}" `mktemp` |
|
273 |
} |
|
274 |
|
|
275 |
case "$1" in |
|
276 |
coverage) shift; coverage $@;; |
|
277 |
benchmark) shift; benchmark $@;; |
|
278 |
*) usage "bad argument $1";; |
|
279 |
esac |