|
1 #!/bin/bash |
|
2 |
|
3 POLYGLOT=1--[===[ |
|
4 |
|
5 set -o pipefail |
|
6 |
|
7 if [[ "$#" == "0" ]]; then |
|
8 echo "Lua mutation testing tool" |
|
9 echo |
|
10 echo "Usage:" |
|
11 echo " $BASH_SOURCE MODULE_NAME SPEC_FILE" |
|
12 echo |
|
13 echo "Requires 'lua', 'ltokenp' and 'busted' in PATH" |
|
14 exit 1; |
|
15 fi |
|
16 |
|
17 MOD_NAME="$1" |
|
18 MOD_FILE="$(lua "$BASH_SOURCE" resolve "$MOD_NAME")" |
|
19 |
|
20 if [[ "$MOD_FILE" == "" || ! -f "$MOD_FILE" ]]; then |
|
21 echo "EE: Failed to locate module '$MOD_NAME' ($MOD_FILE)"; |
|
22 exit 1; |
|
23 fi |
|
24 |
|
25 SPEC_FILE="$2" |
|
26 |
|
27 if [[ "$SPEC_FILE" == "" ]]; then |
|
28 SPEC_FILE="spec/${MOD_NAME/./_}_spec.lua" |
|
29 fi |
|
30 |
|
31 if [[ "$SPEC_FILE" == "" || ! -f "$SPEC_FILE" ]]; then |
|
32 echo "EE: Failed to find test spec file ($SPEC_FILE)" |
|
33 exit 1; |
|
34 fi |
|
35 |
|
36 if ! busted "$SPEC_FILE"; then |
|
37 echo "EE: Tests fail on original source. Fix it"\!; |
|
38 exit 1; |
|
39 fi |
|
40 |
|
41 export MUTANT_N=0 |
|
42 LIVING_MUTANTS=0 |
|
43 |
|
44 FILE_PREFIX="${MOD_FILE%.*}.mutant-" |
|
45 FILE_SUFFIX=".${MOD_FILE##*.}" |
|
46 |
|
47 gen_mutant () { |
|
48 echo "Generating mutant $2 to $3..." |
|
49 ltokenp -s "$BASH_SOURCE" "$1" > "$3" |
|
50 return "$?" |
|
51 } |
|
52 |
|
53 # $1 = MOD_NAME, $2 = MUTANT_N, $3 = SPEC_FILE |
|
54 test_mutant () { |
|
55 ( |
|
56 ulimit -m 131072 # 128MB |
|
57 ulimit -t 16 # 16s |
|
58 ulimit -f 32768 # 128MB (?) |
|
59 exec busted --helper="$BASH_SOURCE" -Xhelper mutate="$1":"$2" "$3" |
|
60 ) >/dev/null |
|
61 return "$?"; |
|
62 } |
|
63 |
|
64 MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}" |
|
65 |
|
66 gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE" |
|
67 while [[ "$?" == "0" ]]; do |
|
68 if ! test_mutant "$MOD_NAME" "$MUTANT_N" "$SPEC_FILE"; then |
|
69 echo "Tests successfully killed mutant $MUTANT_N"; |
|
70 rm "$MUTANT_FILE"; |
|
71 else |
|
72 echo "Mutant $MUTANT_N lives on"\! |
|
73 LIVING_MUTANTS=$((LIVING_MUTANTS+1)) |
|
74 fi |
|
75 MUTANT_N=$((MUTANT_N+1)) |
|
76 MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}" |
|
77 gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE" |
|
78 done |
|
79 |
|
80 if [[ "$?" != "2" ]]; then |
|
81 echo "Failed: $?" |
|
82 exit "$?"; |
|
83 fi |
|
84 |
|
85 MUTANT_SCORE="$(lua -e "print(('%0.2f'):format((1-($LIVING_MUTANTS/$MUTANT_N))*100))")" |
|
86 if test -f mutant-scores.txt; then |
|
87 echo "$MOD_NAME $MUTANT_SCORE" >> mutant-scores.txt |
|
88 fi |
|
89 echo "$MOD_NAME: All $MUTANT_N mutants generated, $LIVING_MUTANTS survived (score: $MUTANT_SCORE%)" |
|
90 rm "$MUTANT_FILE"; # Last file is always unmodified |
|
91 exit 0; |
|
92 ]===] |
|
93 |
|
94 -- busted helper that runs mutations |
|
95 if arg then |
|
96 if arg[1] == "resolve" then |
|
97 local filename = package.searchpath(assert(arg[2], "no module name given"), package.path); |
|
98 if filename then |
|
99 print(filename); |
|
100 end |
|
101 os.exit(filename and 0 or 1); |
|
102 end |
|
103 local mutants = {}; |
|
104 |
|
105 for i = 1, #arg do |
|
106 local opt = arg[i]; |
|
107 print("LOAD", i, opt) |
|
108 local module_name, mutant_n = opt:match("^mutate=([^:]+):(%d+)"); |
|
109 if module_name then |
|
110 mutants[module_name] = tonumber(mutant_n); |
|
111 end |
|
112 end |
|
113 |
|
114 local orig_lua_searcher = package.searchers[2]; |
|
115 |
|
116 local function mutant_searcher(module_name) |
|
117 local mutant_n = mutants[module_name]; |
|
118 if not mutant_n then |
|
119 return orig_lua_searcher(module_name); |
|
120 end |
|
121 local base_file, err = package.searchpath(module_name, package.path); |
|
122 if not base_file then |
|
123 return base_file, err; |
|
124 end |
|
125 local mutant_file = base_file:gsub("%.lua$", (".mutant-%d.lua"):format(mutant_n)); |
|
126 return loadfile(mutant_file), mutant_file; |
|
127 end |
|
128 |
|
129 if next(mutants) then |
|
130 table.insert(package.searchers, 1, mutant_searcher); |
|
131 end |
|
132 end |
|
133 |
|
134 -- filter for ltokenp to mutate scripts |
|
135 do |
|
136 local last_output = {}; |
|
137 local function emit(...) |
|
138 last_output = {...}; |
|
139 io.write(...) |
|
140 io.write(" ") |
|
141 return true; |
|
142 end |
|
143 |
|
144 local did_mutate = false; |
|
145 local count = -1; |
|
146 local threshold = tonumber(os.getenv("MUTANT_N")) or 0; |
|
147 local function should_mutate() |
|
148 count = count + 1; |
|
149 return count == threshold; |
|
150 end |
|
151 |
|
152 local function mutate(name, value) |
|
153 if name == "if" then |
|
154 -- Bypass conditionals |
|
155 if should_mutate() then |
|
156 return emit("if true or"); |
|
157 elseif should_mutate() then |
|
158 return emit("if false and"); |
|
159 end |
|
160 elseif name == "<integer>" then |
|
161 -- Introduce off-by-one errors |
|
162 if should_mutate() then |
|
163 return emit(("%d"):format(tonumber(value)+1)); |
|
164 elseif should_mutate() then |
|
165 return emit(("%d"):format(tonumber(value)-1)); |
|
166 end |
|
167 elseif name == "and" then |
|
168 if should_mutate() then |
|
169 return emit("or"); |
|
170 end |
|
171 elseif name == "or" then |
|
172 if should_mutate() then |
|
173 return emit("and"); |
|
174 end |
|
175 end |
|
176 end |
|
177 |
|
178 local current_line_n, current_line_input, current_line_output = 0, {}, {}; |
|
179 function FILTER(line_n,token,name,value) |
|
180 if current_line_n ~= line_n then -- Finished a line, moving to the next? |
|
181 if did_mutate and did_mutate.line == current_line_n then |
|
182 -- The line we finished was mutated. Store the original and modified outputs. |
|
183 did_mutate.line_original_src = table.concat(current_line_input, " "); |
|
184 did_mutate.line_modified_src = table.concat(current_line_output, " "); |
|
185 end |
|
186 current_line_input = {}; |
|
187 current_line_output = {}; |
|
188 end |
|
189 current_line_n = line_n; |
|
190 if name == "<file>" then return; end |
|
191 if name == "<eof>" then |
|
192 if not did_mutate then |
|
193 return os.exit(2); |
|
194 else |
|
195 emit(("\n-- Mutated line %d (changed '%s' to '%s'):\n"):format(did_mutate.line, did_mutate.original, did_mutate.modified)) |
|
196 emit( ("-- Original: %s\n"):format(did_mutate.line_original_src)) |
|
197 emit( ("-- Modified: %s\n"):format(did_mutate.line_modified_src)); |
|
198 return; |
|
199 end |
|
200 end |
|
201 if name == "<string>" then |
|
202 value = string.format("%q",value); |
|
203 end |
|
204 if mutate(name, value) then |
|
205 did_mutate = { |
|
206 original = value; |
|
207 modified = table.concat(last_output); |
|
208 line = line_n; |
|
209 }; |
|
210 else |
|
211 emit(value); |
|
212 end |
|
213 table.insert(current_line_input, value); |
|
214 table.insert(current_line_output, table.concat(last_output)); |
|
215 end |
|
216 end |
|
217 |