summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xscripts/clusterfuzz/run.py9
-rwxr-xr-xscripts/fuzz_opt.py46
-rw-r--r--scripts/fuzz_shell.js79
-rw-r--r--src/tools/execution-results.h3
-rw-r--r--src/tools/fuzzing.h3
-rw-r--r--src/tools/fuzzing/fuzzing.cpp29
-rw-r--r--test/lit/exec/fuzzing-api.wast19
-rw-r--r--test/passes/fuzz_metrics_noprint.bin.txt52
-rw-r--r--test/passes/fuzz_metrics_passes_noprint.bin.txt53
-rw-r--r--test/passes/translate-to-fuzz_all-features_metrics_noprint.txt83
-rw-r--r--test/unit/test_cluster_fuzz.py22
11 files changed, 279 insertions, 119 deletions
diff --git a/scripts/clusterfuzz/run.py b/scripts/clusterfuzz/run.py
index 8ac880e0d..2fedb6510 100755
--- a/scripts/clusterfuzz/run.py
+++ b/scripts/clusterfuzz/run.py
@@ -200,6 +200,15 @@ def get_js_file_contents(i, output_dir):
print(f'Created {bytes} wasm bytes')
+ # Some of the time, fuzz JSPI (similar to fuzz_opt.py, see details there).
+ if system_random.random() < 0.25:
+ # Prepend the flag to enable JSPI.
+ js = 'var JSPI = 1;\n\n' + js
+
+ # Un-comment the async and await keywords.
+ js = js.replace('/* async */', 'async')
+ js = js.replace('/* await */', 'await')
+
return js
diff --git a/scripts/fuzz_opt.py b/scripts/fuzz_opt.py
index 87f059058..ca7c9e355 100755
--- a/scripts/fuzz_opt.py
+++ b/scripts/fuzz_opt.py
@@ -232,7 +232,11 @@ def randomize_fuzz_settings():
if random.random() < 0.5:
GEN_ARGS += ['--enclose-world']
- print('randomized settings (NaNs, OOB, legalize):', NANS, OOB, LEGALIZE)
+ # Test JSPI somewhat rarely, as it may be slower.
+ global JSPI
+ JSPI = random.random() < 0.25
+
+ print('randomized settings (NaNs, OOB, legalize, JSPI):', NANS, OOB, LEGALIZE, JSPI)
def init_important_initial_contents():
@@ -758,11 +762,39 @@ def run_d8_js(js, args=[], liftoff=True):
return run_vm(cmd)
-FUZZ_SHELL_JS = in_binaryen('scripts', 'fuzz_shell.js')
+# For JSPI, we must customize fuzz_shell.js. We do so the first time we need
+# it, and save the filename here.
+JSPI_JS_FILE = None
+
+
+def get_fuzz_shell_js():
+ js = in_binaryen('scripts', 'fuzz_shell.js')
+
+ if not JSPI:
+ # Just use the normal fuzz shell script.
+ return js
+
+ global JSPI_JS_FILE
+ if JSPI_JS_FILE:
+ # Use the customized file we've already created.
+ return JSPI_JS_FILE
+
+ JSPI_JS_FILE = os.path.abspath('jspi_fuzz_shell.js')
+ with open(JSPI_JS_FILE, 'w') as f:
+ # Enable JSPI.
+ f.write('var JSPI = 1;\n\n')
+
+ # Un-comment the async and await keywords.
+ with open(js) as g:
+ code = g.read()
+ code = code.replace('/* async */', 'async')
+ code = code.replace('/* await */', 'await')
+ f.write(code)
+ return JSPI_JS_FILE
def run_d8_wasm(wasm, liftoff=True, args=[]):
- return run_d8_js(FUZZ_SHELL_JS, [wasm] + args, liftoff=liftoff)
+ return run_d8_js(get_fuzz_shell_js(), [wasm] + args, liftoff=liftoff)
def all_disallowed(features):
@@ -850,7 +882,7 @@ class CompareVMs(TestCaseHandler):
name = 'd8'
def run(self, wasm, extra_d8_flags=[]):
- return run_vm([shared.V8, FUZZ_SHELL_JS] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])
+ return run_vm([shared.V8, get_fuzz_shell_js()] + shared.V8_OPTS + get_v8_extra_flags() + extra_d8_flags + ['--', wasm])
def can_run(self, wasm):
# V8 does not support shared memories when running with
@@ -1160,7 +1192,7 @@ class Wasm2JS(TestCaseHandler):
compare_between_vms(before, interpreter, 'Wasm2JS (vs interpreter)')
def run(self, wasm):
- with open(FUZZ_SHELL_JS) as f:
+ with open(get_fuzz_shell_js()) as f:
wrapper = f.read()
cmd = [in_bin('wasm2js'), wasm, '--emscripten']
# avoid optimizations if we have nans, as we don't handle them with
@@ -1193,6 +1225,10 @@ class Wasm2JS(TestCaseHandler):
# specifically for growth here
if INITIAL_CONTENTS:
return False
+ # We run in node, which lacks JSPI support, and also we need wasm2js to
+ # implement wasm suspending using JS async/await.
+ if JSPI:
+ return False
return all_disallowed(['exception-handling', 'simd', 'threads', 'bulk-memory', 'nontrapping-float-to-int', 'tail-call', 'sign-ext', 'reference-types', 'multivalue', 'gc', 'multimemory', 'memory64'])
diff --git a/scripts/fuzz_shell.js b/scripts/fuzz_shell.js
index 95176bbe6..653146517 100644
--- a/scripts/fuzz_shell.js
+++ b/scripts/fuzz_shell.js
@@ -1,3 +1,17 @@
+// This script can be customized by setting the following variables in code that
+// runs before this script.
+//
+// The binary to be run. (If not set, we get the filename from argv and read
+// from it.)
+var binary;
+// A second binary to be linked in and run as well. (Can also be read from
+// argv.)
+var secondBinary;
+// Whether we are fuzzing JSPI. In addition to this being set, the "async" and
+// "await" keywords must be taken out of the /* KEYWORD */ comments (which they
+// are normally in, so as not to affect normal fuzzing).
+var JSPI;
+
// Shell integration: find argv and set up readBinary().
var argv;
var readBinary;
@@ -25,9 +39,6 @@ if (typeof process === 'object' && typeof require === 'function') {
};
}
-// The binary to be run. This may be set already (by code that runs before this
-// script), and if not, we get the filename from argv.
-var binary;
if (!binary) {
binary = readBinary(argv[0]);
}
@@ -43,7 +54,6 @@ if (argv.length > 0 && argv[argv.length - 1].startsWith('exports:')) {
// If a second parameter is given, it is a second binary that we will link in
// with it.
-var secondBinary;
if (argv[1]) {
secondBinary = readBinary(argv[1]);
}
@@ -163,9 +173,9 @@ function callFunc(func) {
// Calls a given function in a try-catch, swallowing JS exceptions, and return 1
// if we did in fact swallow an exception. Wasm traps are not swallowed (see
// details below).
-function tryCall(func) {
+/* async */ function tryCall(func) {
try {
- func();
+ /* await */ func();
return 0;
} catch (e) {
// We only want to catch exceptions, not wasm traps: traps should still
@@ -243,19 +253,39 @@ var imports = {
},
// Export operations.
- 'call-export': (index) => {
- callFunc(exportList[index].value);
+ 'call-export': /* async */ (index) => {
+ /* await */ callFunc(exportList[index].value);
},
- 'call-export-catch': (index) => {
- return tryCall(() => callFunc(exportList[index].value));
+ 'call-export-catch': /* async */ (index) => {
+ return tryCall(/* async */ () => /* await */ callFunc(exportList[index].value));
},
// Funcref operations.
- 'call-ref': (ref) => {
- callFunc(ref);
+ 'call-ref': /* async */ (ref) => {
+ // This is a direct function reference, and just like an export, it must
+ // be wrapped for JSPI.
+ ref = wrapExportForJSPI(ref);
+ /* await */ callFunc(ref);
+ },
+ 'call-ref-catch': /* async */ (ref) => {
+ ref = wrapExportForJSPI(ref);
+ return tryCall(/* async */ () => /* await */ callFunc(ref));
},
- 'call-ref-catch': (ref) => {
- return tryCall(() => callFunc(ref));
+
+ // Sleep a given amount of ms (when JSPI) and return a given id after that.
+ 'sleep': (ms, id) => {
+ if (!JSPI) {
+ return id;
+ }
+ return new Promise((resolve, reject) => {
+ setTimeout(() => {
+ resolve(id);
+ }, 0); // TODO: Use the ms in some reasonable, deterministic manner.
+ // Rather than actually setTimeout on them we could manage
+ // a queue of pending sleeps manually, and order them based
+ // on the "ms" (which would not be literal ms, but just
+ // how many time units to wait).
+ });
},
},
// Emscripten support.
@@ -274,6 +304,22 @@ if (typeof WebAssembly.Tag !== 'undefined') {
};
}
+// If JSPI is available, wrap the imports and exports.
+if (JSPI) {
+ for (var name of ['sleep', 'call-export', 'call-export-catch', 'call-ref',
+ 'call-ref-catch']) {
+ imports['fuzzing-support'][name] =
+ new WebAssembly.Suspending(imports['fuzzing-support'][name]);
+ }
+}
+
+function wrapExportForJSPI(value) {
+ if (JSPI && typeof value === 'function') {
+ value = WebAssembly.promising(value);
+ }
+ return value;
+}
+
// If a second binary will be linked in then set up the imports for
// placeholders. Any import like (import "placeholder" "0" (func .. will be
// provided by the secondary module, and must be called using an indirection.
@@ -312,13 +358,14 @@ function build(binary) {
// keep the ability to call anything that was ever exported.)
for (var key in instance.exports) {
var value = instance.exports[key];
+ value = wrapExportForJSPI(value);
exports[key] = value;
exportList.push({ name: key, value: value });
}
}
// Run the code by calling exports.
-function callExports() {
+/* async */ function callExports() {
// Call the exports we were told, or if we were not given an explicit list,
// call them all.
var relevantExports = exportsToCall || exportList;
@@ -342,7 +389,7 @@ function callExports() {
try {
console.log('[fuzz-exec] calling ' + name);
- var result = callFunc(value);
+ var result = /* await */ callFunc(value);
if (typeof result !== 'undefined') {
console.log('[fuzz-exec] note result: ' + name + ' => ' + printed(result));
}
diff --git a/src/tools/execution-results.h b/src/tools/execution-results.h
index b8823c3f0..ea822d547 100644
--- a/src/tools/execution-results.h
+++ b/src/tools/execution-results.h
@@ -128,6 +128,9 @@ public:
} catch (const WasmException& e) {
return {Literal(int32_t(1))};
}
+ } else if (import->base == "sleep") {
+ // Do not actually sleep, just return the id.
+ return {arguments[1]};
} else {
WASM_UNREACHABLE("unknown fuzzer import");
}
diff --git a/src/tools/fuzzing.h b/src/tools/fuzzing.h
index 78219045c..f76ed62a5 100644
--- a/src/tools/fuzzing.h
+++ b/src/tools/fuzzing.h
@@ -117,6 +117,7 @@ private:
Name callExportCatchImportName;
Name callRefImportName;
Name callRefCatchImportName;
+ Name sleepImportName;
std::unordered_map<Type, std::vector<Name>> globalsByType;
std::unordered_map<Type, std::vector<Name>> mutableGlobalsByType;
@@ -238,6 +239,7 @@ private:
void addImportCallingSupport();
void addImportThrowingSupport();
void addImportTableSupport();
+ void addImportSleepSupport();
void addHashMemorySupport();
// Special expression makers
@@ -249,6 +251,7 @@ private:
// Call either an export or a ref. We do this from a single function to better
// control the frequency of each.
Expression* makeImportCallCode(Type type);
+ Expression* makeImportSleep(Type type);
Expression* makeMemoryHashLogging();
// Function creation
diff --git a/src/tools/fuzzing/fuzzing.cpp b/src/tools/fuzzing/fuzzing.cpp
index a7f5e0d01..2b9286180 100644
--- a/src/tools/fuzzing/fuzzing.cpp
+++ b/src/tools/fuzzing/fuzzing.cpp
@@ -317,6 +317,7 @@ void TranslateToFuzzReader::build() {
}
addImportLoggingSupport();
addImportCallingSupport();
+ addImportSleepSupport();
modifyInitialFunctions();
// keep adding functions until we run out of input
while (!random.finished()) {
@@ -909,6 +910,24 @@ void TranslateToFuzzReader::addImportTableSupport() {
}
}
+void TranslateToFuzzReader::addImportSleepSupport() {
+ if (!oneIn(4)) {
+ // Fuzz this somewhat rarely, as it may be slow.
+ return;
+ }
+
+ // An import that sleeps for a given number of milliseconds, and also receives
+ // an integer id. It returns that integer id (useful for tracking separate
+ // sleeps).
+ sleepImportName = Names::getValidFunctionName(wasm, "sleep");
+ auto func = std::make_unique<Function>();
+ func->name = sleepImportName;
+ func->module = "fuzzing-support";
+ func->base = "sleep";
+ func->type = Signature({Type::i32, Type::i32}, Type::i32);
+ wasm.addFunction(std::move(func));
+}
+
void TranslateToFuzzReader::addHashMemorySupport() {
// Add memory hasher helper (for the hash, see hash.h). The function looks
// like:
@@ -1090,6 +1109,13 @@ Expression* TranslateToFuzzReader::makeImportCallCode(Type type) {
return builder.makeCall(exportTarget, {index}, type);
}
+Expression* TranslateToFuzzReader::makeImportSleep(Type type) {
+ // Sleep for some ms, and return a given id.
+ auto* ms = make(Type::i32);
+ auto id = make(Type::i32);
+ return builder.makeCall(sleepImportName, {ms, id}, Type::i32);
+}
+
Expression* TranslateToFuzzReader::makeMemoryHashLogging() {
auto* hash = builder.makeCall(std::string("hashMemory"), {}, Type::i32);
return builder.makeCall(logImportNames[Type::i32], {hash}, Type::none);
@@ -1768,6 +1794,9 @@ Expression* TranslateToFuzzReader::_makeConcrete(Type type) {
if (callExportCatchImportName || callRefCatchImportName) {
options.add(FeatureSet::MVP, &Self::makeImportCallCode);
}
+ if (sleepImportName) {
+ options.add(FeatureSet::MVP, &Self::makeImportSleep);
+ }
options.add(FeatureSet::ReferenceTypes, &Self::makeRefIsNull);
options.add(FeatureSet::ReferenceTypes | FeatureSet::GC,
&Self::makeRefEq,
diff --git a/test/lit/exec/fuzzing-api.wast b/test/lit/exec/fuzzing-api.wast
index 7c975cb75..8e251b2ed 100644
--- a/test/lit/exec/fuzzing-api.wast
+++ b/test/lit/exec/fuzzing-api.wast
@@ -19,6 +19,8 @@
(import "fuzzing-support" "call-ref" (func $call.ref (param funcref)))
(import "fuzzing-support" "call-ref-catch" (func $call.ref.catch (param funcref) (result i32)))
+ (import "fuzzing-support" "sleep" (func $sleep (param i32 i32) (result i32)))
+
(table $table 10 20 funcref)
;; Note that the exported table appears first here, but in the binary and in
@@ -284,7 +286,6 @@
;; CHECK: [fuzz-exec] calling ref.calling.trap
;; CHECK-NEXT: [trap unreachable]
- ;; CHECK-NEXT: warning: no passes specified, not doing any work
(func $ref.calling.trap (export "ref.calling.trap")
;; We try to catch an exception here, but the target function traps, which is
;; not something we can catch. We will trap here, and not log at all.
@@ -294,6 +295,18 @@
)
)
)
+
+ ;; CHECK: [fuzz-exec] calling do-sleep
+ ;; CHECK-NEXT: [fuzz-exec] note result: do-sleep => 42
+ ;; CHECK-NEXT: warning: no passes specified, not doing any work
+ (func $do-sleep (export "do-sleep") (result i32)
+ (call $sleep
+ ;; A ridiculous amount of ms, but in the interpreter it is ignored anyhow.
+ (i32.const -1)
+ ;; An id, that is returned back to us.
+ (i32.const 42)
+ )
+ )
)
;; CHECK: [fuzz-exec] calling logging
;; CHECK-NEXT: [LoggingExternalInterface logging 42]
@@ -354,6 +367,10 @@
;; CHECK: [fuzz-exec] calling ref.calling.trap
;; CHECK-NEXT: [trap unreachable]
+
+;; CHECK: [fuzz-exec] calling do-sleep
+;; CHECK-NEXT: [fuzz-exec] note result: do-sleep => 42
+;; CHECK-NEXT: [fuzz-exec] comparing do-sleep
;; CHECK-NEXT: [fuzz-exec] comparing export.calling
;; CHECK-NEXT: [fuzz-exec] comparing export.calling.catching
;; CHECK-NEXT: [fuzz-exec] comparing logging
diff --git a/test/passes/fuzz_metrics_noprint.bin.txt b/test/passes/fuzz_metrics_noprint.bin.txt
index a2f996bcb..bf7e517da 100644
--- a/test/passes/fuzz_metrics_noprint.bin.txt
+++ b/test/passes/fuzz_metrics_noprint.bin.txt
@@ -1,35 +1,35 @@
Metrics
total
- [exports] : 49
- [funcs] : 74
+ [exports] : 46
+ [funcs] : 68
[globals] : 18
[imports] : 4
[memories] : 1
[memory-data] : 24
- [table-data] : 19
+ [table-data] : 22
[tables] : 1
[tags] : 0
- [total] : 5695
- [vars] : 227
- Binary : 430
- Block : 976
- Break : 188
- Call : 272
- CallIndirect : 20
- Const : 876
- Drop : 99
- GlobalGet : 504
- GlobalSet : 373
- If : 299
- Load : 111
- LocalGet : 365
- LocalSet : 299
- Loop : 108
- Nop : 58
- RefFunc : 19
- Return : 74
- Select : 40
- Store : 34
+ [total] : 9465
+ [vars] : 215
+ Binary : 671
+ Block : 1531
+ Break : 370
+ Call : 366
+ CallIndirect : 67
+ Const : 1478
+ Drop : 111
+ GlobalGet : 766
+ GlobalSet : 558
+ If : 514
+ Load : 173
+ LocalGet : 729
+ LocalSet : 550
+ Loop : 202
+ Nop : 133
+ RefFunc : 22
+ Return : 99
+ Select : 84
+ Store : 83
Switch : 2
- Unary : 365
- Unreachable : 183
+ Unary : 682
+ Unreachable : 274
diff --git a/test/passes/fuzz_metrics_passes_noprint.bin.txt b/test/passes/fuzz_metrics_passes_noprint.bin.txt
index c3881104d..814fdb132 100644
--- a/test/passes/fuzz_metrics_passes_noprint.bin.txt
+++ b/test/passes/fuzz_metrics_passes_noprint.bin.txt
@@ -1,34 +1,35 @@
Metrics
total
- [exports] : 30
- [funcs] : 47
+ [exports] : 43
+ [funcs] : 56
[globals] : 17
[imports] : 4
[memories] : 1
[memory-data] : 11
- [table-data] : 18
+ [table-data] : 16
[tables] : 1
[tags] : 0
- [total] : 4738
- [vars] : 133
- Binary : 338
- Block : 781
- Break : 122
- Call : 249
- CallIndirect : 27
- Const : 780
- Drop : 105
- GlobalGet : 378
- GlobalSet : 288
- If : 216
- Load : 79
- LocalGet : 396
- LocalSet : 252
- Loop : 89
- Nop : 43
- RefFunc : 18
- Return : 70
- Select : 37
- Store : 36
- Unary : 294
- Unreachable : 140
+ [total] : 10611
+ [vars] : 184
+ Binary : 754
+ Block : 1699
+ Break : 397
+ Call : 325
+ CallIndirect : 112
+ Const : 1783
+ Drop : 101
+ GlobalGet : 869
+ GlobalSet : 657
+ If : 549
+ Load : 195
+ LocalGet : 893
+ LocalSet : 609
+ Loop : 251
+ Nop : 123
+ RefFunc : 16
+ Return : 78
+ Select : 74
+ Store : 84
+ Switch : 3
+ Unary : 730
+ Unreachable : 309
diff --git a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt
index e2e9b8053..3c2b7bfa2 100644
--- a/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt
+++ b/test/passes/translate-to-fuzz_all-features_metrics_noprint.txt
@@ -1,57 +1,52 @@
Metrics
total
- [exports] : 6
- [funcs] : 6
+ [exports] : 3
+ [funcs] : 3
[globals] : 4
[imports] : 8
[memories] : 1
[memory-data] : 112
- [table-data] : 1
+ [table-data] : 0
[tables] : 1
[tags] : 1
- [total] : 592
- [vars] : 38
- ArrayGet : 2
- ArrayLen : 2
- ArrayNew : 6
- ArrayNewFixed : 4
- ArraySet : 1
+ [total] : 630
+ [vars] : 23
+ ArrayNewFixed : 3
AtomicCmpxchg : 1
AtomicFence : 1
- AtomicRMW : 2
- Binary : 81
- Block : 62
- BrOn : 1
- Break : 11
- Call : 13
- CallIndirect : 2
- Const : 123
- Drop : 2
- GlobalGet : 22
- GlobalSet : 22
- If : 17
- Load : 25
- LocalGet : 63
- LocalSet : 35
- Loop : 6
- Nop : 8
- Pop : 3
- RefAs : 1
- RefEq : 1
- RefFunc : 6
- RefNull : 3
- Return : 4
- SIMDExtract : 2
- Select : 2
- StringConst : 5
+ AtomicNotify : 1
+ Binary : 63
+ Block : 60
+ BrOn : 3
+ Break : 8
+ Call : 4
+ CallRef : 3
+ Const : 129
+ DataDrop : 1
+ Drop : 8
+ GlobalGet : 21
+ GlobalSet : 20
+ I31Get : 1
+ If : 12
+ Load : 17
+ LocalGet : 74
+ LocalSet : 52
+ Loop : 8
+ MemoryFill : 1
+ Nop : 4
+ RefAs : 19
+ RefFunc : 26
+ RefI31 : 1
+ RefIsNull : 1
+ RefNull : 11
+ Return : 2
+ Select : 5
StringEncode : 1
- StringEq : 2
- StringMeasure : 1
- StringWTF16Get : 2
- StructNew : 9
- Try : 3
- TryTable : 3
+ StructGet : 3
+ StructNew : 26
+ Try : 1
+ TryTable : 6
TupleExtract : 1
TupleMake : 2
- Unary : 18
- Unreachable : 11
+ Unary : 20
+ Unreachable : 10
diff --git a/test/unit/test_cluster_fuzz.py b/test/unit/test_cluster_fuzz.py
index 56250d46a..8f1d18104 100644
--- a/test/unit/test_cluster_fuzz.py
+++ b/test/unit/test_cluster_fuzz.py
@@ -274,10 +274,11 @@ class ClusterFuzz(utils.BinaryenTestCase):
print()
# To check for interesting JS file contents, we'll note how many times
- # we build and run the wasm.
+ # we build and run the wasm, and other things like JSPI.
seen_builds = []
seen_calls = []
seen_second_builds = []
+ seen_JSPIs = []
for i in range(1, N + 1):
fuzz_file = os.path.join(temp_dir.name, f'fuzz-binaryen-{i}.js')
@@ -287,6 +288,17 @@ class ClusterFuzz(utils.BinaryenTestCase):
seen_calls.append(js.count('callExports();'))
seen_second_builds.append(js.count('build(secondBinary);'))
+ # If JSPI is enabled, the async and await keywords should be
+ # enabled (uncommented).
+ if 'JSPI = 1' in js:
+ seen_JSPIs.append(1)
+ assert '/* async */' not in js
+ assert '/* await */' not in js
+ else:
+ seen_JSPIs.append(0)
+ assert '/* async */' in js
+ assert '/* await */' in js
+
# There is always one build and one call (those are in the default
# fuzz_shell.js), and we add a couple of operations, each with equal
# probability to be a build or a call, so over the 100 testcases here we
@@ -323,6 +335,14 @@ class ClusterFuzz(utils.BinaryenTestCase):
print()
+ # JSPI is done 1/4 of the time or so.
+ print('JSPIs are distributed as ~ mean 0.25')
+ print(f'mean JSPIs: {statistics.mean(seen_JSPIs)}')
+ self.assertEqual(min(seen_JSPIs), 0)
+ self.assertEqual(max(seen_JSPIs), 1)
+
+ print()
+
# "zzz" in test name so that this runs last. If it runs first, it can be
# confusing as it appears next to the logging of which bundle we use (see
# setUpClass).