Fuzz frequency tuning (#2806)

We had some ad-hoc tuning of which nodes to emit more
frequently in the fuzzer, but it wasn't very good. Things like
loads and stores for example were far too rare. Also it wasn't
easy to adjust the frequencies.

This adds a simple way to adjust them, by passing a size_t
which is the "weight" of that node. Then it just makes that
number of copies of it, making it more likely to be picked.

Example output comparison:

node             before    after
================================
binary           281       365
block            898       649
break            278       144
call             182       290
call_indirect    9         42
const            808       854
drop             43        92
global.get       440       398
global.set       223       171
if               335       254
load             22        84
local.get        429       301
local.set        434       211
loop             176       99
nop              117       54
return           264       197
select           8         33
store            1         39
unary            405       304
unreachable      1         2

Lots of noise here obviously, but there are large increases
for loads and stores compared to before.

Also add a testcase of random data of the typical size the
fuzzer runs, and print metrics on it. This might help us get
a feel for how future tuning changes affect frequencies.

1 files changed, 69 insertions, 97 deletions

diff --git a/src/tools/fuzzing.h b/src/tools/fuzzing.h
index fb83882c2..f4188eb08 100644
--- a/src/tools/fuzzing.h
+++ b/src/tools/fuzzing.h
@@ -845,6 +845,11 @@ private:
     return std::string("label$") + std::to_string(labelIndex++);
   }
 
+  // Weighting for the core make* methods. Some nodes are important enough that
+  // we should do them quite often.
+  static const size_t VeryImportant = 4;
+  static const size_t Important = 2;
+
   // always call the toplevel make(type) command, not the internal specific ones
 
   int nesting = 0;
@@ -884,57 +889,37 @@ private:
     nesting--;
     return ret;
   }
+
   Expression* _makeConcrete(Type type) {
     bool canMakeControlFlow =
       !type.isMulti() || wasm.features.has(FeatureSet::Multivalue);
-    auto choice = upTo(100);
-    if (choice < 10) {
-      return makeConst(type);
-    }
-    if (choice < 30) {
-      return makeLocalSet(type);
-    }
-    if (choice < 50) {
-      return makeLocalGet(type);
-    }
-    if (choice < 60 && canMakeControlFlow) {
-      return makeBlock(type);
-    }
-    if (choice < 70 && canMakeControlFlow) {
-      return makeIf(type);
-    }
-    if (choice < 80 && canMakeControlFlow) {
-      return makeLoop(type);
-    }
-    if (choice < 90 && canMakeControlFlow) {
-      return makeBreak(type);
-    }
     using Self = TranslateToFuzzReader;
     FeatureOptions<Expression* (Self::*)(Type)> options;
+    using WeightedOption = decltype(options)::WeightedOption;
     options.add(FeatureSet::MVP,
-                &Self::makeLocalGet,
-                &Self::makeLocalSet,
-                &Self::makeGlobalGet,
-                &Self::makeConst);
+                WeightedOption{&Self::makeLocalGet, VeryImportant},
+                WeightedOption{&Self::makeLocalSet, VeryImportant},
+                WeightedOption{&Self::makeGlobalGet, Important},
+                WeightedOption{&Self::makeConst, Important});
     if (canMakeControlFlow) {
       options.add(FeatureSet::MVP,
-                  &Self::makeBlock,
-                  &Self::makeIf,
-                  &Self::makeLoop,
-                  &Self::makeBreak,
+                  WeightedOption{&Self::makeBlock, Important},
+                  WeightedOption{&Self::makeIf, Important},
+                  WeightedOption{&Self::makeLoop, Important},
+                  WeightedOption{&Self::makeBreak, Important},
                   &Self::makeCall,
                   &Self::makeCallIndirect);
     }
     if (type.isSingle()) {
       options
         .add(FeatureSet::MVP,
-             &Self::makeUnary,
-             &Self::makeBinary,
+             WeightedOption{&Self::makeUnary, Important},
+             WeightedOption{&Self::makeBinary, Important},
              &Self::makeSelect)
         .add(FeatureSet::Multivalue, &Self::makeTupleExtract);
     }
     if (type.isSingle() && !type.isRef()) {
-      options.add(FeatureSet::MVP, &Self::makeLoad);
+      options.add(FeatureSet::MVP, {&Self::makeLoad, Important});
       options.add(FeatureSet::SIMD, &Self::makeSIMD);
     }
     if (type.isInteger()) {
@@ -958,75 +943,48 @@ private:
         return makeMemoryHashLogging();
       }
     }
-    choice = upTo(100);
-    if (choice < 50) {
-      return makeLocalSet(Type::none);
-    }
-    if (choice < 60) {
-      return makeBlock(Type::none);
-    }
-    if (choice < 70) {
-      return makeIf(Type::none);
-    }
-    if (choice < 80) {
-      return makeLoop(Type::none);
-    }
-    if (choice < 90) {
-      return makeBreak(Type::none);
-    }
     using Self = TranslateToFuzzReader;
-    auto options = FeatureOptions<Expression* (Self::*)(Type)>()
-                     .add(FeatureSet::MVP,
-                          &Self::makeBlock,
-                          &Self::makeIf,
-                          &Self::makeLoop,
-                          &Self::makeBreak,
-                          &Self::makeCall,
-                          &Self::makeCallIndirect,
-                          &Self::makeLocalSet,
-                          &Self::makeStore,
-                          &Self::makeDrop,
-                          &Self::makeNop,
-                          &Self::makeGlobalSet)
-                     .add(FeatureSet::BulkMemory, &Self::makeBulkMemory)
-                     .add(FeatureSet::Atomics, &Self::makeAtomic);
+    auto options = FeatureOptions<Expression* (Self::*)(Type)>();
+    using WeightedOption = decltype(options)::WeightedOption;
+    options
+      .add(FeatureSet::MVP,
+           WeightedOption{&Self::makeLocalSet, VeryImportant},
+           WeightedOption{&Self::makeBlock, Important},
+           WeightedOption{&Self::makeIf, Important},
+           WeightedOption{&Self::makeLoop, Important},
+           WeightedOption{&Self::makeBreak, Important},
+           WeightedOption{&Self::makeStore, Important},
+           &Self::makeCall,
+           &Self::makeCallIndirect,
+           &Self::makeDrop,
+           &Self::makeNop,
+           &Self::makeGlobalSet)
+      .add(FeatureSet::BulkMemory, &Self::makeBulkMemory)
+      .add(FeatureSet::Atomics, &Self::makeAtomic);
     return (this->*pick(options))(Type::none);
   }
 
   Expression* _makeunreachable() {
-    switch (upTo(15)) {
-      case 0:
-        return makeBlock(Type::unreachable);
-      case 1:
-        return makeIf(Type::unreachable);
-      case 2:
-        return makeLoop(Type::unreachable);
-      case 3:
-        return makeBreak(Type::unreachable);
-      case 4:
-        return makeCall(Type::unreachable);
-      case 5:
-        return makeCallIndirect(Type::unreachable);
-      case 6:
-        return makeLocalSet(Type::unreachable);
-      case 7:
-        return makeStore(Type::unreachable);
-      case 8:
-        return makeUnary(Type::unreachable);
-      case 9:
-        return makeBinary(Type::unreachable);
-      case 10:
-        return makeSelect(Type::unreachable);
-      case 11:
-        return makeSwitch(Type::unreachable);
-      case 12:
-        return makeDrop(Type::unreachable);
-      case 13:
-        return makeReturn(Type::unreachable);
-      case 14:
-        return makeUnreachable(Type::unreachable);
-    }
-    WASM_UNREACHABLE("unexpected value");
+    using Self = TranslateToFuzzReader;
+    auto options = FeatureOptions<Expression* (Self::*)(Type)>();
+    using WeightedOption = decltype(options)::WeightedOption;
+    options.add(FeatureSet::MVP,
+                WeightedOption{&Self::makeLocalSet, VeryImportant},
+                WeightedOption{&Self::makeBlock, Important},
+                WeightedOption{&Self::makeIf, Important},
+                WeightedOption{&Self::makeLoop, Important},
+                WeightedOption{&Self::makeBreak, Important},
+                WeightedOption{&Self::makeStore, Important},
+                WeightedOption{&Self::makeUnary, Important},
+                WeightedOption{&Self::makeBinary, Important},
+                WeightedOption{&Self::makeUnreachable, Important},
+                &Self::makeCall,
+                &Self::makeCallIndirect,
+                &Self::makeSelect,
+                &Self::makeSwitch,
+                &Self::makeDrop,
+                &Self::makeReturn);
+    return (this->*pick(options))(Type::unreachable);
   }
 
   // make something with no chance of infinite recursion
@@ -2891,6 +2849,20 @@ private:
       return add(feature, rest...);
     }
 
+    struct WeightedOption {
+      T option;
+      size_t weight;
+    };
+
+    template<typename... Ts>
+    FeatureOptions<T>&
+    add(FeatureSet feature, WeightedOption weightedOption, Ts... rest) {
+      for (size_t i = 0; i < weightedOption.weight; i++) {
+        options[feature].push_back(weightedOption.option);
+      }
+      return add(feature, rest...);
+    }
+
     FeatureOptions<T>& add(FeatureSet feature) { return *this; }
 
     std::map<FeatureSet, std::vector<T>> options;