[GC] Add br_on_func/data/i31 (#3525)

This expands the existing BrOnCast into BrOn that can also handle the
func/data/i31 variants. This is not as elegant as RefIs / RefAs in that BrOnCast
has an extra rtt field, but I think it is still the best option. We already have optional
fields on Break (the value and condition), so making rtt optional is not odd. And
it allows us to share all the behavior of br_on_* which aside from the cast or the
check itself, is identical - returning the value if the branch is not taken, etc.

20 files changed, 204 insertions, 70 deletions

diff --git a/src/gen-s-parser.inc b/src/gen-s-parser.inc
index 255a4bfa8..a22a01c02 100644
--- a/src/gen-s-parser.inc
+++ b/src/gen-s-parser.inc
@@ -71,9 +71,23 @@ switch (op[0]) {
               case 'i':
                 if (strcmp(op, "br_if") == 0) { return makeBreak(s); }
                 goto parse_error;
-              case 'o':
-                if (strcmp(op, "br_on_cast") == 0) { return makeBrOnCast(s); }
-                goto parse_error;
+              case 'o': {
+                switch (op[6]) {
+                  case 'c':
+                    if (strcmp(op, "br_on_cast") == 0) { return makeBrOn(s, BrOnCast); }
+                    goto parse_error;
+                  case 'd':
+                    if (strcmp(op, "br_on_data") == 0) { return makeBrOn(s, BrOnData); }
+                    goto parse_error;
+                  case 'f':
+                    if (strcmp(op, "br_on_func") == 0) { return makeBrOn(s, BrOnFunc); }
+                    goto parse_error;
+                  case 'i':
+                    if (strcmp(op, "br_on_i31") == 0) { return makeBrOn(s, BrOnI31); }
+                    goto parse_error;
+                  default: goto parse_error;
+                }
+              }
               case 't':
                 if (strcmp(op, "br_table") == 0) { return makeBreakTable(s); }
                 goto parse_error;
diff --git a/src/ir/ReFinalize.cpp b/src/ir/ReFinalize.cpp
index b8d620560..808b9d41b 100644
--- a/src/ir/ReFinalize.cpp
+++ b/src/ir/ReFinalize.cpp
@@ -146,7 +146,7 @@ void ReFinalize::visitI31Get(I31Get* curr) { curr->finalize(); }
 void ReFinalize::visitCallRef(CallRef* curr) { curr->finalize(); }
 void ReFinalize::visitRefTest(RefTest* curr) { curr->finalize(); }
 void ReFinalize::visitRefCast(RefCast* curr) { curr->finalize(); }
-void ReFinalize::visitBrOnCast(BrOnCast* curr) {
+void ReFinalize::visitBrOn(BrOn* curr) {
   curr->finalize();
   if (curr->type == Type::unreachable) {
     replaceUntaken(curr->ref, nullptr);
diff --git a/src/ir/branch-utils.h b/src/ir/branch-utils.h
index 5e2cc5184..a9561ef07 100644
--- a/src/ir/branch-utils.h
+++ b/src/ir/branch-utils.h
@@ -80,7 +80,7 @@ void operateOnScopeNameUsesAndSentTypes(Expression* expr, T func) {
       func(name, br->value ? br->value->type : Type::none);
     } else if (auto* sw = expr->dynCast<Switch>()) {
       func(name, sw->value ? sw->value->type : Type::none);
-    } else if (auto* br = expr->dynCast<BrOnCast>()) {
+    } else if (auto* br = expr->dynCast<BrOn>()) {
       func(name, br->getCastType());
     } else {
       WASM_UNREACHABLE("bad br type");
diff --git a/src/ir/cost.h b/src/ir/cost.h
index 8c23c91f8..212933ffb 100644
--- a/src/ir/cost.h
+++ b/src/ir/cost.h
@@ -582,8 +582,11 @@ struct CostAnalyzer : public OverriddenVisitor<CostAnalyzer, Index> {
   Index visitRefCast(RefCast* curr) {
     return 2 + nullCheckCost(curr->ref) + visit(curr->ref) + visit(curr->rtt);
   }
-  Index visitBrOnCast(BrOnCast* curr) {
-    return 3 + nullCheckCost(curr->ref) + visit(curr->ref) + visit(curr->rtt);
+  Index visitBrOn(BrOn* curr) {
+    // BrOnCast has more work to do with the rtt, so add a little there.
+    Index base = curr->op == BrOnCast ? 3 : 2;
+    return base + nullCheckCost(curr->ref) + visit(curr->ref) +
+           maybeVisit(curr->rtt);
   }
   Index visitRttCanon(RttCanon* curr) {
     // TODO: investigate actual RTT costs in VMs
diff --git a/src/ir/effects.h b/src/ir/effects.h
index ca440ce51..c7142dc34 100644
--- a/src/ir/effects.h
+++ b/src/ir/effects.h
@@ -565,9 +565,7 @@ private:
       // Traps if the ref is not null and it has an invalid rtt.
       parent.implicitTrap = true;
     }
-    void visitBrOnCast(BrOnCast* curr) {
-      parent.breakTargets.insert(curr->name);
-    }
+    void visitBrOn(BrOn* curr) { parent.breakTargets.insert(curr->name); }
     void visitRttCanon(RttCanon* curr) {}
     void visitRttSub(RttSub* curr) {}
     void visitStructNew(StructNew* curr) {}
diff --git a/src/js/binaryen.js-post.js b/src/js/binaryen.js-post.js
index 9c9c4e036..fc64d8bbd 100644
--- a/src/js/binaryen.js-post.js
+++ b/src/js/binaryen.js-post.js
@@ -101,7 +101,7 @@ function initializeConstants() {
     'CallRef',
     'RefTest',
     'RefCast',
-    'BrOnCast',
+    'BrOn',
     'RttCanon',
     'RttSub',
     'StructNew',
diff --git a/src/passes/Print.cpp b/src/passes/Print.cpp
index 371d83962..c380eb0f9 100644
--- a/src/passes/Print.cpp
+++ b/src/passes/Print.cpp
@@ -1783,8 +1783,23 @@ struct PrintExpressionContents
     printMedium(o, "ref.cast ");
     printHeapTypeName(o, curr->getCastType().getHeapType());
   }
-  void visitBrOnCast(BrOnCast* curr) {
-    printMedium(o, "br_on_cast ");
+  void visitBrOn(BrOn* curr) {
+    switch (curr->op) {
+      case BrOnCast:
+        printMedium(o, "br_on_cast ");
+        break;
+      case BrOnFunc:
+        printMedium(o, "br_on_func ");
+        break;
+      case BrOnData:
+        printMedium(o, "br_on_data ");
+        break;
+      case BrOnI31:
+        printMedium(o, "br_on_i31 ");
+        break;
+      default:
+        WASM_UNREACHABLE("invalid ref.is_*");
+    }
     printName(curr->name, o);
   }
   void visitRttCanon(RttCanon* curr) {
@@ -2536,12 +2551,14 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
     printFullLine(curr->rtt);
     decIndent();
   }
-  void visitBrOnCast(BrOnCast* curr) {
+  void visitBrOn(BrOn* curr) {
     o << '(';
     PrintExpressionContents(currFunction, o).visit(curr);
     incIndent();
     printFullLine(curr->ref);
-    printFullLine(curr->rtt);
+    if (curr->rtt) {
+      printFullLine(curr->rtt);
+    }
     decIndent();
   }
   void visitRttCanon(RttCanon* curr) {
diff --git a/src/wasm-binary.h b/src/wasm-binary.h
index 8674d00b7..e976a57bf 100644
--- a/src/wasm-binary.h
+++ b/src/wasm-binary.h
@@ -1041,6 +1041,9 @@ enum ASTNodes {
   RefAsFunc = 0x58,
   RefAsData = 0x59,
   RefAsI31 = 0x5a,
+  BrOnFunc = 0x60,
+  BrOnData = 0x61,
+  BrOnI31 = 0x62,
 };
 
 enum MemoryAccess {
@@ -1525,7 +1528,7 @@ public:
   bool maybeVisitI31Get(Expression*& out, uint32_t code);
   bool maybeVisitRefTest(Expression*& out, uint32_t code);
   bool maybeVisitRefCast(Expression*& out, uint32_t code);
-  bool maybeVisitBrOnCast(Expression*& out, uint32_t code);
+  bool maybeVisitBrOn(Expression*& out, uint32_t code);
   bool maybeVisitRttCanon(Expression*& out, uint32_t code);
   bool maybeVisitRttSub(Expression*& out, uint32_t code);
   bool maybeVisitStructNew(Expression*& out, uint32_t code);
diff --git a/src/wasm-builder.h b/src/wasm-builder.h
index 516d1cc15..bb37ed340 100644
--- a/src/wasm-builder.h
+++ b/src/wasm-builder.h
@@ -709,8 +709,10 @@ public:
     ret->finalize();
     return ret;
   }
-  BrOnCast* makeBrOnCast(Name name, Expression* ref, Expression* rtt) {
-    auto* ret = wasm.allocator.alloc<BrOnCast>();
+  BrOn*
+  makeBrOn(BrOnOp op, Name name, Expression* ref, Expression* rtt = nullptr) {
+    auto* ret = wasm.allocator.alloc<BrOn>();
+    ret->op = op;
     ret->name = name;
     ret->ref = ref;
     ret->rtt = rtt;
diff --git a/src/wasm-delegations-fields.h b/src/wasm-delegations-fields.h
index b0be5844c..d4ea25b3d 100644
--- a/src/wasm-delegations-fields.h
+++ b/src/wasm-delegations-fields.h
@@ -588,12 +588,13 @@ switch (DELEGATE_ID) {
     DELEGATE_END(RefCast);
     break;
   }
-  case Expression::Id::BrOnCastId: {
-    DELEGATE_START(BrOnCast);
-    DELEGATE_FIELD_SCOPE_NAME_USE(BrOnCast, name);
-    DELEGATE_FIELD_CHILD(BrOnCast, rtt);
-    DELEGATE_FIELD_CHILD(BrOnCast, ref);
-    DELEGATE_END(BrOnCast);
+  case Expression::Id::BrOnId: {
+    DELEGATE_START(BrOn);
+    DELEGATE_FIELD_INT(BrOn, op);
+    DELEGATE_FIELD_SCOPE_NAME_USE(BrOn, name);
+    DELEGATE_FIELD_OPTIONAL_CHILD(BrOn, rtt);
+    DELEGATE_FIELD_CHILD(BrOn, ref);
+    DELEGATE_END(BrOn);
     break;
   }
   case Expression::Id::RttCanonId: {
diff --git a/src/wasm-delegations.h b/src/wasm-delegations.h
index fe2a73d5b..fd419f508 100644
--- a/src/wasm-delegations.h
+++ b/src/wasm-delegations.h
@@ -69,7 +69,7 @@ DELEGATE(I31Get);
 DELEGATE(CallRef);
 DELEGATE(RefTest);
 DELEGATE(RefCast);
-DELEGATE(BrOnCast);
+DELEGATE(BrOn);
 DELEGATE(RttCanon);
 DELEGATE(RttSub);
 DELEGATE(StructNew);
diff --git a/src/wasm-interpreter.h b/src/wasm-interpreter.h
index ab5644bf4..fb7a3fcea 100644
--- a/src/wasm-interpreter.h
+++ b/src/wasm-interpreter.h
@@ -1469,17 +1469,51 @@ public:
     assert(cast.outcome == cast.Success);
     return cast.castRef;
   }
-  Flow visitBrOnCast(BrOnCast* curr) {
-    NOTE_ENTER("BrOnCast");
-    auto cast = doCast(curr);
-    if (cast.outcome == cast.Break) {
-      return cast.breaking;
+  Flow visitBrOn(BrOn* curr) {
+    NOTE_ENTER("BrOn");
+    // BrOnCast uses the casting infrastructure, so handle it first.
+    if (curr->op == BrOnCast) {
+      auto cast = doCast(curr);
+      if (cast.outcome == cast.Break) {
+        return cast.breaking;
+      }
+      if (cast.outcome == cast.Null || cast.outcome == cast.Failure) {
+        return cast.originalRef;
+      }
+      assert(cast.outcome == cast.Success);
+      return Flow(curr->name, cast.castRef);
+    }
+    // The others do a simpler check for the type.
+    Flow flow = visit(curr->ref);
+    if (flow.breaking()) {
+      return flow;
     }
-    if (cast.outcome == cast.Null || cast.outcome == cast.Failure) {
-      return cast.originalRef;
+    const auto& value = flow.getSingleValue();
+    NOTE_EVAL1(value);
+    if (value.isNull()) {
+      return {value};
     }
-    assert(cast.outcome == cast.Success);
-    return Flow(curr->name, cast.castRef);
+    switch (curr->op) {
+      case BrOnFunc:
+        if (!value.type.isFunction()) {
+          return {value};
+        }
+        break;
+      case BrOnData:
+        if (!value.isGCData()) {
+          return {value};
+        }
+        break;
+      case BrOnI31:
+        if (value.type.getHeapType() != HeapType::i31) {
+          return {value};
+        }
+        break;
+      default:
+        WASM_UNREACHABLE("invalid br_on_*");
+    }
+    // No problems: take the branch.
+    return Flow(curr->name, value);
   }
   Flow visitRttCanon(RttCanon* curr) { return Literal(curr->type); }
   Flow visitRttSub(RttSub* curr) {
diff --git a/src/wasm-s-parser.h b/src/wasm-s-parser.h
index 98cd0560b..921ef6b4a 100644
--- a/src/wasm-s-parser.h
+++ b/src/wasm-s-parser.h
@@ -258,7 +258,7 @@ private:
   Expression* makeI31Get(Element& s, bool signed_);
   Expression* makeRefTest(Element& s);
   Expression* makeRefCast(Element& s);
-  Expression* makeBrOnCast(Element& s);
+  Expression* makeBrOn(Element& s, BrOnOp op);
   Expression* makeRttCanon(Element& s);
   Expression* makeRttSub(Element& s);
   Expression* makeStructNew(Element& s, bool default_);
diff --git a/src/wasm.h b/src/wasm.h
index 041f7b49b..2102972b3 100644
--- a/src/wasm.h
+++ b/src/wasm.h
@@ -555,6 +555,13 @@ enum RefAsOp {
   RefAsI31,
 };
 
+enum BrOnOp {
+  BrOnCast,
+  BrOnFunc,
+  BrOnData,
+  BrOnI31,
+};
+
 //
 // Expressions
 //
@@ -635,7 +642,7 @@ public:
     CallRefId,
     RefTestId,
     RefCastId,
-    BrOnCastId,
+    BrOnId,
     RttCanonId,
     RttSubId,
     StructNewId,
@@ -1381,12 +1388,15 @@ public:
   Type getCastType();
 };
 
-class BrOnCast : public SpecificExpression<Expression::BrOnCastId> {
+class BrOn : public SpecificExpression<Expression::BrOnId> {
 public:
-  BrOnCast(MixedArena& allocator) {}
+  BrOn(MixedArena& allocator) {}
 
+  BrOnOp op;
   Name name;
   Expression* ref;
+
+  // BrOnCast has an rtt that is used in the cast.
   Expression* rtt;
 
   void finalize();
diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index 4a82c84a9..64a40f2ca 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -3019,7 +3019,7 @@ BinaryConsts::ASTNodes WasmBinaryBuilder::readExpression(Expression*& curr) {
       if (maybeVisitRefCast(curr, opcode)) {
         break;
       }
-      if (maybeVisitBrOnCast(curr, opcode)) {
+      if (maybeVisitBrOn(curr, opcode)) {
         break;
       }
       if (maybeVisitRttCanon(curr, opcode)) {
@@ -5825,17 +5825,31 @@ bool WasmBinaryBuilder::maybeVisitRefCast(Expression*& out, uint32_t code) {
   return true;
 }
 
-bool WasmBinaryBuilder::maybeVisitBrOnCast(Expression*& out, uint32_t code) {
-  if (code != BinaryConsts::BrOnCast) {
-    return false;
+bool WasmBinaryBuilder::maybeVisitBrOn(Expression*& out, uint32_t code) {
+  BrOnOp op;
+  switch (code) {
+    case BinaryConsts::BrOnCast:
+      op = BrOnCast;
+      break;
+    case BinaryConsts::BrOnFunc:
+      op = BrOnFunc;
+      break;
+    case BinaryConsts::BrOnData:
+      op = BrOnData;
+      break;
+    case BinaryConsts::BrOnI31:
+      op = BrOnI31;
+      break;
+    default:
+      return false;
   }
   auto name = getBreakTarget(getU32LEB()).name;
-  auto* rtt = popNonVoidExpression();
-  if (!rtt->type.isRtt()) {
-    throwError("bad rtt for br_on_cast");
+  Expression* rtt = nullptr;
+  if (op == BrOnCast) {
+    rtt = popNonVoidExpression();
   }
   auto* ref = popNonVoidExpression();
-  out = Builder(wasm).makeBrOnCast(name, ref, rtt);
+  out = Builder(wasm).makeBrOn(op, name, ref, rtt);
   return true;
 }
 
diff --git a/src/wasm/wasm-s-parser.cpp b/src/wasm/wasm-s-parser.cpp
index 9cf2f3be9..3075f70d7 100644
--- a/src/wasm/wasm-s-parser.cpp
+++ b/src/wasm/wasm-s-parser.cpp
@@ -2130,18 +2130,21 @@ Expression* SExpressionWasmBuilder::makeRefCast(Element& s) {
   return Builder(wasm).makeRefCast(ref, rtt);
 }
 
-Expression* SExpressionWasmBuilder::makeBrOnCast(Element& s) {
+Expression* SExpressionWasmBuilder::makeBrOn(Element& s, BrOnOp op) {
   auto name = getLabel(*s[1]);
   auto* ref = parseExpression(*s[2]);
-  auto* rtt = parseExpression(*s[3]);
+  Expression* rtt = nullptr;
   Builder builder(wasm);
-  if (rtt->type == Type::unreachable) {
-    // An unreachable rtt is not supported: the text format does not provide the
-    // type, so if it's unreachable we should not even create a br_on_cast in
-    // such a case, as we'd have no idea what it casts to.
-    return builder.makeSequence(builder.makeDrop(ref), rtt);
+  if (op == BrOnCast) {
+    rtt = parseExpression(*s[3]);
+    if (rtt->type == Type::unreachable) {
+      // An unreachable rtt is not supported: the text format does not provide
+      // the type, so if it's unreachable we should not even create a br_on_cast
+      // in such a case, as we'd have no idea what it casts to.
+      return builder.makeSequence(builder.makeDrop(ref), rtt);
+    }
   }
-  return builder.makeBrOnCast(name, ref, rtt);
+  return builder.makeBrOn(op, name, ref, rtt);
 }
 
 Expression* SExpressionWasmBuilder::makeRttCanon(Element& s) {
diff --git a/src/wasm/wasm-stack.cpp b/src/wasm/wasm-stack.cpp
index 197581e33..0ccca0e23 100644
--- a/src/wasm/wasm-stack.cpp
+++ b/src/wasm/wasm-stack.cpp
@@ -1990,9 +1990,25 @@ void BinaryInstWriter::visitRefCast(RefCast* curr) {
   o << int8_t(BinaryConsts::GCPrefix) << U32LEB(BinaryConsts::RefCast);
 }
 
-void BinaryInstWriter::visitBrOnCast(BrOnCast* curr) {
-  o << int8_t(BinaryConsts::GCPrefix) << U32LEB(BinaryConsts::BrOnCast)
-    << U32LEB(getBreakIndex(curr->name));
+void BinaryInstWriter::visitBrOn(BrOn* curr) {
+  o << int8_t(BinaryConsts::GCPrefix);
+  switch (curr->op) {
+    case BrOnCast:
+      o << U32LEB(BinaryConsts::BrOnCast);
+      break;
+    case BrOnFunc:
+      o << U32LEB(BinaryConsts::BrOnFunc);
+      break;
+    case BrOnData:
+      o << U32LEB(BinaryConsts::BrOnData);
+      break;
+    case BrOnI31:
+      o << U32LEB(BinaryConsts::BrOnI31);
+      break;
+    default:
+      WASM_UNREACHABLE("invalid br_on_*");
+  }
+  o << U32LEB(getBreakIndex(curr->name));
 }
 
 void BinaryInstWriter::visitRttCanon(RttCanon* curr) {
diff --git a/src/wasm/wasm-validator.cpp b/src/wasm/wasm-validator.cpp
index c6d0d9fe7..6a1b72476 100644
--- a/src/wasm/wasm-validator.cpp
+++ b/src/wasm/wasm-validator.cpp
@@ -344,7 +344,7 @@ public:
   void visitI31Get(I31Get* curr);
   void visitRefTest(RefTest* curr);
   void visitRefCast(RefCast* curr);
-  void visitBrOnCast(BrOnCast* curr);
+  void visitBrOn(BrOn* curr);
   void visitRttCanon(RttCanon* curr);
   void visitRttSub(RttSub* curr);
   void visitStructNew(StructNew* curr);
@@ -2207,7 +2207,7 @@ void FunctionValidator::visitRefCast(RefCast* curr) {
   }
 }
 
-void FunctionValidator::visitBrOnCast(BrOnCast* curr) {
+void FunctionValidator::visitBrOn(BrOn* curr) {
   shouldBeTrue(getModule()->features.hasGC(),
                curr,
                "br_on_cast requires gc to be enabled");
@@ -2215,12 +2215,17 @@ void FunctionValidator::visitBrOnCast(BrOnCast* curr) {
     shouldBeTrue(
       curr->ref->type.isRef(), curr, "br_on_cast ref must have ref type");
   }
-  // Note that an unreachable rtt is not supported: the text and binary formats
-  // do not provide the type, so if it's unreachable we should not even create
-  // a br_on_cast in such a case, as we'd have no idea what it casts to.
-  shouldBeTrue(
-    curr->rtt->type.isRtt(), curr, "br_on_cast rtt must have rtt type");
-  noteBreak(curr->name, curr->getCastType(), curr);
+  if (curr->op == BrOnCast) {
+    // Note that an unreachable rtt is not supported: the text and binary
+    // formats do not provide the type, so if it's unreachable we should not
+    // even create a br_on_cast in such a case, as we'd have no idea what it
+    // casts to.
+    shouldBeTrue(
+      curr->rtt->type.isRtt(), curr, "br_on_cast rtt must have rtt type");
+    noteBreak(curr->name, curr->getCastType(), curr);
+  } else {
+    shouldBeTrue(curr->rtt == nullptr, curr, "non-cast BrOn must not have rtt");
+  }
 }
 
 void FunctionValidator::visitRttCanon(RttCanon* curr) {
diff --git a/src/wasm/wasm.cpp b/src/wasm/wasm.cpp
index 1e4ff6e3e..a62435412 100644
--- a/src/wasm/wasm.cpp
+++ b/src/wasm/wasm.cpp
@@ -931,15 +931,29 @@ void RefCast::finalize() {
 
 Type RefCast::getCastType() { return doGetCastType(this); }
 
-void BrOnCast::finalize() {
-  if (ref->type == Type::unreachable || rtt->type == Type::unreachable) {
+void BrOn::finalize() {
+  if (ref->type == Type::unreachable ||
+      (rtt && rtt->type == Type::unreachable)) {
     type = Type::unreachable;
   } else {
     type = ref->type;
   }
 }
 
-Type BrOnCast::getCastType() { return Type(rtt->type.getHeapType(), Nullable); }
+Type BrOn::getCastType() {
+  switch (op) {
+    case BrOnCast:
+      return Type(rtt->type.getHeapType(), Nullable);
+    case BrOnFunc:
+      return Type::funcref;
+    case BrOnData:
+      return Type::dataref;
+    case BrOnI31:
+      return Type::i31ref;
+    default:
+      WASM_UNREACHABLE("invalid br_on_*");
+  }
+}
 
 void RttCanon::finalize() {
   // Nothing to do - the type must have been set already during construction.
@@ -1032,7 +1046,7 @@ void RefAs::finalize() {
       type = Type::i31ref;
       break;
     default:
-      WASM_UNREACHABLE("unimplemented ref.is_*");
+      WASM_UNREACHABLE("invalid ref.as_*");
   }
 }
 
diff --git a/src/wasm2js.h b/src/wasm2js.h
index 23d8a7b8b..43c5f4cbd 100644
--- a/src/wasm2js.h
+++ b/src/wasm2js.h
@@ -2206,7 +2206,7 @@ Ref Wasm2JSBuilder::processFunctionBody(Module* m,
       unimplemented(curr);
       WASM_UNREACHABLE("unimp");
     }
-    Ref visitBrOnCast(BrOnCast* curr) {
+    Ref visitBrOn(BrOn* curr) {
       unimplemented(curr);
       WASM_UNREACHABLE("unimp");
     }