Update br_on_cast binary and text format (#5734)

The final versions of the br_on_cast and br_on_cast_fail instructions have two
reference type annotations: one for the input type and one for the cast target
type. In the binary format, this is represented as a flags byte followed by two
encoded heap types. Since these instructions have been in flux for a while, do
not attempt to maintain backward compatibility with older versions of the
instructions. Instead, upgrade all of the tests at once to use the new versions
of the instructions. Drop some binary tests of deprecated instruction encodings
that would be more effort to update than they're worth.

6 files changed, 40 insertions, 46 deletions

diff --git a/src/ir/module-utils.cpp b/src/ir/module-utils.cpp
index d3ca1c192..7e202a5da 100644
--- a/src/ir/module-utils.cpp
+++ b/src/ir/module-utils.cpp
@@ -91,6 +91,7 @@ struct CodeScanner
       counts.note(cast->castType);
     } else if (auto* cast = curr->dynCast<BrOn>()) {
       if (cast->op == BrOnCast || cast->op == BrOnCastFail) {
+        counts.note(cast->ref->type);
         counts.note(cast->castType);
       }
     } else if (auto* get = curr->dynCast<StructGet>()) {
diff --git a/src/passes/Print.cpp b/src/passes/Print.cpp
index 0d74d43f1..4bcc042a1 100644
--- a/src/passes/Print.cpp
+++ b/src/passes/Print.cpp
@@ -2172,10 +2172,9 @@ struct PrintExpressionContents
         printMedium(o, "br_on_cast ");
         printName(curr->name, o);
         o << ' ';
-        if (curr->castType.isNullable()) {
-          printMedium(o, "null ");
-        }
-        printHeapType(o, curr->castType.getHeapType(), wasm);
+        printType(o, curr->ref->type, wasm);
+        o << ' ';
+        printType(o, curr->castType, wasm);
         return;
       case BrOnCastFail:
         // TODO: These instructions are deprecated, so stop emitting them.
@@ -2197,10 +2196,9 @@ struct PrintExpressionContents
         printMedium(o, "br_on_cast_fail ");
         printName(curr->name, o);
         o << ' ';
-        if (curr->castType.isNullable()) {
-          printMedium(o, "null ");
-        }
-        printHeapType(o, curr->castType.getHeapType(), wasm);
+        printType(o, curr->ref->type, wasm);
+        o << ' ';
+        printType(o, curr->castType, wasm);
         return;
     }
     WASM_UNREACHABLE("Unexpected br_on* op");
diff --git a/src/wasm-binary.h b/src/wasm-binary.h
index d1b7b4923..0a1fad332 100644
--- a/src/wasm-binary.h
+++ b/src/wasm-binary.h
@@ -1119,16 +1119,12 @@ enum ASTNodes {
   I31GetU = 0x22,
   RefTest = 0x40,
   RefCast = 0x41,
-  BrOnCast = 0x42,
-  BrOnCastFail = 0x43,
+  BrOnCast = 0x4e,
+  BrOnCastFail = 0x4f,
   RefTestStatic = 0x44,
   RefCastStatic = 0x45,
-  BrOnCastStatic = 0x46,
-  BrOnCastStaticFail = 0x47,
   RefTestNull = 0x48,
   RefCastNull = 0x49,
-  BrOnCastNull = 0x4a,
-  BrOnCastFailNull = 0x4b,
   RefCastNop = 0x4c,
   RefAsFunc = 0x58,
   RefAsI31 = 0x5a,
diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index 91309c906..133272a67 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -7034,14 +7034,10 @@ bool WasmBinaryBuilder::maybeVisitBrOn(Expression*& out, uint32_t code) {
     case BinaryConsts::BrOnNonNull:
       op = BrOnNonNull;
       break;
-    case BinaryConsts::BrOnCastStatic:
     case BinaryConsts::BrOnCast:
-    case BinaryConsts::BrOnCastNull:
       op = BrOnCast;
       break;
-    case BinaryConsts::BrOnCastStaticFail:
     case BinaryConsts::BrOnCastFail:
-    case BinaryConsts::BrOnCastFailNull:
       op = BrOnCastFail;
       break;
     case BinaryConsts::BrOnFunc:
@@ -7063,18 +7059,24 @@ bool WasmBinaryBuilder::maybeVisitBrOn(Expression*& out, uint32_t code) {
     default:
       return false;
   }
-  auto name = getBreakTarget(getU32LEB()).name;
-  if (castType == Type::none && (op == BrOnCast || op == BrOnCastFail)) {
-    auto nullability = (code == BinaryConsts::BrOnCastNull ||
-                        code == BinaryConsts::BrOnCastFailNull)
-                         ? Nullable
-                         : NonNullable;
-    bool legacy = code == BinaryConsts::BrOnCastStatic ||
-                  code == BinaryConsts::BrOnCastStaticFail;
-    auto type = legacy ? getIndexedHeapType() : getHeapType();
-    castType = Type(type, nullability);
+  uint8_t flags = 0;
+  if (op == BrOnCast || op == BrOnCastFail) {
+    flags = getInt8();
   }
+  auto name = getBreakTarget(getU32LEB()).name;
   auto* ref = popNonVoidExpression();
+  if (op == BrOnCast || op == BrOnCastFail) {
+    auto inputNullability = (flags & 1) ? Nullable : NonNullable;
+    auto castNullability = (flags & 2) ? Nullable : NonNullable;
+    auto inputHeapType = getHeapType();
+    auto castHeapType = getHeapType();
+    auto inputType = Type(inputHeapType, inputNullability);
+    castType = Type(castHeapType, castNullability);
+    if (!Type::isSubType(ref->type, inputType)) {
+      throwError(std::string("Invalid reference type for ") +
+                 ((op == BrOnCast) ? "br_on_cast" : "br_on_cast_fail"));
+    }
+  }
   out = Builder(wasm).makeBrOn(op, name, ref, castType);
   return true;
 }
diff --git a/src/wasm/wasm-s-parser.cpp b/src/wasm/wasm-s-parser.cpp
index 6adfb6149..a51ad3716 100644
--- a/src/wasm/wasm-s-parser.cpp
+++ b/src/wasm/wasm-s-parser.cpp
@@ -2892,16 +2892,16 @@ Expression* SExpressionWasmBuilder::makeBrOnCast(Element& s,
                                                  bool onFail) {
   int i = 1;
   auto name = getLabel(*s[i++]);
+  std::optional<Type> inputType;
   if (!castType) {
-    auto nullability = NonNullable;
-    if (s[i]->str().str == "null") {
-      nullability = Nullable;
-      ++i;
-    }
-    auto type = parseHeapType(*s[i++]);
-    castType = Type(type, nullability);
+    inputType = elementToType(*s[i++]);
+    castType = elementToType(*s[i++]);
   }
   auto* ref = parseExpression(*s[i]);
+  if (inputType && !Type::isSubType(ref->type, *inputType)) {
+    throw ParseException(
+      "br_on_cast* ref type does not match expected type", s.line, s.col);
+  }
   auto op = onFail ? BrOnCastFail : BrOnCast;
   return Builder(wasm).makeBrOn(op, name, ref, *castType);
 }
diff --git a/src/wasm/wasm-stack.cpp b/src/wasm/wasm-stack.cpp
index 7a4dd3983..ee457da69 100644
--- a/src/wasm/wasm-stack.cpp
+++ b/src/wasm/wasm-stack.cpp
@@ -2045,25 +2045,22 @@ void BinaryInstWriter::visitBrOn(BrOn* curr) {
       o << U32LEB(getBreakIndex(curr->name));
       return;
     case BrOnCast:
+    case BrOnCastFail: {
       o << int8_t(BinaryConsts::GCPrefix);
-      if (curr->castType.isNullable()) {
-        o << U32LEB(BinaryConsts::BrOnCastNull);
-      } else {
+      if (curr->op == BrOnCast) {
         o << U32LEB(BinaryConsts::BrOnCast);
-      }
-      o << U32LEB(getBreakIndex(curr->name));
-      parent.writeHeapType(curr->castType.getHeapType());
-      return;
-    case BrOnCastFail:
-      o << int8_t(BinaryConsts::GCPrefix);
-      if (curr->castType.isNullable()) {
-        o << U32LEB(BinaryConsts::BrOnCastFailNull);
       } else {
         o << U32LEB(BinaryConsts::BrOnCastFail);
       }
+      assert(curr->ref->type.isRef());
+      uint8_t flags = (curr->ref->type.isNullable() ? 1 : 0) |
+                      (curr->castType.isNullable() ? 2 : 0);
+      o << flags;
       o << U32LEB(getBreakIndex(curr->name));
+      parent.writeHeapType(curr->ref->type.getHeapType());
       parent.writeHeapType(curr->castType.getHeapType());
       return;
+    }
   }
   WASM_UNREACHABLE("invalid br_on_*");
 }