Update br_on_cast binary and text format (#5762)

The final versions of the br_on_cast and br_on_cast_fail instructions have two
reference type annotations: one for the input type and one for the cast target
type. In the binary format, this is represented as a flags byte followed by two
encoded heap types. Upgrade all of the tests at once to use the new versions of
the instructions and drop support for the old instructions from the text parser.
Keep support in the binary parser to avoid breaking users, though. Drop some
binary tests of deprecated instruction encodings that would be more effort to
update than they're worth.

Re-land with fixes of #5734 

6 files changed, 61 insertions, 46 deletions

diff --git a/src/ir/module-utils.cpp b/src/ir/module-utils.cpp
index d3ca1c192..7e202a5da 100644
--- a/src/ir/module-utils.cpp
+++ b/src/ir/module-utils.cpp
@@ -91,6 +91,7 @@ struct CodeScanner
       counts.note(cast->castType);
     } else if (auto* cast = curr->dynCast<BrOn>()) {
       if (cast->op == BrOnCast || cast->op == BrOnCastFail) {
+        counts.note(cast->ref->type);
         counts.note(cast->castType);
       }
     } else if (auto* get = curr->dynCast<StructGet>()) {
diff --git a/src/passes/Print.cpp b/src/passes/Print.cpp
index 0d74d43f1..4bcc042a1 100644
--- a/src/passes/Print.cpp
+++ b/src/passes/Print.cpp
@@ -2172,10 +2172,9 @@ struct PrintExpressionContents
         printMedium(o, "br_on_cast ");
         printName(curr->name, o);
         o << ' ';
-        if (curr->castType.isNullable()) {
-          printMedium(o, "null ");
-        }
-        printHeapType(o, curr->castType.getHeapType(), wasm);
+        printType(o, curr->ref->type, wasm);
+        o << ' ';
+        printType(o, curr->castType, wasm);
         return;
       case BrOnCastFail:
         // TODO: These instructions are deprecated, so stop emitting them.
@@ -2197,10 +2196,9 @@ struct PrintExpressionContents
         printMedium(o, "br_on_cast_fail ");
         printName(curr->name, o);
         o << ' ';
-        if (curr->castType.isNullable()) {
-          printMedium(o, "null ");
-        }
-        printHeapType(o, curr->castType.getHeapType(), wasm);
+        printType(o, curr->ref->type, wasm);
+        o << ' ';
+        printType(o, curr->castType, wasm);
         return;
     }
     WASM_UNREACHABLE("Unexpected br_on* op");
diff --git a/src/wasm-binary.h b/src/wasm-binary.h
index d1b7b4923..81cc80d22 100644
--- a/src/wasm-binary.h
+++ b/src/wasm-binary.h
@@ -1119,16 +1119,16 @@ enum ASTNodes {
   I31GetU = 0x22,
   RefTest = 0x40,
   RefCast = 0x41,
-  BrOnCast = 0x42,
-  BrOnCastFail = 0x43,
+  BrOnCastLegacy = 0x42,
+  BrOnCastFailLegacy = 0x43,
+  BrOnCastNullLegacy = 0x4a,
+  BrOnCastFailNullLegacy = 0x4b,
+  BrOnCast = 0x4e,
+  BrOnCastFail = 0x4f,
   RefTestStatic = 0x44,
   RefCastStatic = 0x45,
-  BrOnCastStatic = 0x46,
-  BrOnCastStaticFail = 0x47,
   RefTestNull = 0x48,
   RefCastNull = 0x49,
-  BrOnCastNull = 0x4a,
-  BrOnCastFailNull = 0x4b,
   RefCastNop = 0x4c,
   RefAsFunc = 0x58,
   RefAsI31 = 0x5a,
diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index 7b3f741b6..ed27dd81a 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -7043,14 +7043,14 @@ bool WasmBinaryBuilder::maybeVisitBrOn(Expression*& out, uint32_t code) {
     case BinaryConsts::BrOnNonNull:
       op = BrOnNonNull;
       break;
-    case BinaryConsts::BrOnCastStatic:
     case BinaryConsts::BrOnCast:
-    case BinaryConsts::BrOnCastNull:
+    case BinaryConsts::BrOnCastLegacy:
+    case BinaryConsts::BrOnCastNullLegacy:
       op = BrOnCast;
       break;
-    case BinaryConsts::BrOnCastStaticFail:
     case BinaryConsts::BrOnCastFail:
-    case BinaryConsts::BrOnCastFailNull:
+    case BinaryConsts::BrOnCastFailLegacy:
+    case BinaryConsts::BrOnCastFailNullLegacy:
       op = BrOnCastFail;
       break;
     case BinaryConsts::BrOnFunc:
@@ -7072,18 +7072,37 @@ bool WasmBinaryBuilder::maybeVisitBrOn(Expression*& out, uint32_t code) {
     default:
       return false;
   }
-  auto name = getBreakTarget(getU32LEB()).name;
-  if (castType == Type::none && (op == BrOnCast || op == BrOnCastFail)) {
-    auto nullability = (code == BinaryConsts::BrOnCastNull ||
-                        code == BinaryConsts::BrOnCastFailNull)
-                         ? Nullable
-                         : NonNullable;
-    bool legacy = code == BinaryConsts::BrOnCastStatic ||
-                  code == BinaryConsts::BrOnCastStaticFail;
-    auto type = legacy ? getIndexedHeapType() : getHeapType();
-    castType = Type(type, nullability);
+  bool hasInputAnnotation =
+    code == BinaryConsts::BrOnCast || code == BinaryConsts::BrOnCastFail;
+  uint8_t flags = 0;
+  if (hasInputAnnotation) {
+    flags = getInt8();
   }
+  auto name = getBreakTarget(getU32LEB()).name;
   auto* ref = popNonVoidExpression();
+  if (op == BrOnCast || op == BrOnCastFail) {
+    Nullability inputNullability, castNullability;
+    HeapType inputHeapType, castHeapType;
+    if (hasInputAnnotation) {
+      inputNullability = (flags & 1) ? Nullable : NonNullable;
+      castNullability = (flags & 2) ? Nullable : NonNullable;
+      inputHeapType = getHeapType();
+    } else {
+      castNullability = (code == BinaryConsts::BrOnCastNullLegacy ||
+                         code == BinaryConsts::BrOnCastFailNullLegacy)
+                          ? Nullable
+                          : NonNullable;
+    }
+    castHeapType = getHeapType();
+    castType = Type(castHeapType, castNullability);
+    if (hasInputAnnotation) {
+      auto inputType = Type(inputHeapType, inputNullability);
+      if (!Type::isSubType(ref->type, inputType)) {
+        throwError(std::string("Invalid reference type for ") +
+                   ((op == BrOnCast) ? "br_on_cast" : "br_on_cast_fail"));
+      }
+    }
+  }
   out = Builder(wasm).makeBrOn(op, name, ref, castType);
   return true;
 }
diff --git a/src/wasm/wasm-s-parser.cpp b/src/wasm/wasm-s-parser.cpp
index 6adfb6149..a51ad3716 100644
--- a/src/wasm/wasm-s-parser.cpp
+++ b/src/wasm/wasm-s-parser.cpp
@@ -2892,16 +2892,16 @@ Expression* SExpressionWasmBuilder::makeBrOnCast(Element& s,
                                                  bool onFail) {
   int i = 1;
   auto name = getLabel(*s[i++]);
+  std::optional<Type> inputType;
   if (!castType) {
-    auto nullability = NonNullable;
-    if (s[i]->str().str == "null") {
-      nullability = Nullable;
-      ++i;
-    }
-    auto type = parseHeapType(*s[i++]);
-    castType = Type(type, nullability);
+    inputType = elementToType(*s[i++]);
+    castType = elementToType(*s[i++]);
   }
   auto* ref = parseExpression(*s[i]);
+  if (inputType && !Type::isSubType(ref->type, *inputType)) {
+    throw ParseException(
+      "br_on_cast* ref type does not match expected type", s.line, s.col);
+  }
   auto op = onFail ? BrOnCastFail : BrOnCast;
   return Builder(wasm).makeBrOn(op, name, ref, *castType);
 }
diff --git a/src/wasm/wasm-stack.cpp b/src/wasm/wasm-stack.cpp
index 7a4dd3983..ee457da69 100644
--- a/src/wasm/wasm-stack.cpp
+++ b/src/wasm/wasm-stack.cpp
@@ -2045,25 +2045,22 @@ void BinaryInstWriter::visitBrOn(BrOn* curr) {
       o << U32LEB(getBreakIndex(curr->name));
       return;
     case BrOnCast:
+    case BrOnCastFail: {
       o << int8_t(BinaryConsts::GCPrefix);
-      if (curr->castType.isNullable()) {
-        o << U32LEB(BinaryConsts::BrOnCastNull);
-      } else {
+      if (curr->op == BrOnCast) {
         o << U32LEB(BinaryConsts::BrOnCast);
-      }
-      o << U32LEB(getBreakIndex(curr->name));
-      parent.writeHeapType(curr->castType.getHeapType());
-      return;
-    case BrOnCastFail:
-      o << int8_t(BinaryConsts::GCPrefix);
-      if (curr->castType.isNullable()) {
-        o << U32LEB(BinaryConsts::BrOnCastFailNull);
       } else {
         o << U32LEB(BinaryConsts::BrOnCastFail);
       }
+      assert(curr->ref->type.isRef());
+      uint8_t flags = (curr->ref->type.isNullable() ? 1 : 0) |
+                      (curr->castType.isNullable() ? 2 : 0);
+      o << flags;
       o << U32LEB(getBreakIndex(curr->name));
+      parent.writeHeapType(curr->ref->type.getHeapType());
       parent.writeHeapType(curr->castType.getHeapType());
       return;
+    }
   }
   WASM_UNREACHABLE("invalid br_on_*");
 }