Make `TypeBuilder::build()` fallible (#4474)

It is possible for type building to fail, for example if the declared nominal
supertypes form a cycle or are structurally invalid. Previously we would report
a fatal error and kill the program from inside `TypeBuilder::build()` in these
situations, but this handles errors at the wrong layer of the code base and is
inconvenient for testing the error cases.

In preparation for testing the new error cases introduced by isorecursive
typing, make type building fallible and add new tests for existing error cases.
Also fix supertype cycle detection, which it turns out did not work correctly.

3 files changed, 87 insertions, 27 deletions

diff --git a/test/example/type-builder-nominal.cpp b/test/example/type-builder-nominal.cpp
index 5d3bcc17c..470415fa6 100644
--- a/test/example/type-builder-nominal.cpp
+++ b/test/example/type-builder-nominal.cpp
@@ -47,7 +47,7 @@ void test_builder() {
   std::cout << "(ref null $array) => " << refNullArray << "\n";
   std::cout << "(rtt 0 $array) => " << rttArray << "\n\n";
 
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   Type newRefSig = Type(built[0], NonNullable);
   Type newRefStruct = Type(built[1], NonNullable);
@@ -89,7 +89,7 @@ void test_canonicalization() {
   builder[2] = Signature(Type::none, Type::none);
   builder[3] = Signature(Type::none, Type::none);
 
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   assert(built[0] != struct_);
   assert(built[1] != struct_);
@@ -115,7 +115,7 @@ void test_basic() {
   builder[4] = HeapType::any;
   builder[5] = HeapType::i31;
 
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   assert(built[0].getSignature() == Signature(Type::anyref, Type::i31ref));
   assert(built[1].getSignature() == built[0].getSignature());
@@ -134,7 +134,7 @@ void test_signatures(bool warm) {
   Type tempRef = builder.getTempRefType(builder[0], Nullable);
   builder[0] = Signature(Type::i31ref, Type::anyref);
   builder[1] = Signature(tempRef, tempRef);
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   HeapType small = Signature(Type::i31ref, Type::anyref);
   HeapType big =
@@ -159,7 +159,7 @@ void test_recursive() {
       TypeBuilder builder(1);
       Type temp = builder.getTempRefType(builder[0], Nullable);
       builder[0] = Signature(Type::none, temp);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n\n";
     assert(built[0] == built[0].getSignature().results.getHeapType());
@@ -175,7 +175,7 @@ void test_recursive() {
       Type temp1 = builder.getTempRefType(builder[1], Nullable);
       builder[0] = Signature(Type::none, temp1);
       builder[1] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n\n";
@@ -199,7 +199,7 @@ void test_recursive() {
       builder[2] = Signature(Type::none, temp3);
       builder[3] = Signature(Type::none, temp4);
       builder[4] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n";
@@ -241,7 +241,7 @@ void test_recursive() {
       builder[3] = Signature();
       builder[4] = Signature(Type::none, temp0);
       builder[5] = Signature(Type::none, temp1);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n";
@@ -268,7 +268,7 @@ void test_recursive() {
       Type temp0 = builder.getTempRefType(builder[0], Nullable);
       builder[0] = Signature(Type::none, temp0);
       builder[1] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n\n";
@@ -322,7 +322,7 @@ void test_subtypes() {
       builder[0] = Signature(Type::none, Type::none);
       builder[1] = Struct{};
       builder[2] = Array(Field(Type::i32, Mutable));
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[0], built[0]) == built[0]);
     assert(LUB(built[1], built[1]) == built[1]);
@@ -341,7 +341,7 @@ void test_subtypes() {
       builder[2] = Signature(Type::none, Type::anyref);
       builder[3] = Signature(Type::none, structRef0);
       builder[4] = Signature(Type::none, structRef1);
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[0], built[1]) == HeapType::data);
     assert(LUB(built[2], built[3]) == HeapType::func);
@@ -357,7 +357,7 @@ void test_subtypes() {
       builder[0] = Struct{};
       builder[1] = Struct{};
       builder[2] = Struct{};
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[0], built[2]) == HeapType::data);
   }
@@ -378,7 +378,7 @@ void test_subtypes() {
       builder[3] = Signature(Type::i32, Type::anyref);
       builder[4] = Array(Field(Type::i32, Mutable));
       builder[5] = Array(Field(Type::i32, Mutable));
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[0], built[1]) == built[1]);
     assert(LUB(built[2], built[3]) == built[3]);
@@ -394,7 +394,7 @@ void test_subtypes() {
       builder[1] =
         Struct({Field(Type::i32, Immutable), Field(Type::i32, Immutable)});
       builder[1].subTypeOf(builder[0]);
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[1], built[0]) == built[0]);
   }
@@ -407,7 +407,7 @@ void test_subtypes() {
       builder[0] = Struct({Field(Type::anyref, Immutable)});
       builder[1] = Struct({Field(Type::funcref, Immutable)});
       builder[1].subTypeOf(builder[0]);
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[1], built[0]) == built[0]);
   }
@@ -427,7 +427,7 @@ void test_subtypes() {
       builder[1] = Struct({Field(d, Immutable)});
       builder[2] = Struct({Field(a, Immutable)});
       builder[3] = Struct({Field(b, Immutable)});
-      built = builder.build();
+      built = *builder.build();
     }
     assert(LUB(built[0], built[1]) == built[0]);
     assert(LUB(built[2], built[3]) == built[2]);
diff --git a/test/example/type-builder.cpp b/test/example/type-builder.cpp
index 0fba565f1..3c6c8eeda 100644
--- a/test/example/type-builder.cpp
+++ b/test/example/type-builder.cpp
@@ -31,7 +31,7 @@ void test_canonicalization() {
   builder[2] = Signature(Type::none, Type::none);
   builder[3] = Signature(Type::none, Type::none);
 
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   assert(built[0] == struct_);
   assert(built[1] == struct_);
@@ -55,7 +55,7 @@ void test_basic() {
   builder[4] = HeapType::any;
   builder[5] = HeapType::i31;
 
-  std::vector<HeapType> built = builder.build();
+  std::vector<HeapType> built = *builder.build();
 
   assert(built[0] == Signature(Type::anyref, Type::i31ref));
   assert(built[1] == built[0]);
@@ -75,7 +75,7 @@ void test_recursive() {
       TypeBuilder builder(1);
       Type temp = builder.getTempRefType(builder[0], Nullable);
       builder[0] = Signature(Type::none, temp);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n\n";
     assert(built[0] == built[0].getSignature().results.getHeapType());
@@ -91,7 +91,7 @@ void test_recursive() {
       Type temp1 = builder.getTempRefType(builder[1], Nullable);
       builder[0] = Signature(Type::none, temp1);
       builder[1] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n\n";
@@ -115,7 +115,7 @@ void test_recursive() {
       builder[2] = Signature(Type::none, temp3);
       builder[3] = Signature(Type::none, temp4);
       builder[4] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n";
@@ -151,7 +151,7 @@ void test_recursive() {
       builder[3] = Signature();
       builder[4] = Signature(Type::none, temp0);
       builder[5] = Signature(Type::none, temp1);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n";
@@ -178,7 +178,7 @@ void test_recursive() {
       Type temp0 = builder.getTempRefType(builder[0], Nullable);
       builder[0] = Signature(Type::none, temp0);
       builder[1] = Signature(Type::none, temp0);
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n\n";
@@ -197,7 +197,7 @@ void test_recursive() {
       builder[0] = Signature(anyref, temp0);
       builder[1] = Signature(anyref, temp0);
       builder[2] = HeapType::any;
-      built = builder.build();
+      built = *builder.build();
     }
     std::cout << built[0] << "\n";
     std::cout << built[1] << "\n\n";
@@ -379,7 +379,7 @@ void test_lub() {
       Struct({Field(tempB, Immutable), Field(Type::eqref, Immutable)});
     builder[1] =
       Struct({Field(tempA, Immutable), Field(Type::funcref, Immutable)});
-    auto built = builder.build();
+    auto built = *builder.build();
     Type a(built[0], Nullable);
     Type b(built[1], Nullable);
 
@@ -387,7 +387,7 @@ void test_lub() {
     Type tempLub = builder.getTempRefType(lubBuilder[0], Nullable);
     lubBuilder[0] =
       Struct({Field(tempLub, Immutable), Field(Type::anyref, Immutable)});
-    built = lubBuilder.build();
+    built = *lubBuilder.build();
     Type lub(built[0], Nullable);
 
     assert(LUB(a, b) == lub);
diff --git a/test/gtest/type-builder.cpp b/test/gtest/type-builder.cpp
index 1030dc9d4..9f1a99c5d 100644
--- a/test/gtest/type-builder.cpp
+++ b/test/gtest/type-builder.cpp
@@ -112,7 +112,9 @@ TEST_F(EquirecursiveTest, Basics) {
   builder[1] = struct_;
   builder[2] = array;
 
-  std::vector<HeapType> built = builder.build();
+  auto result = builder.build();
+  ASSERT_TRUE(result);
+  std::vector<HeapType> built = *result;
   ASSERT_EQ(built.size(), size_t{3});
 
   // The built types should have the correct kinds.
@@ -141,3 +143,61 @@ TEST_F(EquirecursiveTest, Basics) {
   EXPECT_NE(newRefNullArray, refNullArray);
   EXPECT_NE(newRttArray, rttArray);
 }
+
+static void testDirectSelfSupertype() {
+  // Type is directly a supertype of itself.
+  TypeBuilder builder(1);
+  builder[0] = Struct{};
+  builder[0].subTypeOf(builder[0]);
+
+  auto result = builder.build();
+  EXPECT_FALSE(result);
+
+  const auto* error = result.getError();
+  ASSERT_TRUE(error);
+  EXPECT_EQ(error->reason, TypeBuilder::ErrorReason::SelfSupertype);
+  EXPECT_EQ(error->index, 0u);
+}
+
+TEST_F(NominalTest, DirectSelfSupertype) { testDirectSelfSupertype(); }
+TEST_F(IsorecursiveTest, DirectSelfSupertype) { testDirectSelfSupertype(); }
+
+static void testIndirectSelfSupertype() {
+  // Type is indirectly a supertype of itself.
+  TypeBuilder builder(2);
+  builder.createRecGroup(0, 2);
+  builder[0] = Struct{};
+  builder[1] = Struct{};
+  builder[0].subTypeOf(builder[1]);
+  builder[1].subTypeOf(builder[0]);
+
+  auto result = builder.build();
+  EXPECT_FALSE(result);
+
+  const auto* error = result.getError();
+  ASSERT_TRUE(error);
+  EXPECT_EQ(error->reason, TypeBuilder::ErrorReason::SelfSupertype);
+  EXPECT_EQ(error->index, 0u);
+}
+
+TEST_F(NominalTest, IndirectSelfSupertype) { testIndirectSelfSupertype(); }
+TEST_F(IsorecursiveTest, IndirectSelfSupertype) { testIndirectSelfSupertype(); }
+
+static void testInvalidSupertype() {
+  TypeBuilder builder(2);
+  builder.createRecGroup(0, 2);
+  builder[0] = Struct{};
+  builder[1] = Struct({Field(Type::i32, Immutable)});
+  builder[0].subTypeOf(builder[1]);
+
+  auto result = builder.build();
+  EXPECT_FALSE(result);
+
+  const auto* error = result.getError();
+  ASSERT_TRUE(error);
+  EXPECT_EQ(error->reason, TypeBuilder::ErrorReason::InvalidSupertype);
+  EXPECT_EQ(error->index, 0u);
+}
+
+TEST_F(NominalTest, InvalidSupertype) { testInvalidSupertype(); }
+TEST_F(IsorecursiveTest, InvalidSupertype) { testInvalidSupertype(); }