Disallow --nominal with GC (#4758)

Nominal types don't make much sense without GC, and in particular trying to emit
them with typed function references but not GC enabled can result in invalid
binaries because nominal types do not respect the type ordering constraints
required by the typed function references proposal. Making this change was
mostly straightforward, but required fixing the fuzzer to use --nominal only
when GC is enabled and required exiting early from nominal-only optimizations
when GC was not enabled.

Fixes #4756.

8 files changed, 27 insertions, 0 deletions

diff --git a/src/passes/ConstantFieldPropagation.cpp b/src/passes/ConstantFieldPropagation.cpp
index 2b314eecc..d2b08ea30 100644
--- a/src/passes/ConstantFieldPropagation.cpp
+++ b/src/passes/ConstantFieldPropagation.cpp
@@ -176,6 +176,9 @@ struct PCVScanner
 
 struct ConstantFieldPropagation : public Pass {
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "ConstantFieldPropagation requires nominal typing";
     }
diff --git a/src/passes/GlobalRefining.cpp b/src/passes/GlobalRefining.cpp
index e43f8ef59..6f2f19842 100644
--- a/src/passes/GlobalRefining.cpp
+++ b/src/passes/GlobalRefining.cpp
@@ -32,6 +32,9 @@ namespace {
 
 struct GlobalRefining : public Pass {
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "GlobalRefining requires nominal typing";
     }
diff --git a/src/passes/GlobalStructInference.cpp b/src/passes/GlobalStructInference.cpp
index c7eec2b94..b2717b850 100644
--- a/src/passes/GlobalStructInference.cpp
+++ b/src/passes/GlobalStructInference.cpp
@@ -62,6 +62,9 @@ struct GlobalStructInference : public Pass {
   std::unordered_map<HeapType, std::vector<Name>> typeGlobals;
 
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "GlobalStructInference requires nominal typing";
     }
diff --git a/src/passes/GlobalTypeOptimization.cpp b/src/passes/GlobalTypeOptimization.cpp
index 9b2a1f269..865fe8f69 100644
--- a/src/passes/GlobalTypeOptimization.cpp
+++ b/src/passes/GlobalTypeOptimization.cpp
@@ -112,6 +112,9 @@ struct GlobalTypeOptimization : public Pass {
   std::unordered_map<HeapType, std::vector<Index>> indexesAfterRemovals;
 
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "GlobalTypeOptimization requires nominal typing";
     }
diff --git a/src/passes/SignaturePruning.cpp b/src/passes/SignaturePruning.cpp
index 4f142eb97..ecf6d7e21 100644
--- a/src/passes/SignaturePruning.cpp
+++ b/src/passes/SignaturePruning.cpp
@@ -48,6 +48,9 @@ struct SignaturePruning : public Pass {
   std::unordered_map<HeapType, Signature> newSignatures;
 
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "SignaturePruning requires nominal typing";
     }
diff --git a/src/passes/SignatureRefining.cpp b/src/passes/SignatureRefining.cpp
index 0574df434..8a8c45872 100644
--- a/src/passes/SignatureRefining.cpp
+++ b/src/passes/SignatureRefining.cpp
@@ -48,6 +48,9 @@ struct SignatureRefining : public Pass {
   std::unordered_map<HeapType, Signature> newSignatures;
 
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "SignatureRefining requires nominal typing";
     }
diff --git a/src/passes/TypeRefining.cpp b/src/passes/TypeRefining.cpp
index 0f5169f8f..c755c1af6 100644
--- a/src/passes/TypeRefining.cpp
+++ b/src/passes/TypeRefining.cpp
@@ -78,6 +78,9 @@ struct TypeRefining : public Pass {
   StructUtils::StructValuesMap<FieldInfo> finalInfos;
 
   void run(PassRunner* runner, Module* module) override {
+    if (!module->features.hasGC()) {
+      return;
+    }
     if (getTypeSystem() != TypeSystem::Nominal) {
       Fatal() << "TypeRefining requires nominal typing";
     }
diff --git a/src/tools/tool-options.h b/src/tools/tool-options.h
index bc948fc97..c15291c8d 100644
--- a/src/tools/tool-options.h
+++ b/src/tools/tool-options.h
@@ -176,6 +176,12 @@ struct ToolOptions : public Options {
   void applyFeatures(Module& module) const {
     module.features.enable(enabledFeatures);
     module.features.disable(disabledFeatures);
+    // Non-default type systems only make sense with GC enabled. TODO: Error on
+    // non-GC equirecursive types as well once we make isorecursive the default
+    // if we don't remove equirecursive types entirely.
+    if (!module.features.hasGC() && getTypeSystem() == TypeSystem::Nominal) {
+      Fatal() << "Nominal typing is only allowed when GC is enabled";
+    }
   }
 
 private: