[NominalFuzzing] MergeSimilarFunctions: handle nominal types properly (#4602)

This fixes two bugs: First, we need to compare the nominal types of function
constants when looking for constants to "merge", not just their structure.
Second, when creating the new function we must use the proper type of
those constants, and not just another type.

2 files changed, 325 insertions, 2 deletions

diff --git a/src/passes/MergeSimilarFunctions.cpp b/src/passes/MergeSimilarFunctions.cpp
index d6e616c10..9edf327fc 100644
--- a/src/passes/MergeSimilarFunctions.cpp
+++ b/src/passes/MergeSimilarFunctions.cpp
@@ -118,7 +118,7 @@ struct ParamInfo {
       return (*literals)[0].type;
     } else if (auto callees = std::get_if<std::vector<Name>>(&values)) {
       auto* callee = module->getFunction((*callees)[0]);
-      return Type(callee->getSig(), NonNullable);
+      return Type(callee->type, NonNullable);
     } else {
       WASM_UNREACHABLE("unexpected const value type");
     }
@@ -246,7 +246,7 @@ bool MergeSimilarFunctions::areInEquvalentClass(Function* lhs,
       }
       auto* lhsCallee = module->getFunction(lhsCast->target);
       auto* rhsCallee = module->getFunction(rhsCast->target);
-      if (lhsCallee->getSig() != rhsCallee->getSig()) {
+      if (lhsCallee->type != rhsCallee->type) {
         return false;
       }
 
diff --git a/test/lit/passes/merge-similar-functions_types.wast b/test/lit/passes/merge-similar-functions_types.wast
new file mode 100644
index 000000000..497c734e1
--- /dev/null
+++ b/test/lit/passes/merge-similar-functions_types.wast
@@ -0,0 +1,323 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py --all-items and should not be edited.
+;; RUN: foreach %s %t wasm-opt --merge-similar-functions -all -S -o - \
+;; RUN:   | filecheck %s
+;; RUN: foreach %s %t wasm-opt --merge-similar-functions -all --nominal -S -o - \
+;; RUN:   | filecheck %s --check-prefix=NOMNL
+
+;; Calls to functions $2 and $3 are the only differences between the contents
+;; of $0 and $1, so we want to merge them and pass ref.funcs of $2 and $3.
+;; However, their nominal types differ, so in nominal typing we cannot do so.
+(module
+ ;; CHECK:      (type $type$0 (func))
+ ;; NOMNL:      (type $type$0 (func_subtype func))
+ (type $type$0 (func_subtype func))
+ ;; NOMNL:      (type $type$1 (func_subtype func))
+ (type $type$1 (func_subtype func))
+ ;; NOMNL:      (type $type$2 (func_subtype func))
+ (type $type$2 (func_subtype func))
+ (type $type$3 (func_subtype (param f32) (result f32) func))
+ (type $type$4 (func_subtype (param f64) (result f64) func))
+ ;; CHECK:      (type $ref|$type$0|_=>_none (func (param (ref $type$0))))
+
+ ;; CHECK:      (elem declare func $2 $3)
+
+ ;; CHECK:      (func $0
+ ;; CHECK-NEXT:  (call $byn$mgfn-shared$0
+ ;; CHECK-NEXT:   (ref.func $2)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $0 (type $type$0)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (call $2)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT: )
+ (func $0 (type $type$0)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (call $2)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+ )
+ ;; CHECK:      (func $1
+ ;; CHECK-NEXT:  (call $byn$mgfn-shared$0
+ ;; CHECK-NEXT:   (ref.func $3)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $1 (type $type$0)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (call $3)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT:  (nop)
+ ;; NOMNL-NEXT: )
+ (func $1 (type $type$0)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (call $3)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+ )
+ ;; CHECK:      (func $2
+ ;; CHECK-NEXT:  (drop
+ ;; CHECK-NEXT:   (i32.const 17)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $2 (type $type$1)
+ ;; NOMNL-NEXT:  (drop
+ ;; NOMNL-NEXT:   (i32.const 17)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $2 (type $type$1)
+  (drop
+   (i32.const 17)
+  )
+ )
+ ;; CHECK:      (func $3
+ ;; CHECK-NEXT:  (drop
+ ;; CHECK-NEXT:   (i32.const 999)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $3 (type $type$2)
+ ;; NOMNL-NEXT:  (drop
+ ;; NOMNL-NEXT:   (i32.const 999)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $3 (type $type$2)
+  (drop
+   (i32.const 999)
+  )
+ )
+)
+
+
+
+
+;; CHECK:      (func $byn$mgfn-shared$0 (param $0 (ref $type$0))
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (call_ref
+;; CHECK-NEXT:   (local.get $0)
+;; CHECK-NEXT:  )
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT: )
+(module
+ ;; As above, but now the nominal types do match, so we can optimize in all
+ ;; modes.
+
+ ;; CHECK:      (type $type$0 (func))
+ ;; NOMNL:      (type $type$1 (func_subtype func))
+
+ ;; NOMNL:      (type $type$0 (func_subtype func))
+ (type $type$0 (func_subtype func))
+ (type $type$1 (func_subtype func))
+ (type $type$3 (func_subtype (param f32) (result f32) func))
+ (type $type$4 (func_subtype (param f64) (result f64) func))
+ ;; CHECK:      (type $ref|$type$0|_=>_none (func (param (ref $type$0))))
+
+ ;; CHECK:      (global $global$0 (mut i32) (i32.const 10))
+ ;; NOMNL:      (type $ref|$type$1|_=>_none (func_subtype (param (ref $type$1)) func))
+
+ ;; NOMNL:      (global $global$0 (mut i32) (i32.const 10))
+ (global $global$0 (mut i32) (i32.const 10))
+ ;; CHECK:      (memory $0 (shared 16 17))
+ ;; NOMNL:      (memory $0 (shared 16 17))
+ (memory $0 (shared 16 17))
+ ;; CHECK:      (elem declare func $2 $3)
+
+ ;; CHECK:      (func $0
+ ;; CHECK-NEXT:  (call $byn$mgfn-shared$0
+ ;; CHECK-NEXT:   (ref.func $2)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (elem declare func $2 $3)
+
+ ;; NOMNL:      (func $0 (type $type$0)
+ ;; NOMNL-NEXT:  (call $byn$mgfn-shared$0
+ ;; NOMNL-NEXT:   (ref.func $2)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $0 (type $type$0)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (call $2)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+ )
+ ;; CHECK:      (func $1
+ ;; CHECK-NEXT:  (call $byn$mgfn-shared$0
+ ;; CHECK-NEXT:   (ref.func $3)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $1 (type $type$0)
+ ;; NOMNL-NEXT:  (call $byn$mgfn-shared$0
+ ;; NOMNL-NEXT:   (ref.func $3)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $1 (type $type$0)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (call $3)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+  (nop)
+ )
+ ;; CHECK:      (func $2
+ ;; CHECK-NEXT:  (drop
+ ;; CHECK-NEXT:   (i32.const 17)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $2 (type $type$1)
+ ;; NOMNL-NEXT:  (drop
+ ;; NOMNL-NEXT:   (i32.const 17)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $2 (type $type$1)
+  (drop
+   (i32.const 17)
+  )
+ )
+ ;; CHECK:      (func $3
+ ;; CHECK-NEXT:  (drop
+ ;; CHECK-NEXT:   (i32.const 999)
+ ;; CHECK-NEXT:  )
+ ;; CHECK-NEXT: )
+ ;; NOMNL:      (func $3 (type $type$1)
+ ;; NOMNL-NEXT:  (drop
+ ;; NOMNL-NEXT:   (i32.const 999)
+ ;; NOMNL-NEXT:  )
+ ;; NOMNL-NEXT: )
+ (func $3 (type $type$1)
+  (drop
+   (i32.const 999)
+  )
+ )
+)
+;; CHECK:      (func $byn$mgfn-shared$0 (param $0 (ref $type$0))
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (call_ref
+;; CHECK-NEXT:   (local.get $0)
+;; CHECK-NEXT:  )
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT:  (nop)
+;; CHECK-NEXT: )
+
+;; NOMNL:      (func $byn$mgfn-shared$0 (type $ref|$type$1|_=>_none) (param $0 (ref $type$1))
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (call_ref
+;; NOMNL-NEXT:   (local.get $0)
+;; NOMNL-NEXT:  )
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT:  (nop)
+;; NOMNL-NEXT: )