Change the Literal class's operator== to be bitwise (#1661)

The change means that nan values will be compared bitwise when writing A == B, and so the float rule of a nan is different from itself would not apply.

I think this is a safer default. In particular this PR fixes a fuzz bug in the rse pass, which placed Literals in a hash table, and due to nan != nan, an infinite loop... Also, looks like we really want a bitwise comparison pretty much everywhere anyhow, as can be seen in the diff here. Really the single place we need a floaty comparison is in the intepreter where we implement f32.eq etc., and there the code was already using the proper code path anyhow.

2 files changed, 58 insertions, 0 deletions

diff --git a/test/passes/rse.txt b/test/passes/rse.txt
index ba31b58cc..7ca2894e1 100644
--- a/test/passes/rse.txt
+++ b/test/passes/rse.txt
@@ -430,4 +430,33 @@
    )
   )
  )
+ (func $fuzz-nan (; 18 ;) (type $2)
+  (local $0 f64)
+  (local $1 f64)
+  (block $block
+   (br_if $block
+    (i32.const 0)
+   )
+   (loop $loop
+    (set_local $1
+     (get_local $0)
+    )
+    (set_local $0
+     (f64.const -nan:0xfffffffffff87)
+    )
+    (br_if $loop
+     (i32.const 1)
+    )
+   )
+  )
+  (set_local $0
+   (get_local $1)
+  )
+  (if
+   (i32.const 0)
+   (drop
+    (get_local $0)
+   )
+  )
+ )
 )
diff --git a/test/passes/rse.wast b/test/passes/rse.wast
index 6a0ada018..3b3d38ec1 100644
--- a/test/passes/rse.wast
+++ b/test/passes/rse.wast
@@ -248,5 +248,34 @@
     )
    )
   )
+  (func $fuzz-nan
+   (local $0 f64)
+   (local $1 f64)
+   (block $block
+    (br_if $block
+     (i32.const 0)
+    )
+    (loop $loop
+     (set_local $1
+      (get_local $0)
+     )
+     (set_local $0
+      (f64.const -nan:0xfffffffffff87)
+     )
+     (br_if $loop
+      (i32.const 1)
+     )
+    )
+   )
+   (set_local $0 ;; make them equal
+    (get_local $1)
+   )
+   (if
+    (i32.const 0)
+    (set_local $1 ;; we can drop this
+     (get_local $0)
+    )
+   )
+  )
 )