Fix binaryenjs testing (#2810)

These tests are now optional. However, if you run them and the
build is not found they will now error out, in order to avoid silently
failing.

1 files changed, 8 insertions, 2 deletions

diff --git a/auto_update_tests.py b/auto_update_tests.py
index 107a5f178..5cfff9c0e 100755
--- a/auto_update_tests.py
+++ b/auto_update_tests.py
@@ -243,12 +243,18 @@ TEST_SUITES = OrderedDict([
 
 
 def main():
+    all_suites = TEST_SUITES.keys()
+    skip_by_default = ['binaryenjs']
+
     if shared.options.list_suites:
-        for suite in TEST_SUITES.keys():
+        for suite in all_suites:
             print(suite)
         return 0
 
-    for test in shared.requested or TEST_SUITES.keys():
+    if not shared.requested:
+        shared.requested = [s for s in all_suites if s not in skip_by_default]
+
+    for test in shared.requested:
         TEST_SUITES[test]()
 
     print('\n[ success! ]')