Binary format code section offset tracking (#2515)

Optionally track the binary format code section offsets,
that is, when loading a binary, remember where each IR
node was read from. This is necessary for DWARF
debug info, as these are the offsets DWARF refers to.

(Note that eventually we may want to do something
else, like first read the DWARF and only then add
debug info annotations into the IR in a more LLVM-like
manner, but this is more straightforward and should be
enough to update debug lines and ranges).

This tracking adds noticeable overhead - every single
IR node adds an entry in a map - so avoid it unless
actually necessary. Specifically, if the user passes in
-g and there are actually DWARF sections in the
binary, and we are not about to remove those sections,
then we need it.

Print binary format code section offsets in text, when
printing with -g. This will help debug and test dwarf
support. It looks like

;; code offset: 0x7

as an annotation right before each node.

Also add support for -g in wasm-opt tests (unlike
a pass, it has just one - as a prefix).

Helps #2400

10 files changed, 118 insertions, 8 deletions

diff --git a/src/passes/Print.cpp b/src/passes/Print.cpp
index 1704e0145..cbf3c5a24 100644
--- a/src/passes/Print.cpp
+++ b/src/passes/Print.cpp
@@ -1392,6 +1392,7 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
   Module* currModule = nullptr;
   Function* currFunction = nullptr;
   Function::DebugLocation lastPrintedLocation;
+  bool debugInfo;
 
   std::unordered_map<Name, Index> functionIndexes;
 
@@ -1421,6 +1422,16 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
       if (iter != debugLocations.end()) {
         printDebugLocation(iter->second);
       }
+      // show a binary position, if there is one
+      if (debugInfo) {
+        auto iter = currFunction->binaryLocations.find(curr);
+        if (iter != currFunction->binaryLocations.end()) {
+          Colors::grey(o);
+          o << ";; code offset: 0x" << iter->second << '\n';
+          restoreNormalColor(o);
+          doIndent(o, indent);
+        }
+      }
     }
   }
 
@@ -1437,6 +1448,10 @@ struct PrintSExpression : public OverriddenVisitor<PrintSExpression> {
 
   void setFull(bool full_) { full = full_; }
 
+  void setPrintStackIR(bool printStackIR_) { printStackIR = printStackIR_; }
+
+  void setDebugInfo(bool debugInfo_) { debugInfo = debugInfo_; }
+
   void incIndent() {
     if (minify) {
       return;
@@ -2321,6 +2336,7 @@ public:
 
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2337,6 +2353,7 @@ public:
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
     print.setMinify(true);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2353,6 +2370,7 @@ public:
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
     print.setFull(true);
+    print.setDebugInfo(runner->options.debugInfo);
     print.visitModule(module);
   }
 };
@@ -2368,7 +2386,8 @@ public:
 
   void run(PassRunner* runner, Module* module) override {
     PrintSExpression print(o);
-    print.printStackIR = true;
+    print.setDebugInfo(runner->options.debugInfo);
+    print.setPrintStackIR(true);
     print.visitModule(module);
   }
 };
diff --git a/src/tools/wasm-emscripten-finalize.cpp b/src/tools/wasm-emscripten-finalize.cpp
index b45b61b8f..a32a265cd 100644
--- a/src/tools/wasm-emscripten-finalize.cpp
+++ b/src/tools/wasm-emscripten-finalize.cpp
@@ -49,6 +49,7 @@ int main(int argc, const char* argv[]) {
   std::string dataSegmentFile;
   bool emitBinary = true;
   bool debugInfo = false;
+  bool DWARF = false;
   bool isSideModule = false;
   bool legalizeJavaScriptFFI = true;
   bool checkStackOverflow = false;
@@ -71,6 +72,11 @@ int main(int argc, const char* argv[]) {
          "Emit names section in wasm binary (or full debuginfo in wast)",
          Options::Arguments::Zero,
          [&debugInfo](Options*, const std::string&) { debugInfo = true; })
+    .add("--dwarf",
+         "",
+         "Update DWARF debug info",
+         Options::Arguments::Zero,
+         [&DWARF](Options*, const std::string&) { DWARF = true; })
     .add("--emit-text",
          "-S",
          "Emit text instead of binary for the output file",
@@ -164,6 +170,7 @@ int main(int argc, const char* argv[]) {
 
   Module wasm;
   ModuleReader reader;
+  reader.setDWARF(DWARF);
   try {
     reader.read(infile, wasm, inputSourceMapFilename);
   } catch (ParseException& p) {
diff --git a/src/tools/wasm-metadce.cpp b/src/tools/wasm-metadce.cpp
index 7d91b3e6f..3b889b984 100644
--- a/src/tools/wasm-metadce.cpp
+++ b/src/tools/wasm-metadce.cpp
@@ -498,6 +498,7 @@ int main(int argc, const char* argv[]) {
       std::cerr << "reading...\n";
     }
     ModuleReader reader;
+    reader.setDWARF(debugInfo);
     try {
       reader.read(options.extra["infile"], wasm);
     } catch (ParseException& p) {
diff --git a/src/tools/wasm-opt.cpp b/src/tools/wasm-opt.cpp
index 7b04f5449..2fd78a3da 100644
--- a/src/tools/wasm-opt.cpp
+++ b/src/tools/wasm-opt.cpp
@@ -43,7 +43,7 @@
 using namespace wasm;
 
 // runs a command and returns its output TODO: portability, return code checking
-std::string runCommand(std::string command) {
+static std::string runCommand(std::string command) {
 #ifdef __linux__
   std::string output;
   const int MAX_BUFFER = 1024;
@@ -59,6 +59,15 @@ std::string runCommand(std::string command) {
 #endif
 }
 
+static bool willRemoveDebugInfo(const std::vector<std::string>& passes) {
+  for (auto& pass : passes) {
+    if (pass == "strip" || pass == "strip-debug" || pass == "strip-dwarf") {
+      return true;
+    }
+  }
+  return false;
+}
+
 //
 // main
 //
@@ -210,6 +219,10 @@ int main(int argc, const char* argv[]) {
 
   if (!translateToFuzz) {
     ModuleReader reader;
+    // Enable DWARF parsing if we were asked for debug info, and were not
+    // asked to remove it.
+    reader.setDWARF(options.passOptions.debugInfo &&
+                    !willRemoveDebugInfo(options.passes));
     try {
       reader.read(options.extra["infile"], wasm, inputSourceMapFilename);
     } catch (ParseException& p) {
diff --git a/src/wasm-binary.h b/src/wasm-binary.h
index 36e3ab3d8..1c3a430fc 100644
--- a/src/wasm-binary.h
+++ b/src/wasm-binary.h
@@ -1053,10 +1053,12 @@ class WasmBinaryBuilder {
   const std::vector<char>& input;
   std::istream* sourceMap;
   std::pair<uint32_t, Function::DebugLocation> nextDebugLocation;
+  bool DWARF = false;
 
   size_t pos = 0;
   Index startIndex = -1;
   std::set<Function::DebugLocation> debugLocation;
+  size_t codeSectionLocation;
 
   std::set<BinaryConsts::Section> seenSections;
 
@@ -1068,6 +1070,7 @@ public:
     : wasm(wasm), allocator(wasm.allocator), input(input), sourceMap(nullptr),
       nextDebugLocation(0, {0, 0, 0}), debugLocation() {}
 
+  void setDWARF(bool value) { DWARF = value; }
   void read();
   void readUserSection(size_t payloadLen);
 
@@ -1275,6 +1278,9 @@ public:
   void visitBrOnExn(BrOnExn* curr);
 
   void throwError(std::string text);
+
+private:
+  bool hasDWARFSections();
 };
 
 } // namespace wasm
diff --git a/src/wasm-io.h b/src/wasm-io.h
index ad6b358fd..77d2506c4 100644
--- a/src/wasm-io.h
+++ b/src/wasm-io.h
@@ -29,6 +29,10 @@ namespace wasm {
 
 class ModuleReader {
 public:
+  // If DWARF support is enabled, we track the locations of all IR nodes in
+  // the binary, so that we can update DWARF sections later when writing.
+  void setDWARF(bool DWARF_) { DWARF = DWARF_; }
+
   // read text
   void readText(std::string filename, Module& wasm);
   // read binary
@@ -43,7 +47,13 @@ public:
   bool isBinaryFile(std::string filename);
 
 private:
+  bool DWARF = false;
+
   void readStdin(Module& wasm, std::string sourceMapFilename);
+
+  void readBinaryData(std::vector<char>& input,
+                      Module& wasm,
+                      std::string sourceMapFilename);
 };
 
 class ModuleWriter {
diff --git a/src/wasm.h b/src/wasm.h
index f98e3031a..bfa0c8947 100644
--- a/src/wasm.h
+++ b/src/wasm.h
@@ -1154,6 +1154,7 @@ public:
   std::map<Index, Name> localNames;
   std::map<Name, Index> localIndices;
 
+  // Source maps debugging info: map expression nodes to their file, line, col.
   struct DebugLocation {
     uint32_t fileIndex, lineNumber, columnNumber;
     bool operator==(const DebugLocation& other) const {
@@ -1175,6 +1176,10 @@ public:
   std::set<DebugLocation> prologLocation;
   std::set<DebugLocation> epilogLocation;
 
+  // General debugging info: map every instruction to its original position in
+  // the binary, relative to the beginning of the code section.
+  std::unordered_map<Expression*, uint32_t> binaryLocations;
+
   size_t getNumParams();
   size_t getNumVars();
   size_t getNumLocals();
@@ -1344,6 +1349,8 @@ public:
   Name start;
 
   std::vector<UserSection> userSections;
+
+  // Source maps debug info.
   std::vector<std::string> debugInfoFileNames;
 
   // `features` are the features allowed to be used in this module and should be
diff --git a/src/wasm/CMakeLists.txt b/src/wasm/CMakeLists.txt
index 38bbc97fb..916fbfb63 100644
--- a/src/wasm/CMakeLists.txt
+++ b/src/wasm/CMakeLists.txt
@@ -2,6 +2,7 @@ set(wasm_SOURCES
   literal.cpp
   wasm.cpp
   wasm-binary.cpp
+  wasm-debug.cpp
   wasm-emscripten.cpp
   wasm-debug.cpp
   wasm-interpreter.cpp
diff --git a/src/wasm/wasm-binary.cpp b/src/wasm/wasm-binary.cpp
index 77bdc23bb..71dd0fc21 100644
--- a/src/wasm/wasm-binary.cpp
+++ b/src/wasm/wasm-binary.cpp
@@ -21,6 +21,7 @@
 #include "support/bits.h"
 #include "support/debug.h"
 #include "wasm-binary.h"
+#include "wasm-debug.h"
 #include "wasm-stack.h"
 
 #define DEBUG_TYPE "binary"
@@ -731,7 +732,42 @@ void WasmBinaryWriter::finishUp() {
 
 // reader
 
+bool WasmBinaryBuilder::hasDWARFSections() {
+  assert(pos == 0);
+  getInt32(); // magic
+  getInt32(); // version
+  bool has = false;
+  while (more()) {
+    uint32_t sectionCode = getU32LEB();
+    uint32_t payloadLen = getU32LEB();
+    if (uint64_t(pos) + uint64_t(payloadLen) > input.size()) {
+      throwError("Section extends beyond end of input");
+    }
+    auto oldPos = pos;
+    if (sectionCode == BinaryConsts::Section::User) {
+      auto sectionName = getInlineString();
+      if (Debug::isDWARFSection(sectionName)) {
+        has = true;
+        break;
+      }
+    }
+    pos = oldPos + payloadLen;
+  }
+  pos = 0;
+  return has;
+}
+
 void WasmBinaryBuilder::read() {
+  if (DWARF) {
+    // In order to update dwarf, we must store info about each IR node's
+    // binary position. This has noticeable memory overhead, so we don't do it
+    // by default: the user must request it by setting "DWARF", and even if so
+    // we scan ahead to see that there actually *are* DWARF sections, so that
+    // we don't do unnecessary work.
+    if (!hasDWARFSections()) {
+      DWARF = false;
+    }
+  }
 
   readHeader();
   readSourceMapHeader();
@@ -740,7 +776,7 @@ void WasmBinaryBuilder::read() {
   while (more()) {
     uint32_t sectionCode = getU32LEB();
     uint32_t payloadLen = getU32LEB();
-    if (pos + payloadLen > input.size()) {
+    if (uint64_t(pos) + uint64_t(payloadLen) > input.size()) {
       throwError("Section extends beyond end of input");
     }
 
@@ -1252,6 +1288,9 @@ void WasmBinaryBuilder::readFunctionSignatures() {
 
 void WasmBinaryBuilder::readFunctions() {
   BYN_TRACE("== readFunctions\n");
+  if (DWARF) {
+    codeSectionLocation = pos;
+  }
   size_t total = getU32LEB();
   if (total != functionSignatures.size()) {
     throwError("invalid function section size, must equal types");
@@ -1972,6 +2011,7 @@ BinaryConsts::ASTNodes WasmBinaryBuilder::readExpression(Expression*& curr) {
   if (debugLocation.size()) {
     currDebugLocation.insert(*debugLocation.begin());
   }
+  size_t startPos = pos;
   uint8_t code = getInt8();
   BYN_TRACE("readExpression seeing " << (int)code << std::endl);
   switch (code) {
@@ -2165,8 +2205,13 @@ BinaryConsts::ASTNodes WasmBinaryBuilder::readExpression(Expression*& curr) {
       break;
     }
   }
-  if (curr && currDebugLocation.size()) {
-    currFunction->debugLocations[curr] = *currDebugLocation.begin();
+  if (curr) {
+    if (currDebugLocation.size()) {
+      currFunction->debugLocations[curr] = *currDebugLocation.begin();
+    }
+    if (DWARF && currFunction) {
+      currFunction->binaryLocations[curr] = startPos - codeSectionLocation;
+    }
   }
   BYN_TRACE("zz recurse from " << depth-- << " at " << pos << std::endl);
   return BinaryConsts::ASTNodes(code);
diff --git a/src/wasm/wasm-io.cpp b/src/wasm/wasm-io.cpp
index d874494c0..8fa714d9c 100644
--- a/src/wasm/wasm-io.cpp
+++ b/src/wasm/wasm-io.cpp
@@ -45,11 +45,12 @@ void ModuleReader::readText(std::string filename, Module& wasm) {
   readTextData(input, wasm);
 }
 
-static void readBinaryData(std::vector<char>& input,
-                           Module& wasm,
-                           std::string sourceMapFilename) {
+void ModuleReader::readBinaryData(std::vector<char>& input,
+                                  Module& wasm,
+                                  std::string sourceMapFilename) {
   std::unique_ptr<std::ifstream> sourceMapStream;
   WasmBinaryBuilder parser(wasm, input);
+  parser.setDWARF(DWARF);
   if (sourceMapFilename.size()) {
     sourceMapStream = make_unique<std::ifstream>();
     sourceMapStream->open(sourceMapFilename);