/* * Copyright 2023 WebAssembly Community Group participants * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include "support/name.h" #include "support/result.h" #ifndef parser_lexer_h #define parser_lexer_h namespace wasm::WATParser { struct TextPos { size_t line; size_t col; bool operator==(const TextPos& other) const; bool operator!=(const TextPos& other) const { return !(*this == other); } friend std::ostream& operator<<(std::ostream& os, const TextPos& pos); }; // ====== // Tokens // ====== struct LParenTok { bool operator==(const LParenTok&) const { return true; } friend std::ostream& operator<<(std::ostream&, const LParenTok&); }; struct RParenTok { bool operator==(const RParenTok&) const { return true; } friend std::ostream& operator<<(std::ostream&, const RParenTok&); }; struct IdTok { // Whether this ID has `$"..."` format bool isStr; // If the ID is a string ID and contains escapes, this is its contents. std::optional str; bool operator==(const IdTok&) const { return true; } friend std::ostream& operator<<(std::ostream&, const IdTok&); }; enum Sign { NoSign, Pos, Neg }; struct IntTok { uint64_t n; Sign sign; bool operator==(const IntTok&) const; friend std::ostream& operator<<(std::ostream&, const IntTok&); }; struct FloatTok { // The payload if we lexed a nan with payload. We cannot store the payload // directly in `d` because we do not know at this point whether we are parsing // an f32 or f64 and therefore we do not know what the allowable payloads are. // No payload with NaN means to use the default payload for the expected float // width. std::optional nanPayload; double d; bool operator==(const FloatTok&) const; friend std::ostream& operator<<(std::ostream&, const FloatTok&); }; struct StringTok { // If the string contains escapes, this is its contents. std::optional str; bool operator==(const StringTok& other) const { return str == other.str; } friend std::ostream& operator<<(std::ostream&, const StringTok&); }; struct KeywordTok { bool operator==(const KeywordTok&) const { return true; } friend std::ostream& operator<<(std::ostream&, const KeywordTok&); }; struct Token { using Data = std::variant; std::string_view span; Data data; // ==================== // Token classification // ==================== bool isLParen() const { return std::get_if(&data); } bool isRParen() const { return std::get_if(&data); } std::optional getKeyword() const { if (std::get_if(&data)) { return span; } return {}; } template std::optional getU() const; template std::optional getS() const; template std::optional getI() const; std::optional getF64() const; std::optional getF32() const; std::optional getString() const; std::optional getID() const; bool operator==(const Token&) const; friend std::ostream& operator<<(std::ostream& os, const Token&); }; // =========== // Annotations // =========== struct Annotation { Name kind; std::string_view contents; }; extern Name srcAnnotationKind; // ===== // Lexer // ===== struct Lexer { private: std::string_view buffer; size_t index = 0; std::optional curr; std::vector annotations; public: Lexer(std::string_view buffer) : buffer(buffer) { setIndex(0); } size_t getIndex() const { return index; } void setIndex(size_t i) { index = i; advance(); } bool takeLParen() { if (!curr || !curr->isLParen()) { return false; } advance(); return true; } bool peekLParen() { return Lexer(*this).takeLParen(); } bool takeRParen() { if (!curr || !curr->isRParen()) { return false; } advance(); return true; } bool peekRParen() { return Lexer(*this).takeRParen(); } bool takeUntilParen() { while (true) { if (!curr) { return false; } if (curr->isLParen() || curr->isRParen()) { return true; } advance(); } } std::optional takeID() { if (curr) { if (auto id = curr->getID()) { advance(); // See comment on takeName. return Name(std::string(*id)); } } return {}; } std::optional takeKeyword() { if (curr) { if (auto keyword = curr->getKeyword()) { advance(); return *keyword; } } return {}; } std::optional peekKeyword() { return Lexer(*this).takeKeyword(); } bool takeKeyword(std::string_view expected) { if (curr) { if (auto keyword = curr->getKeyword()) { if (*keyword == expected) { advance(); return true; } } } return false; } std::optional takeOffset() { using namespace std::string_view_literals; if (curr) { if (auto keyword = curr->getKeyword()) { if (keyword->substr(0, 7) != "offset="sv) { return {}; } Lexer subLexer(keyword->substr(7)); if (subLexer.empty()) { return {}; } if (auto o = subLexer.curr->getU()) { subLexer.advance(); if (subLexer.empty()) { advance(); return o; } } } } return std::nullopt; } std::optional takeAlign() { using namespace std::string_view_literals; if (curr) { if (auto keyword = curr->getKeyword()) { if (keyword->substr(0, 6) != "align="sv) { return {}; } Lexer subLexer(keyword->substr(6)); if (subLexer.empty()) { return {}; } if (auto a = subLexer.curr->getU()) { subLexer.advance(); if (subLexer.empty()) { advance(); return a; } } } } return {}; } template std::optional takeU() { if (curr) { if (auto n = curr->getU()) { advance(); return n; } } return std::nullopt; } template std::optional takeI() { if (curr) { if (auto n = curr->getI()) { advance(); return n; } } return std::nullopt; } std::optional takeU64() { return takeU(); } std::optional takeI64() { return takeI(); } std::optional takeU32() { return takeU(); } std::optional takeI32() { return takeI(); } std::optional takeI16() { return takeI(); } std::optional takeU8() { return takeU(); } std::optional takeI8() { return takeI(); } std::optional takeF64() { if (curr) { if (auto d = curr->getF64()) { advance(); return d; } } return std::nullopt; } std::optional takeF32() { if (curr) { if (auto f = curr->getF32()) { advance(); return f; } } return std::nullopt; } std::optional takeString() { if (curr) { if (auto s = curr->getString()) { std::string ret(*s); advance(); return ret; } } return {}; } std::optional takeName() { // TODO: Move this to lexer and validate UTF. if (auto str = takeString()) { // Copy to a std::string to make sure we have a null terminator, otherwise // the `Name` constructor won't work correctly. // TODO: Update `Name` to use string_view instead of char* and/or to take // rvalue strings to avoid this extra copy. return Name(std::string(*str)); } return {}; } bool takeSExprStart(std::string_view expected) { auto original = *this; if (takeLParen() && takeKeyword(expected)) { return true; } *this = original; return false; } bool peekSExprStart(std::string_view expected) { auto original = *this; if (!takeLParen()) { return false; } bool ret = takeKeyword(expected); *this = original; return ret; } std::string_view next() const { return buffer.substr(index); } void advance() { annotations.clear(); skipSpace(); lexToken(); } bool empty() const { return !curr; } TextPos position(const char* c) const; TextPos position(size_t i) const { return position(buffer.data() + i); } TextPos position(std::string_view span) const { return position(span.data()); } TextPos position() const { return position(getPos()); } size_t getPos() const { if (curr) { return getIndex() - curr->span.size(); } return getIndex(); } [[nodiscard]] Err err(size_t pos, std::string reason) { std::stringstream msg; msg << position(pos) << ": error: " << reason; return Err{msg.str()}; } [[nodiscard]] Err err(std::string reason) { return err(getPos(), reason); } const std::vector getAnnotations() { return annotations; } std::vector takeAnnotations() { return std::move(annotations); } void setAnnotations(std::vector&& annotations) { this->annotations = std::move(annotations); } private: void skipSpace(); void lexToken(); }; } // namespace wasm::WATParser #endif // parser_lexer_h