diff options
| -rw-r--r-- | src/parser/input-impl.h | 255 | ||||
| -rw-r--r-- | src/parser/input.h | 69 | ||||
| -rw-r--r-- | src/parser/lexer.h | 246 | ||||
| -rw-r--r-- | test/gtest/wat-lexer.cpp | 2263 |
4 files changed, 1083 insertions, 1750 deletions
diff --git a/src/parser/input-impl.h b/src/parser/input-impl.h deleted file mode 100644 index e3cf52015..000000000 --- a/src/parser/input-impl.h +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright 2023 WebAssembly Community Group participants - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "input.h" - -#ifndef parser_input_impl_h -#define parser_input_impl_h - -inline std::optional<Token> ParseInput::peek() { - if (!empty()) { - return *lexer; - } - return {}; -} - -inline bool ParseInput::takeLParen() { - auto t = peek(); - if (!t || !t->isLParen()) { - return false; - } - ++lexer; - return true; -} - -inline bool ParseInput::takeRParen() { - auto t = peek(); - if (!t || !t->isRParen()) { - return false; - } - ++lexer; - return true; -} - -inline bool ParseInput::takeUntilParen() { - while (true) { - auto t = peek(); - if (!t) { - return false; - } - if (t->isLParen() || t->isRParen()) { - return true; - } - ++lexer; - } -} - -inline std::optional<Name> ParseInput::takeID() { - if (auto t = peek()) { - if (auto id = t->getID()) { - ++lexer; - // See comment on takeName. - return Name(std::string(*id)); - } - } - return {}; -} - -inline std::optional<std::string_view> ParseInput::takeKeyword() { - if (auto t = peek()) { - if (auto keyword = t->getKeyword()) { - ++lexer; - return *keyword; - } - } - return {}; -} - -inline bool ParseInput::takeKeyword(std::string_view expected) { - if (auto t = peek()) { - if (auto keyword = t->getKeyword()) { - if (*keyword == expected) { - ++lexer; - return true; - } - } - } - return false; -} - -inline std::optional<uint64_t> ParseInput::takeOffset() { - if (auto t = peek()) { - if (auto keyword = t->getKeyword()) { - if (keyword->substr(0, 7) != "offset="sv) { - return {}; - } - Lexer subLexer(keyword->substr(7)); - if (subLexer == subLexer.end()) { - return {}; - } - if (auto o = subLexer->getU<uint64_t>()) { - ++subLexer; - if (subLexer == subLexer.end()) { - ++lexer; - return o; - } - } - } - } - return std::nullopt; -} - -inline std::optional<uint32_t> ParseInput::takeAlign() { - if (auto t = peek()) { - if (auto keyword = t->getKeyword()) { - if (keyword->substr(0, 6) != "align="sv) { - return {}; - } - Lexer subLexer(keyword->substr(6)); - if (subLexer == subLexer.end()) { - return {}; - } - if (auto a = subLexer->getU<uint32_t>()) { - ++subLexer; - if (subLexer == subLexer.end()) { - ++lexer; - return a; - } - } - } - } - return {}; -} - -template<typename T> inline std::optional<T> ParseInput::takeU() { - if (auto t = peek()) { - if (auto n = t->getU<T>()) { - ++lexer; - return n; - } - } - return std::nullopt; -} - -template<typename T> inline std::optional<T> ParseInput::takeI() { - if (auto t = peek()) { - if (auto n = t->getI<T>()) { - ++lexer; - return n; - } - } - return std::nullopt; -} - -inline std::optional<uint64_t> ParseInput::takeU64() { - return takeU<uint64_t>(); -} - -inline std::optional<uint64_t> ParseInput::takeI64() { - return takeI<uint64_t>(); -} - -inline std::optional<uint32_t> ParseInput::takeU32() { - return takeU<uint64_t>(); -} - -inline std::optional<uint32_t> ParseInput::takeI32() { - return takeI<uint32_t>(); -} - -inline std::optional<uint16_t> ParseInput::takeI16() { - return takeI<uint16_t>(); -} - -inline std::optional<uint8_t> ParseInput::takeU8() { return takeU<uint8_t>(); } - -inline std::optional<uint8_t> ParseInput::takeI8() { return takeI<uint8_t>(); } - -inline std::optional<double> ParseInput::takeF64() { - if (auto t = peek()) { - if (auto d = t->getF64()) { - ++lexer; - return d; - } - } - return std::nullopt; -} - -inline std::optional<float> ParseInput::takeF32() { - if (auto t = peek()) { - if (auto f = t->getF32()) { - ++lexer; - return f; - } - } - return std::nullopt; -} - -inline std::optional<std::string> ParseInput::takeString() { - if (auto t = peek()) { - if (auto s = t->getString()) { - ++lexer; - return std::string(*s); - } - } - return {}; -} - -inline std::optional<Name> ParseInput::takeName() { - // TODO: Move this to lexer and validate UTF. - if (auto str = takeString()) { - // Copy to a std::string to make sure we have a null terminator, otherwise - // the `Name` constructor won't work correctly. - // TODO: Update `Name` to use string_view instead of char* and/or to take - // rvalue strings to avoid this extra copy. - return Name(std::string(*str)); - } - return {}; -} - -inline bool ParseInput::takeSExprStart(std::string_view expected) { - auto original = lexer; - if (takeLParen() && takeKeyword(expected)) { - return true; - } - lexer = original; - return false; -} - -inline bool ParseInput::peekSExprStart(std::string_view expected) { - auto original = lexer; - if (!takeLParen()) { - return false; - } - bool ret = takeKeyword(expected); - lexer = original; - return ret; -} - -inline Index ParseInput::getPos() { - if (auto t = peek()) { - return lexer.getIndex() - t->span.size(); - } - return lexer.getIndex(); -} - -inline Err ParseInput::err(Index pos, std::string reason) { - std::stringstream msg; - msg << lexer.position(pos) << ": error: " << reason; - return Err{msg.str()}; -} - -#endif // parser_input_impl_h diff --git a/src/parser/input.h b/src/parser/input.h index 6086ed1a5..f83f5a40a 100644 --- a/src/parser/input.h +++ b/src/parser/input.h @@ -41,40 +41,47 @@ struct ParseInput { bool empty() { return lexer.empty(); } - std::optional<Token> peek(); - bool takeLParen(); - bool takeRParen(); - bool takeUntilParen(); - std::optional<Name> takeID(); - std::optional<std::string_view> takeKeyword(); - bool takeKeyword(std::string_view expected); - std::optional<uint64_t> takeOffset(); - std::optional<uint32_t> takeAlign(); - std::optional<uint64_t> takeU64(); - std::optional<uint64_t> takeI64(); - std::optional<uint32_t> takeU32(); - std::optional<uint32_t> takeI32(); - std::optional<uint16_t> takeI16(); - std::optional<uint8_t> takeU8(); - std::optional<uint8_t> takeI8(); - std::optional<double> takeF64(); - std::optional<float> takeF32(); - std::optional<std::string> takeString(); - std::optional<Name> takeName(); - bool takeSExprStart(std::string_view expected); - bool peekSExprStart(std::string_view expected); + // TODO: Remove this useless layer of abstraction between the Lexer and + // Parser. + std::optional<Token> peek() { return lexer.peek(); } + bool takeLParen() { return lexer.takeLParen(); } + bool takeRParen() { return lexer.takeRParen(); } + bool takeUntilParen() { return lexer.takeUntilParen(); } + std::optional<Name> takeID() { return lexer.takeID(); } + std::optional<std::string_view> takeKeyword() { return lexer.takeKeyword(); } + bool takeKeyword(std::string_view expected) { + return lexer.takeKeyword(expected); + } + std::optional<uint64_t> takeOffset() { return lexer.takeOffset(); } + std::optional<uint32_t> takeAlign() { return lexer.takeAlign(); } + std::optional<uint64_t> takeU64() { return lexer.takeU64(); } + std::optional<uint64_t> takeI64() { return lexer.takeI64(); } + std::optional<uint32_t> takeU32() { return lexer.takeU32(); } + std::optional<uint32_t> takeI32() { return lexer.takeI32(); } + std::optional<uint16_t> takeI16() { return lexer.takeI16(); } + std::optional<uint8_t> takeU8() { return lexer.takeU8(); } + std::optional<uint8_t> takeI8() { return lexer.takeI8(); } + std::optional<double> takeF64() { return lexer.takeF64(); } + std::optional<float> takeF32() { return lexer.takeF32(); } + std::optional<std::string> takeString() { return lexer.takeString(); } + std::optional<Name> takeName() { return lexer.takeName(); } + bool takeSExprStart(std::string_view expected) { + return lexer.takeSExprStart(expected); + } + bool peekSExprStart(std::string_view expected) { + return lexer.peekSExprStart(expected); + } - Index getPos(); - [[nodiscard]] Err err(Index pos, std::string reason); - [[nodiscard]] Err err(std::string reason) { return err(getPos(), reason); } + Index getPos() { return lexer.getPos(); } -private: - template<typename T> std::optional<T> takeU(); - template<typename T> std::optional<T> takeS(); - template<typename T> std::optional<T> takeI(); -}; + [[nodiscard]] Err err(Index pos, std::string reason) { + std::stringstream msg; + msg << lexer.position(pos) << ": error: " << reason; + return Err{msg.str()}; + } -#include "input-impl.h" + [[nodiscard]] Err err(std::string reason) { return err(getPos(), reason); } +}; } // namespace wasm::WATParser diff --git a/src/parser/lexer.h b/src/parser/lexer.h index f0da151f9..8f9bd103a 100644 --- a/src/parser/lexer.h +++ b/src/parser/lexer.h @@ -23,6 +23,8 @@ #include <string_view> #include <variant> +#include "support/name.h" + #ifndef parser_lexer_h #define parser_lexer_h @@ -147,13 +149,6 @@ struct Token { // positions are computed on demand rather than eagerly because they are // typically only needed when there is an error to report. struct Lexer { - using iterator = Lexer; - using difference_type = std::ptrdiff_t; - using value_type = Token; - using pointer = const Token*; - using reference = const Token&; - using iterator_category = std::forward_iterator_tag; - private: std::string_view buffer; size_t index = 0; @@ -169,51 +164,238 @@ public: void setIndex(size_t i) { index = i; - skipSpace(); - lexToken(); + advance(); } - std::string_view next() const { return buffer.substr(index); } - Lexer& operator++() { - // Preincrement - skipSpace(); - lexToken(); - return *this; + std::optional<Token> peek() const { return curr; } + + bool takeLParen() { + auto t = peek(); + if (!t || !t->isLParen()) { + return false; + } + advance(); + return true; } - Lexer operator++(int) { - // Postincrement - Lexer ret = *this; - ++(*this); - return ret; + bool takeRParen() { + auto t = peek(); + if (!t || !t->isRParen()) { + return false; + } + advance(); + return true; + } + + bool takeUntilParen() { + while (true) { + auto t = peek(); + if (!t) { + return false; + } + if (t->isLParen() || t->isRParen()) { + return true; + } + advance(); + } + } + + std::optional<Name> takeID() { + if (auto t = peek()) { + if (auto id = t->getID()) { + advance(); + // See comment on takeName. + return Name(std::string(*id)); + } + } + return {}; + } + + std::optional<std::string_view> takeKeyword() { + if (auto t = peek()) { + if (auto keyword = t->getKeyword()) { + advance(); + return *keyword; + } + } + return {}; + } + + bool takeKeyword(std::string_view expected) { + if (auto t = peek()) { + if (auto keyword = t->getKeyword()) { + if (*keyword == expected) { + advance(); + return true; + } + } + } + return false; + } + + std::optional<uint64_t> takeOffset() { + using namespace std::string_view_literals; + if (auto t = peek()) { + if (auto keyword = t->getKeyword()) { + if (keyword->substr(0, 7) != "offset="sv) { + return {}; + } + Lexer subLexer(keyword->substr(7)); + if (subLexer.empty()) { + return {}; + } + if (auto o = subLexer.peek()->getU<uint64_t>()) { + subLexer.advance(); + if (subLexer.empty()) { + advance(); + return o; + } + } + } + } + return std::nullopt; + } + + std::optional<uint32_t> takeAlign() { + using namespace std::string_view_literals; + if (auto t = peek()) { + if (auto keyword = t->getKeyword()) { + if (keyword->substr(0, 6) != "align="sv) { + return {}; + } + Lexer subLexer(keyword->substr(6)); + if (subLexer.empty()) { + return {}; + } + if (auto a = subLexer.peek()->getU<uint32_t>()) { + subLexer.advance(); + if (subLexer.empty()) { + advance(); + return a; + } + } + } + } + return {}; + } + + template<typename T> std::optional<T> takeU() { + if (auto t = peek()) { + if (auto n = t->getU<T>()) { + advance(); + return n; + } + } + return std::nullopt; + } + + template<typename T> std::optional<T> takeI() { + if (auto t = peek()) { + if (auto n = t->getI<T>()) { + advance(); + return n; + } + } + return std::nullopt; + } + + std::optional<uint64_t> takeU64() { return takeU<uint64_t>(); } + + std::optional<uint64_t> takeI64() { return takeI<uint64_t>(); } + + std::optional<uint32_t> takeU32() { return takeU<uint32_t>(); } + + std::optional<uint32_t> takeI32() { return takeI<uint32_t>(); } + + std::optional<uint16_t> takeI16() { return takeI<uint16_t>(); } + + std::optional<uint8_t> takeU8() { return takeU<uint8_t>(); } + + std::optional<uint8_t> takeI8() { return takeI<uint8_t>(); } + + std::optional<double> takeF64() { + if (auto t = peek()) { + if (auto d = t->getF64()) { + advance(); + return d; + } + } + return std::nullopt; + } + + std::optional<float> takeF32() { + if (auto t = peek()) { + if (auto f = t->getF32()) { + advance(); + return f; + } + } + return std::nullopt; } - const Token& operator*() { return *curr; } - const Token* operator->() { return &*curr; } + std::optional<std::string> takeString() { + if (auto t = peek()) { + if (auto s = t->getString()) { + advance(); + return std::string(*s); + } + } + return {}; + } - bool operator==(const Lexer& other) const { - // The iterator is equal to the end sentinel when there is no current token. - if (!curr && !other.curr) { + std::optional<Name> takeName() { + // TODO: Move this to lexer and validate UTF. + if (auto str = takeString()) { + // Copy to a std::string to make sure we have a null terminator, otherwise + // the `Name` constructor won't work correctly. + // TODO: Update `Name` to use string_view instead of char* and/or to take + // rvalue strings to avoid this extra copy. + return Name(std::string(*str)); + } + return {}; + } + + bool takeSExprStart(std::string_view expected) { + auto original = *this; + if (takeLParen() && takeKeyword(expected)) { return true; } - // Otherwise they are equivalent when they are at the same position. - return index == other.index; + *this = original; + return false; } - bool operator!=(const Lexer& other) const { return !(*this == other); } + bool peekSExprStart(std::string_view expected) { + auto original = *this; + if (!takeLParen()) { + return false; + } + bool ret = takeKeyword(expected); + *this = original; + return ret; + } - Lexer begin() { return *this; } + std::string_view next() const { return buffer.substr(index); } - Lexer end() const { return Lexer(); } + void advance() { + skipSpace(); + lexToken(); + } - bool empty() const { return *this == end(); } + bool empty() const { return !curr; } TextPos position(const char* c) const; TextPos position(size_t i) const { return position(buffer.data() + i); } TextPos position(std::string_view span) const { return position(span.data()); } - TextPos position(Token tok) const { return position(tok.span); } + TextPos position() const { return position(getPos()); } + + size_t getPos() const { + if (auto t = peek()) { + return getIndex() - t->span.size(); + } + return getIndex(); + } private: void skipSpace(); diff --git a/test/gtest/wat-lexer.cpp b/test/gtest/wat-lexer.cpp index b62644682..8e31b9ab9 100644 --- a/test/gtest/wat-lexer.cpp +++ b/test/gtest/wat-lexer.cpp @@ -23,76 +23,57 @@ using namespace wasm::WATParser; using namespace std::string_view_literals; TEST(LexerTest, LexWhitespace) { - Token one{"1"sv, IntTok{1, NoSign}}; - Token two{"2"sv, IntTok{2, NoSign}}; - Token three{"3"sv, IntTok{3, NoSign}}; - Token four{"4"sv, IntTok{4, NoSign}}; - Token five{"5"sv, IntTok{5, NoSign}}; - Lexer lexer(" 1\t2\n3\r4 \n\n\t 5 "sv); - auto it = lexer.begin(); - ASSERT_NE(it, lexer.end()); - Token t1 = *it++; - ASSERT_NE(it, lexer.end()); - Token t2 = *it++; - ASSERT_NE(it, lexer.end()); - Token t3 = *it++; - ASSERT_NE(it, lexer.end()); - Token t4 = *it++; - ASSERT_NE(it, lexer.end()); - Token t5 = *it++; - EXPECT_EQ(it, lexer.end()); - - EXPECT_EQ(t1, one); - EXPECT_EQ(t2, two); - EXPECT_EQ(t3, three); - EXPECT_EQ(t4, four); - EXPECT_EQ(t5, five); - - EXPECT_EQ(lexer.position(t1), (TextPos{1, 1})); - EXPECT_EQ(lexer.position(t2), (TextPos{1, 3})); - EXPECT_EQ(lexer.position(t3), (TextPos{2, 0})); - EXPECT_EQ(lexer.position(t4), (TextPos{2, 2})); - EXPECT_EQ(lexer.position(t5), (TextPos{4, 2})); + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{1, 1})); + EXPECT_EQ(lexer.takeI32(), 1); + + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{1, 3})); + EXPECT_EQ(lexer.takeI32(), 2); + + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{2, 0})); + EXPECT_EQ(lexer.takeI32(), 3); + + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{2, 2})); + EXPECT_EQ(lexer.takeI32(), 4); + + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{4, 2})); + EXPECT_EQ(lexer.takeI32(), 5); + + ASSERT_TRUE(lexer.empty()); } TEST(LexerTest, LexLineComment) { - Token one{"1"sv, IntTok{1, NoSign}}; - Token six{"6"sv, IntTok{6, NoSign}}; - Lexer lexer("1;; whee! 2 3\t4\r5\n6"sv); - auto it = lexer.begin(); - Token t1 = *it++; - ASSERT_NE(it, lexer.end()); - Token t2 = *it++; - EXPECT_EQ(it, lexer.end()); + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{1, 0})); + EXPECT_EQ(lexer.takeI32(), 1); - EXPECT_EQ(t1, one); - EXPECT_EQ(t2, six); + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{2, 0})); + EXPECT_EQ(lexer.takeI32(), 6); - EXPECT_EQ(lexer.position(t1), (TextPos{1, 0})); - EXPECT_EQ(lexer.position(t2), (TextPos{2, 0})); + EXPECT_TRUE(lexer.empty()); } TEST(LexerTest, LexBlockComment) { - Token one{"1"sv, IntTok{1, NoSign}}; - Token six{"6"sv, IntTok{6, NoSign}}; - Lexer lexer("1(; whoo! 2\n (; \n3\n ;) 4 (;) 5 ;) \n;)6"sv); - auto it = lexer.begin(); - Token t1 = *it++; - ASSERT_NE(it, lexer.end()); - Token t2 = *it++; - EXPECT_EQ(it, lexer.end()); + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{1, 0})); + EXPECT_EQ(lexer.takeI32(), 1); - EXPECT_EQ(t1, one); - EXPECT_EQ(t2, six); + ASSERT_FALSE(lexer.empty()); + EXPECT_EQ(lexer.position(), (TextPos{5, 2})); + EXPECT_EQ(lexer.takeI32(), 6); - EXPECT_EQ(lexer.position(t1), (TextPos{1, 0})); - EXPECT_EQ(lexer.position(t2), (TextPos{5, 2})); + EXPECT_TRUE(lexer.empty()); } TEST(LexerTest, LexParens) { @@ -101,1446 +82,864 @@ TEST(LexerTest, LexParens) { Lexer lexer("(())"sv); - auto it = lexer.begin(); - ASSERT_NE(it, lexer.end()); - Token t1 = *it++; - ASSERT_NE(it, lexer.end()); - Token t2 = *it++; - ASSERT_NE(it, lexer.end()); - Token t3 = *it++; - ASSERT_NE(it, lexer.end()); - Token t4 = *it++; - EXPECT_EQ(it, lexer.end()); - - EXPECT_EQ(t1, left); - EXPECT_EQ(t2, left); - EXPECT_EQ(t3, right); - EXPECT_EQ(t4, right); - EXPECT_TRUE(left.isLParen()); - EXPECT_TRUE(right.isRParen()); + ASSERT_FALSE(lexer.empty()); + EXPECT_TRUE(lexer.takeLParen()); + + ASSERT_FALSE(lexer.empty()); + EXPECT_TRUE(lexer.takeLParen()); + + ASSERT_FALSE(lexer.empty()); + EXPECT_TRUE(lexer.takeRParen()); + + ASSERT_FALSE(lexer.empty()); + EXPECT_TRUE(lexer.takeRParen()); + + EXPECT_TRUE(lexer.empty()); } TEST(LexerTest, LexInt) { - { - Lexer lexer("0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0"sv, IntTok{0, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0"sv, IntTok{0, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0"sv, IntTok{0, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"1"sv, IntTok{1, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+1"sv, IntTok{1, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-1"sv, IntTok{-1ull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0010"sv, IntTok{10, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0010"sv, IntTok{10, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0010"sv, IntTok{-10ull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("9999"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"9999"sv, IntTok{9999, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+9999"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+9999"sv, IntTok{9999, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-9999"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-9999"sv, IntTok{-9999ull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("12_34"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"12_34"sv, IntTok{1234, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("1_2_3_4"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"1_2_3_4"sv, IntTok{1234, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("_1234"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("1234_"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("12__34"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("12cd56"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("18446744073709551615"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"18446744073709551615"sv, IntTok{-1ull, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - // 64-bit unsigned overflow! - Lexer lexer("18446744073709551616"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"18446744073709551616"sv, - FloatTok{{}, 18446744073709551616.}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+9223372036854775807"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+9223372036854775807"sv, IntTok{INT64_MAX, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+9223372036854775808"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+9223372036854775808"sv, - IntTok{uint64_t(INT64_MAX) + 1, Pos}}; - ; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-9223372036854775808"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-9223372036854775808"sv, IntTok{uint64_t(INT64_MIN), Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-9223372036854775809"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-9223372036854775809"sv, - IntTok{uint64_t(INT64_MIN) - 1, Neg}}; - EXPECT_EQ(*lexer, expected); - } + EXPECT_EQ(Lexer("0"sv).takeU8(), uint8_t(0)); + EXPECT_EQ(Lexer("0"sv).takeI8(), uint8_t(0)); + EXPECT_EQ(Lexer("0"sv).takeI16(), uint16_t(0)); + EXPECT_EQ(Lexer("0"sv).takeU32(), uint32_t(0)); + EXPECT_EQ(Lexer("0"sv).takeI32(), uint32_t(0)); + EXPECT_EQ(Lexer("0"sv).takeU64(), uint64_t(0)); + EXPECT_EQ(Lexer("0"sv).takeI64(), uint64_t(0)); + + EXPECT_FALSE(Lexer("+0"sv).takeU8()); + EXPECT_EQ(Lexer("+0"sv).takeI8(), uint8_t(0)); + + EXPECT_FALSE(Lexer("-0"sv).takeU8()); + EXPECT_EQ(Lexer("-0"sv).takeI8(), uint8_t(0)); + + EXPECT_EQ(Lexer("1"sv).takeU8(), uint8_t(1)); + EXPECT_EQ(Lexer("1"sv).takeI8(), uint8_t(1)); + + EXPECT_FALSE(Lexer("+1"sv).takeU8()); + EXPECT_EQ(Lexer("+1"sv).takeI8(), uint8_t(1)); + + EXPECT_FALSE(Lexer("-1"sv).takeU8()); + EXPECT_EQ(Lexer("-1").takeI8(), uint8_t(-1)); + + EXPECT_EQ(Lexer("0010"sv).takeU8(), uint8_t(10)); + EXPECT_EQ(Lexer("0010"sv).takeI8(), uint8_t(10)); + + EXPECT_FALSE(Lexer("+0010"sv).takeU8()); + EXPECT_EQ(Lexer("+0010"sv).takeI8(), uint8_t(10)); + + EXPECT_FALSE(Lexer("-0010"sv).takeU8()); + EXPECT_EQ(Lexer("-0010"sv).takeI8(), uint8_t(-10)); + + EXPECT_FALSE(Lexer("9999"sv).takeU8()); + EXPECT_EQ(Lexer("9999"sv).takeI16(), uint16_t(9999)); + EXPECT_EQ(Lexer("9999"sv).takeU32(), uint32_t(9999)); + EXPECT_EQ(Lexer("9999"sv).takeI32(), uint32_t(9999)); + + EXPECT_FALSE(Lexer("+9999"sv).takeU32()); + EXPECT_EQ(Lexer("+9999"sv).takeI32(), uint32_t(9999)); + + EXPECT_FALSE(Lexer("-9999"sv).takeU32()); + EXPECT_EQ(Lexer("-9999"sv).takeI32(), uint32_t(-9999)); + + EXPECT_EQ(Lexer("12_34"sv).takeU32(), uint32_t(1234)); + EXPECT_EQ(Lexer("12_34"sv).takeI32(), uint32_t(1234)); + + EXPECT_EQ(Lexer("1_2_3_4"sv).takeU32(), uint32_t(1234)); + EXPECT_EQ(Lexer("1_2_3_4"sv).takeI32(), uint32_t(1234)); + + EXPECT_FALSE(Lexer("_1234"sv).takeU32()); + EXPECT_FALSE(Lexer("_1234"sv).takeI32()); + + EXPECT_FALSE(Lexer("1234_"sv).takeU32()); + EXPECT_FALSE(Lexer("1234_"sv).takeI32()); + + EXPECT_FALSE(Lexer("12__34"sv).takeU32()); + EXPECT_FALSE(Lexer("12__34"sv).takeI32()); + + EXPECT_FALSE(Lexer("12cd56"sv).takeU32()); + EXPECT_FALSE(Lexer("12cd56"sv).takeI32()); + + EXPECT_EQ(Lexer("18446744073709551615"sv).takeU64(), uint64_t(-1)); + EXPECT_EQ(Lexer("18446744073709551615"sv).takeI64(), uint64_t(-1)); + + // 64-bit overflow! + EXPECT_FALSE(Lexer("18446744073709551616"sv).takeU64()); + EXPECT_FALSE(Lexer("18446744073709551616"sv).takeI64()); + + EXPECT_FALSE(Lexer("+9223372036854775807"sv).takeU64()); + EXPECT_EQ(Lexer("+9223372036854775807"sv).takeI64(), INT64_MAX); + + EXPECT_EQ(Lexer("9223372036854775808"sv).takeU64(), uint64_t(INT64_MAX) + 1); + EXPECT_EQ(Lexer("9223372036854775808"sv).takeI64(), uint64_t(INT64_MAX) + 1); + + EXPECT_FALSE(Lexer("+9223372036854775808"sv).takeU64()); + EXPECT_FALSE(Lexer("+9223372036854775808"sv).takeI64()); + + EXPECT_FALSE(Lexer("-9223372036854775808"sv).takeU64()); + EXPECT_EQ(Lexer("-9223372036854775808"sv).takeI64(), uint64_t(INT64_MIN)); + + EXPECT_FALSE(Lexer("-9223372036854775809"sv).takeU64()); + EXPECT_FALSE(Lexer("-9223372036854775809"sv).takeI64()); } TEST(LexerTest, LexHexInt) { - { - Lexer lexer("0x0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x0"sv, IntTok{0, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0x0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0x0"sv, IntTok{0, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0x0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0x0"sv, IntTok{0, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x1"sv, IntTok{1, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0x1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0x1"sv, IntTok{1, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0x1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0x1"sv, IntTok{-1ull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x0010"sv, IntTok{16, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0x0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0x0010"sv, IntTok{16, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0x0010"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0x0010"sv, IntTok{-16ull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0xabcdef"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0xabcdef"sv, IntTok{0xabcdef, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0xABCDEF"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0xABCDEF"sv, IntTok{0xabcdef, Pos}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0xAbCdEf"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0xAbCdEf"sv, IntTok{-0xabcdefull, Neg}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x12_34"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x12_34"sv, IntTok{0x1234, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x1_2_3_4"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x1_2_3_4"sv, IntTok{0x1234, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("_0x1234"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x_1234"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x1234_"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x12__34"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0xg"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x120x34"sv); - EXPECT_TRUE(lexer.empty()); - } + EXPECT_EQ(Lexer("0x0"sv).takeU8(), uint8_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeI8(), uint8_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeI16(), uint16_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeU32(), uint32_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeI32(), uint32_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeU64(), uint64_t(0)); + EXPECT_EQ(Lexer("0x0"sv).takeI64(), uint64_t(0)); + + EXPECT_FALSE(Lexer("+0x0"sv).takeU8()); + EXPECT_EQ(Lexer("+0x0"sv).takeI8(), uint8_t(0)); + + EXPECT_FALSE(Lexer("-0x0"sv).takeU8()); + EXPECT_EQ(Lexer("-0x0"sv).takeI8(), uint8_t(0)); + + EXPECT_EQ(Lexer("0x1"sv).takeU8(), uint8_t(1)); + EXPECT_EQ(Lexer("0x1"sv).takeI8(), uint8_t(1)); + + EXPECT_FALSE(Lexer("+0x1"sv).takeU8()); + EXPECT_EQ(Lexer("+0x1"sv).takeI8(), uint8_t(1)); + + EXPECT_FALSE(Lexer("-0x1"sv).takeU8()); + EXPECT_EQ(Lexer("-0x1").takeI8(), uint8_t(-1)); + + EXPECT_EQ(Lexer("0x0010"sv).takeU8(), uint8_t(16)); + EXPECT_EQ(Lexer("0x0010"sv).takeI8(), uint8_t(16)); + + EXPECT_FALSE(Lexer("+0x0010"sv).takeU8()); + EXPECT_EQ(Lexer("+0x0010"sv).takeI8(), uint8_t(16)); + + EXPECT_FALSE(Lexer("-0x0010"sv).takeU8()); + EXPECT_EQ(Lexer("-0x0010"sv).takeI8(), uint8_t(-16)); + + EXPECT_FALSE(Lexer("0xabcdef"sv).takeU8()); + EXPECT_EQ(Lexer("0xabcdef"sv).takeU32(), uint32_t(0xabcdef)); + EXPECT_EQ(Lexer("0xabcdef"sv).takeI32(), uint32_t(0xabcdef)); + + EXPECT_FALSE(Lexer("+0xABCDEF"sv).takeU32()); + EXPECT_EQ(Lexer("+0xABCDEF"sv).takeI32(), uint32_t(0xabcdef)); + + EXPECT_FALSE(Lexer("-0xAbCdEf"sv).takeU32()); + EXPECT_EQ(Lexer("-0xAbCdEf"sv).takeI32(), uint32_t(-0xabcdef)); + + EXPECT_EQ(Lexer("0x12_34"sv).takeU32(), uint32_t(0x1234)); + EXPECT_EQ(Lexer("0x12_34"sv).takeI32(), uint32_t(0x1234)); + + EXPECT_EQ(Lexer("0x1_2_3_4"sv).takeU32(), uint32_t(0x1234)); + EXPECT_EQ(Lexer("0x1_2_3_4"sv).takeI32(), uint32_t(0x1234)); + + EXPECT_FALSE(Lexer("_0x1234"sv).takeU32()); + EXPECT_FALSE(Lexer("_0x1234"sv).takeI32()); + + EXPECT_FALSE(Lexer("0x_1234"sv).takeU32()); + EXPECT_FALSE(Lexer("0x_1234"sv).takeI32()); + + EXPECT_FALSE(Lexer("0x1234_"sv).takeU32()); + EXPECT_FALSE(Lexer("0x1234_"sv).takeI32()); + + EXPECT_FALSE(Lexer("0x12__34"sv).takeU32()); + EXPECT_FALSE(Lexer("0x12__34"sv).takeI32()); + + EXPECT_FALSE(Lexer("0xg"sv).takeU32()); + EXPECT_FALSE(Lexer("0xg"sv).takeI32()); + + EXPECT_FALSE(Lexer("0x120x34"sv).takeU32()); + EXPECT_FALSE(Lexer("0x120x34"sv).takeI32()); } TEST(LexerTest, ClassifyInt) { - { - Lexer lexer("0"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - ASSERT_TRUE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0ull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0ull); - EXPECT_EQ(*lexer->getU<uint32_t>(), 0u); - EXPECT_EQ(*lexer->getS<int32_t>(), 0); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0u); - EXPECT_EQ(*lexer->getF64(), 0.0); - EXPECT_EQ(*lexer->getF32(), 0.0); - EXPECT_FALSE(std::signbit(*lexer->getF64())); - EXPECT_FALSE(std::signbit(*lexer->getF32())); - } - { - Lexer lexer("+0"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0ull); - EXPECT_EQ(*lexer->getS<int32_t>(), 0); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0u); - EXPECT_EQ(*lexer->getF64(), 0.0); - EXPECT_EQ(*lexer->getF32(), 0.0); - EXPECT_FALSE(std::signbit(*lexer->getF64())); - EXPECT_FALSE(std::signbit(*lexer->getF32())); - } - { - Lexer lexer("-0"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0ull); - EXPECT_EQ(*lexer->getS<int32_t>(), 0); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0u); - EXPECT_EQ(*lexer->getF64(), -0.0); - EXPECT_EQ(*lexer->getF32(), -0.0); - ASSERT_TRUE(std::signbit(*lexer->getF64())); - ASSERT_TRUE(std::signbit(*lexer->getF32())); - } - { - Lexer lexer("0x7fff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - ASSERT_TRUE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0x7fffffffull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0x7fffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x7fffffffull); - EXPECT_EQ(*lexer->getU<uint32_t>(), 0x7fffffffu); - EXPECT_EQ(*lexer->getS<int32_t>(), 0x7fffffff); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0x7fffffffu); - EXPECT_EQ(*lexer->getF64(), 0x7fffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0x7fffffff.p0f); - } - { - Lexer lexer("0x8000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - ASSERT_TRUE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0x80000000ull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0x80000000ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x80000000ull); - EXPECT_EQ(*lexer->getU<uint32_t>(), 0x80000000u); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0x80000000u); - EXPECT_EQ(*lexer->getF64(), 0x80000000.p0); - EXPECT_EQ(*lexer->getF32(), 0x80000000.p0f); - } - { - Lexer lexer("+0x7fff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0x7fffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x7fffffffull); - EXPECT_EQ(*lexer->getS<int32_t>(), 0x7fffffff); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0x7fffffffu); - EXPECT_EQ(*lexer->getF64(), 0x7fffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0x7fffffff.p0f); - } - { - Lexer lexer("+0x8000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0x80000000ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x80000000ull); - EXPECT_EQ(*lexer->getF64(), 0x80000000.p0); - EXPECT_EQ(*lexer->getF32(), 0x80000000.p0f); - } - { - Lexer lexer("-0x8000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - ASSERT_TRUE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), -0x80000000ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), -0x80000000ull); - EXPECT_EQ(*lexer->getS<int32_t>(), -0x7fffffffll - 1); - EXPECT_EQ(*lexer->getI<uint32_t>(), -0x80000000u); - EXPECT_EQ(*lexer->getF64(), -0x80000000.p0); - EXPECT_EQ(*lexer->getF32(), -0x80000000.p0f); - } - { - Lexer lexer("-0x8000_0001"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), -0x80000001ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), -0x80000001ull); - EXPECT_EQ(*lexer->getF64(), -0x80000001.p0); - EXPECT_EQ(*lexer->getF32(), -0x80000001.p0f); - } - { - Lexer lexer("0xffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - ASSERT_TRUE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - ASSERT_TRUE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0xffffffffull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0xffffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0xffffffffull); - EXPECT_EQ(*lexer->getU<uint32_t>(), 0xffffffffu); - EXPECT_EQ(*lexer->getI<uint32_t>(), 0xffffffffu); - EXPECT_EQ(*lexer->getF64(), 0xffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0xffffffff.p0f); - } - { - Lexer lexer("0x1_0000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0x100000000ull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0x100000000ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x100000000ull); - EXPECT_EQ(*lexer->getF64(), 0x100000000.p0); - EXPECT_EQ(*lexer->getF32(), 0x100000000.p0f); - } - { - Lexer lexer("+0xffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0xffffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0xffffffffull); - EXPECT_EQ(*lexer->getF64(), 0xffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0xffffffff.p0f); - } - { - Lexer lexer("+0x1_0000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0x100000000ll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x100000000ull); - EXPECT_EQ(*lexer->getF64(), 0x100000000.p0); - EXPECT_EQ(*lexer->getF32(), 0x100000000.p0f); - } - { - Lexer lexer("0x7fff_ffff_ffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0x7fffffffffffffffull); - EXPECT_EQ(*lexer->getS<int64_t>(), 0x7fffffffffffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x7fffffffffffffffull); - EXPECT_EQ(*lexer->getF64(), 0x7fffffffffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0x7fffffffffffffff.p0f); - } - { - Lexer lexer("+0x7fff_ffff_ffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), 0x7fffffffffffffffll); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0x7fffffffffffffffull); - EXPECT_EQ(*lexer->getF64(), 0x7fffffffffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0x7fffffffffffffff.p0f); - } - { - Lexer lexer("-0x8000_0000_0000_0000"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - ASSERT_TRUE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getS<int64_t>(), -0x7fffffffffffffffll - 1); - EXPECT_EQ(*lexer->getI<uint64_t>(), -0x8000000000000000ull); - EXPECT_EQ(*lexer->getF64(), -0x8000000000000000.p0); - EXPECT_EQ(*lexer->getF32(), -0x8000000000000000.p0f); - } - { - Lexer lexer("0xffff_ffff_ffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - ASSERT_TRUE(lexer->getU<uint64_t>()); - EXPECT_FALSE(lexer->getS<int64_t>()); - ASSERT_TRUE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getU<uint64_t>(), 0xffffffffffffffffull); - EXPECT_EQ(*lexer->getI<uint64_t>(), 0xffffffffffffffffull); - EXPECT_EQ(*lexer->getF64(), 0xffffffffffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0xffffffffffffffff.p0f); - } - { - Lexer lexer("+0xffff_ffff_ffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - - EXPECT_FALSE(lexer->getU<uint64_t>()); - EXPECT_FALSE(lexer->getS<int64_t>()); - EXPECT_FALSE(lexer->getI<uint64_t>()); - EXPECT_FALSE(lexer->getU<uint32_t>()); - EXPECT_FALSE(lexer->getS<int32_t>()); - EXPECT_FALSE(lexer->getI<uint32_t>()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - - EXPECT_EQ(*lexer->getF64(), 0xffffffffffffffff.p0); - EXPECT_EQ(*lexer->getF32(), 0xffffffffffffffff.p0f); - } + ASSERT_FALSE(Lexer("0"sv).empty()); + ASSERT_TRUE(Lexer("0"sv).takeU64()); + ASSERT_TRUE(Lexer("0"sv).takeI64()); + ASSERT_TRUE(Lexer("0"sv).takeU32()); + ASSERT_TRUE(Lexer("0"sv).takeI32()); + ASSERT_TRUE(Lexer("0"sv).takeF64()); + ASSERT_TRUE(Lexer("0"sv).takeF32()); + EXPECT_EQ(*Lexer("0"sv).takeU64(), 0ull); + EXPECT_EQ(*Lexer("0"sv).takeI64(), 0ull); + EXPECT_EQ(*Lexer("0"sv).takeU32(), 0u); + EXPECT_EQ(*Lexer("0"sv).takeI32(), 0u); + EXPECT_EQ(*Lexer("0"sv).takeF64(), 0.0); + EXPECT_EQ(*Lexer("0"sv).takeF32(), 0.0); + EXPECT_FALSE(std::signbit(*Lexer("0"sv).takeF64())); + EXPECT_FALSE(std::signbit(*Lexer("0"sv).takeF32())); + + ASSERT_FALSE(Lexer("+0"sv).empty()); + EXPECT_FALSE(Lexer("+0"sv).takeU64()); + ASSERT_TRUE(Lexer("+0"sv).takeI64()); + EXPECT_FALSE(Lexer("+0"sv).takeU32()); + ASSERT_TRUE(Lexer("+0"sv).takeI32()); + ASSERT_TRUE(Lexer("+0"sv).takeF64()); + ASSERT_TRUE(Lexer("+0"sv).takeF32()); + EXPECT_EQ(*Lexer("+0"sv).takeI64(), 0ull); + EXPECT_EQ(*Lexer("+0"sv).takeI32(), 0u); + EXPECT_EQ(*Lexer("+0"sv).takeF64(), 0.0); + EXPECT_EQ(*Lexer("+0"sv).takeF32(), 0.0); + EXPECT_FALSE(std::signbit(*Lexer("+0"sv).takeF64())); + EXPECT_FALSE(std::signbit(*Lexer("+0"sv).takeF32())); + + ASSERT_FALSE(Lexer("-0"sv).empty()); + EXPECT_FALSE(Lexer("-0"sv).takeU64()); + ASSERT_TRUE(Lexer("-0"sv).takeI64()); + EXPECT_FALSE(Lexer("-0"sv).takeU32()); + ASSERT_TRUE(Lexer("-0"sv).takeI32()); + ASSERT_TRUE(Lexer("-0"sv).takeF64()); + ASSERT_TRUE(Lexer("-0"sv).takeF32()); + EXPECT_EQ(*Lexer("-0"sv).takeI64(), 0ull); + EXPECT_EQ(*Lexer("-0"sv).takeI32(), 0u); + EXPECT_EQ(*Lexer("-0"sv).takeF64(), -0.0); + EXPECT_EQ(*Lexer("-0"sv).takeF32(), -0.0); + ASSERT_TRUE(std::signbit(*Lexer("-0"sv).takeF64())); + ASSERT_TRUE(std::signbit(*Lexer("-0"sv).takeF32())); + + ASSERT_FALSE(Lexer("0x7fff_ffff"sv).empty()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeI64()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeU32()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("0x7fff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeU64(), 0x7fffffffull); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeI64(), 0x7fffffffull); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeU32(), 0x7fffffffu); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeI32(), 0x7fffffffu); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeF64(), 0x7fffffff.p0); + EXPECT_EQ(*Lexer("0x7fff_ffff"sv).takeF32(), 0x7fffffff.p0f); + + ASSERT_FALSE(Lexer("0x8000_0000"sv).empty()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeI64()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeU32()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("0x8000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeU64(), 0x80000000ull); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeI64(), 0x80000000ull); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeU32(), 0x80000000u); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeI32(), 0x80000000u); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeF64(), 0x80000000.p0); + EXPECT_EQ(*Lexer("0x8000_0000"sv).takeF32(), 0x80000000.p0f); + + ASSERT_FALSE(Lexer("+0x7fff_ffff"sv).empty()); + EXPECT_FALSE(Lexer("+0x7fff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("+0x7fff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("+0x7fff_ffff"sv).takeU32()); + ASSERT_TRUE(Lexer("+0x7fff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("+0x7fff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("+0x7fff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("+0x7fff_ffff"sv).takeI64(), 0x7fffffffull); + EXPECT_EQ(*Lexer("+0x7fff_ffff"sv).takeI32(), 0x7fffffffu); + EXPECT_EQ(*Lexer("+0x7fff_ffff"sv).takeF64(), 0x7fffffff.p0); + EXPECT_EQ(*Lexer("+0x7fff_ffff"sv).takeF32(), 0x7fffffff.p0f); + + ASSERT_FALSE(Lexer("+0x8000_0000"sv).empty()); + EXPECT_FALSE(Lexer("+0x8000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("+0x8000_0000"sv).takeI64()); + EXPECT_FALSE(Lexer("+0x8000_0000"sv).takeU32()); + EXPECT_FALSE(Lexer("+0x8000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("+0x8000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("+0x8000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("+0x8000_0000"sv).takeI64(), 0x80000000ull); + EXPECT_EQ(*Lexer("+0x8000_0000"sv).takeF64(), 0x80000000.p0); + EXPECT_EQ(*Lexer("+0x8000_0000"sv).takeF32(), 0x80000000.p0f); + + ASSERT_FALSE(Lexer("-0x8000_0000"sv).empty()); + EXPECT_FALSE(Lexer("-0x8000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("-0x8000_0000"sv).takeI64()); + EXPECT_FALSE(Lexer("-0x8000_0000"sv).takeU32()); + ASSERT_TRUE(Lexer("-0x8000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("-0x8000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("-0x8000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("-0x8000_0000"sv).takeI64(), -0x80000000ull); + EXPECT_EQ(*Lexer("-0x8000_0000"sv).takeI32(), -0x80000000u); + EXPECT_EQ(*Lexer("-0x8000_0000"sv).takeF64(), -0x80000000.p0); + EXPECT_EQ(*Lexer("-0x8000_0000"sv).takeF32(), -0x80000000.p0f); + + ASSERT_FALSE(Lexer("-0x8000_0001"sv).empty()); + EXPECT_FALSE(Lexer("-0x8000_0001"sv).takeU64()); + ASSERT_TRUE(Lexer("-0x8000_0001"sv).takeI64()); + EXPECT_FALSE(Lexer("-0x8000_0001"sv).takeU32()); + EXPECT_FALSE(Lexer("-0x8000_0001"sv).takeI32()); + ASSERT_TRUE(Lexer("-0x8000_0001"sv).takeF64()); + ASSERT_TRUE(Lexer("-0x8000_0001"sv).takeF32()); + EXPECT_EQ(*Lexer("-0x8000_0001"sv).takeI64(), -0x80000001ull); + EXPECT_EQ(*Lexer("-0x8000_0001"sv).takeF64(), -0x80000001.p0); + EXPECT_EQ(*Lexer("-0x8000_0001"sv).takeF32(), -0x80000001.p0f); + + ASSERT_FALSE(Lexer("0xffff_ffff"sv).empty()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeI64()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeU32()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("0xffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeU64(), 0xffffffffull); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeI64(), 0xffffffffull); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeU32(), 0xffffffffu); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeI32(), 0xffffffffu); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeF64(), 0xffffffff.p0); + EXPECT_EQ(*Lexer("0xffff_ffff"sv).takeF32(), 0xffffffff.p0f); + + ASSERT_FALSE(Lexer("0x1_0000_0000"sv).empty()); + ASSERT_TRUE(Lexer("0x1_0000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("0x1_0000_0000"sv).takeI64()); + EXPECT_FALSE(Lexer("0x1_0000_0000"sv).takeU32()); + EXPECT_FALSE(Lexer("0x1_0000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("0x1_0000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("0x1_0000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("0x1_0000_0000"sv).takeU64(), 0x100000000ull); + EXPECT_EQ(*Lexer("0x1_0000_0000"sv).takeI64(), 0x100000000ull); + EXPECT_EQ(*Lexer("0x1_0000_0000"sv).takeF64(), 0x100000000.p0); + EXPECT_EQ(*Lexer("0x1_0000_0000"sv).takeF32(), 0x100000000.p0f); + + ASSERT_FALSE(Lexer("+0xffff_ffff"sv).empty()); + EXPECT_FALSE(Lexer("+0xffff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("+0xffff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("+0xffff_ffff"sv).takeU32()); + EXPECT_FALSE(Lexer("+0xffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("+0xffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("+0xffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("+0xffff_ffff"sv).takeI64(), 0xffffffffull); + EXPECT_EQ(*Lexer("+0xffff_ffff"sv).takeF64(), 0xffffffff.p0); + EXPECT_EQ(*Lexer("+0xffff_ffff"sv).takeF32(), 0xffffffff.p0f); + + ASSERT_FALSE(Lexer("+0x1_0000_0000"sv).empty()); + EXPECT_FALSE(Lexer("+0x1_0000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("+0x1_0000_0000"sv).takeI64()); + EXPECT_FALSE(Lexer("+0x1_0000_0000"sv).takeU32()); + EXPECT_FALSE(Lexer("+0x1_0000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("+0x1_0000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("+0x1_0000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("+0x1_0000_0000"sv).takeI64(), 0x100000000ull); + EXPECT_EQ(*Lexer("+0x1_0000_0000"sv).takeF64(), 0x100000000.p0); + EXPECT_EQ(*Lexer("+0x1_0000_0000"sv).takeF32(), 0x100000000.p0f); + + ASSERT_FALSE(Lexer("0x7fff_ffff_ffff_ffff"sv).empty()); + ASSERT_TRUE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeU32()); + EXPECT_FALSE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("0x7fff_ffff_ffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("0x7fff_ffff_ffff_ffff"sv).takeU64(), 0x7fffffffffffffffull); + EXPECT_EQ(*Lexer("0x7fff_ffff_ffff_ffff"sv).takeI64(), 0x7fffffffffffffffull); + EXPECT_EQ(*Lexer("0x7fff_ffff_ffff_ffff"sv).takeF64(), 0x7fffffffffffffff.p0); + EXPECT_EQ(*Lexer("0x7fff_ffff_ffff_ffff"sv).takeF32(), + 0x7fffffffffffffff.p0f); + + ASSERT_FALSE(Lexer("+0x7fff_ffff_ffff_ffff"sv).empty()); + EXPECT_FALSE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeU32()); + EXPECT_FALSE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("+0x7fff_ffff_ffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("+0x7fff_ffff_ffff_ffff"sv).takeI64(), + 0x7fffffffffffffffull); + EXPECT_EQ(*Lexer("+0x7fff_ffff_ffff_ffff"sv).takeF64(), + 0x7fffffffffffffff.p0); + EXPECT_EQ(*Lexer("+0x7fff_ffff_ffff_ffff"sv).takeF32(), + 0x7fffffffffffffff.p0f); + + ASSERT_FALSE(Lexer("-0x8000_0000_0000_0000"sv).empty()); + EXPECT_FALSE(Lexer("-0x8000_0000_0000_0000"sv).takeU64()); + ASSERT_TRUE(Lexer("-0x8000_0000_0000_0000"sv).takeI64()); + EXPECT_FALSE(Lexer("-0x8000_0000_0000_0000"sv).takeU32()); + EXPECT_FALSE(Lexer("-0x8000_0000_0000_0000"sv).takeI32()); + ASSERT_TRUE(Lexer("-0x8000_0000_0000_0000"sv).takeF64()); + ASSERT_TRUE(Lexer("-0x8000_0000_0000_0000"sv).takeF32()); + EXPECT_EQ(*Lexer("-0x8000_0000_0000_0000"sv).takeI64(), + -0x8000000000000000ull); + EXPECT_EQ(*Lexer("-0x8000_0000_0000_0000"sv).takeF64(), + -0x8000000000000000.p0); + EXPECT_EQ(*Lexer("-0x8000_0000_0000_0000"sv).takeF32(), + -0x8000000000000000.p0f); + + ASSERT_FALSE(Lexer("0xffff_ffff_ffff_ffff"sv).empty()); + ASSERT_TRUE(Lexer("0xffff_ffff_ffff_ffff"sv).takeU64()); + ASSERT_TRUE(Lexer("0xffff_ffff_ffff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("0xffff_ffff_ffff_ffff"sv).takeU32()); + EXPECT_FALSE(Lexer("0xffff_ffff_ffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("0xffff_ffff_ffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("0xffff_ffff_ffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("0xffff_ffff_ffff_ffff"sv).takeU64(), 0xffffffffffffffffull); + EXPECT_EQ(*Lexer("0xffff_ffff_ffff_ffff"sv).takeI64(), 0xffffffffffffffffull); + EXPECT_EQ(*Lexer("0xffff_ffff_ffff_ffff"sv).takeF64(), 0xffffffffffffffff.p0); + EXPECT_EQ(*Lexer("0xffff_ffff_ffff_ffff"sv).takeF32(), + 0xffffffffffffffff.p0f); + + ASSERT_FALSE(Lexer("+0xffff_ffff_ffff_ffff"sv).empty()); + EXPECT_FALSE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeU64()); + EXPECT_FALSE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeI64()); + EXPECT_FALSE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeU32()); + EXPECT_FALSE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeI32()); + ASSERT_TRUE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeF64()); + ASSERT_TRUE(Lexer("+0xffff_ffff_ffff_ffff"sv).takeF32()); + EXPECT_EQ(*Lexer("+0xffff_ffff_ffff_ffff"sv).takeF64(), + 0xffffffffffffffff.p0); + EXPECT_EQ(*Lexer("+0xffff_ffff_ffff_ffff"sv).takeF32(), + 0xffffffffffffffff.p0f); } TEST(LexerTest, LexFloat) { - { - Lexer lexer("42"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42"sv, IntTok{42, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42."sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42."sv, FloatTok{{}, 42.}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.5"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.5"sv, FloatTok{{}, 42.5}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42e0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42e0"sv, FloatTok{{}, 42e0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.e1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.e1"sv, FloatTok{{}, 42.e1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42E1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42E1"sv, FloatTok{{}, 42E1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42e+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42e+2"sv, FloatTok{{}, 42e+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.E-02"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.E-02"sv, FloatTok{{}, 42.E-02}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.0e0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.0e0"sv, FloatTok{{}, 42.0e0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.0E1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.0E1"sv, FloatTok{{}, 42.0E1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.0e+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.0e+2"sv, FloatTok{{}, 42.0e+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("42.0E-2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"42.0E-2"sv, FloatTok{{}, 42.0E-2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+42.0e+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+42.0e+2"sv, FloatTok{{}, +42.0e+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-42.0e+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-42.0e+2"sv, FloatTok{{}, -42.0e+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("4_2.0_0e+0_2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"4_2.0_0e+0_2"sv, FloatTok{{}, 42.00e+02}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42.junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42.0junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42.Ejunk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42.e-junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42.e-10junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("+"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42e"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42eABC"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42e0xABC"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("+-42"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("-+42"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42e+-0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42e-+0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42p0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("42P0"sv); - EXPECT_TRUE(lexer.empty()); - } + EXPECT_EQ(Lexer("42"sv).takeF32(), 42.0f); + EXPECT_EQ(Lexer("42"sv).takeF64(), 42.0); + + EXPECT_EQ(Lexer("42.5"sv).takeF32(), 42.5f); + EXPECT_EQ(Lexer("42.5"sv).takeF64(), 42.5); + + EXPECT_EQ(Lexer("42e0"sv).takeF32(), 42e0f); + EXPECT_EQ(Lexer("42e0"sv).takeF64(), 42e0); + + EXPECT_EQ(Lexer("42.e1"sv).takeF32(), 42.e1f); + EXPECT_EQ(Lexer("42.e1"sv).takeF64(), 42.e1); + + EXPECT_EQ(Lexer("42E1"sv).takeF32(), 42E1f); + EXPECT_EQ(Lexer("42E1"sv).takeF64(), 42E1); + + EXPECT_EQ(Lexer("42e+2"sv).takeF32(), 42e+2f); + EXPECT_EQ(Lexer("42e+2"sv).takeF64(), 42e+2); + + EXPECT_EQ(Lexer("42.E-02"sv).takeF32(), 42.E-02f); + EXPECT_EQ(Lexer("42.E-02"sv).takeF64(), 42.E-02); + + EXPECT_EQ(Lexer("42.0e0"sv).takeF32(), 42.0e0f); + EXPECT_EQ(Lexer("42.0e0"sv).takeF64(), 42.0e0); + + EXPECT_EQ(Lexer("42.0E1"sv).takeF32(), 42.0E1f); + EXPECT_EQ(Lexer("42.0E1"sv).takeF64(), 42.0E1); + + EXPECT_EQ(Lexer("42.0e+2"sv).takeF32(), 42.0e+2f); + EXPECT_EQ(Lexer("42.0e+2"sv).takeF64(), 42.0e+2); + + EXPECT_EQ(Lexer("42.0E-2"sv).takeF32(), 42.0E-2f); + EXPECT_EQ(Lexer("42.0E-2"sv).takeF64(), 42.0E-2); + + EXPECT_EQ(Lexer("+42.0e+2"sv).takeF32(), +42.0e+2f); + EXPECT_EQ(Lexer("+42.0e+2"sv).takeF64(), +42.0e+2); + + EXPECT_EQ(Lexer("-42.0e+2"sv).takeF32(), -42.0e+2f); + EXPECT_EQ(Lexer("-42.0e+2"sv).takeF64(), -42.0e+2); + + EXPECT_EQ(Lexer("4_2.0_0e+0_2"sv).takeF32(), 42.00e+02f); + EXPECT_EQ(Lexer("4_2.0_0e+0_2"sv).takeF64(), 42.00e+02); + + EXPECT_FALSE(Lexer("+junk"sv).takeF32()); + EXPECT_FALSE(Lexer("+junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42junk"sv).takeF32()); + EXPECT_FALSE(Lexer("42junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42.junk"sv).takeF32()); + EXPECT_FALSE(Lexer("42.junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42.0junk"sv).takeF32()); + EXPECT_FALSE(Lexer("42.0junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42.Ejunk"sv).takeF32()); + EXPECT_FALSE(Lexer("42.Ejunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42.e-junk"sv).takeF32()); + EXPECT_FALSE(Lexer("42.e-junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("42.e-10junk"sv).takeF32()); + EXPECT_FALSE(Lexer("42.e-10junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("+"sv).takeF32()); + EXPECT_FALSE(Lexer("+"sv).takeF64()); + + EXPECT_FALSE(Lexer("42e"sv).takeF32()); + EXPECT_FALSE(Lexer("42e"sv).takeF64()); + + EXPECT_FALSE(Lexer("42eABC"sv).takeF32()); + EXPECT_FALSE(Lexer("42eABC"sv).takeF64()); + + EXPECT_FALSE(Lexer("42e0xABC"sv).takeF32()); + EXPECT_FALSE(Lexer("42e0xABC"sv).takeF64()); + + EXPECT_FALSE(Lexer("+-42"sv).takeF32()); + EXPECT_FALSE(Lexer("+-42"sv).takeF64()); + + EXPECT_FALSE(Lexer("-+42"sv).takeF32()); + EXPECT_FALSE(Lexer("-+42"sv).takeF64()); + + EXPECT_FALSE(Lexer("42e+-0"sv).takeF32()); + EXPECT_FALSE(Lexer("42e+-0"sv).takeF64()); + + EXPECT_FALSE(Lexer("42e-+0"sv).takeF32()); + EXPECT_FALSE(Lexer("42e-+0"sv).takeF64()); + + EXPECT_FALSE(Lexer("42p0"sv).takeF32()); + EXPECT_FALSE(Lexer("42p0"sv).takeF64()); + + EXPECT_FALSE(Lexer("42P0"sv).takeF32()); + EXPECT_FALSE(Lexer("42P0"sv).takeF64()); } TEST(LexerTest, LexHexFloat) { - { - Lexer lexer("0x4B"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B"sv, IntTok{0x4B, NoSign}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B."sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B."sv, FloatTok{{}, 0x4Bp0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.5"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.5"sv, FloatTok{{}, 0x4B.5p0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4Bp0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4Bp0"sv, FloatTok{{}, 0x4Bp0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.p1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.p1"sv, FloatTok{{}, 0x4B.p1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4BP1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4BP1"sv, FloatTok{{}, 0x4BP1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4Bp+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4Bp+2"sv, FloatTok{{}, 0x4Bp+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.P-02"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.P-02"sv, FloatTok{{}, 0x4B.P-02}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.0p0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.0p0"sv, FloatTok{{}, 0x4B.0p0}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.0P1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.0P1"sv, FloatTok{{}, 0x4B.0P1}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.0p+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.0p+2"sv, FloatTok{{}, 0x4B.0p+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4B.0P-2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4B.0P-2"sv, FloatTok{{}, 0x4B.0P-2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+0x4B.0p+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+0x4B.0p+2"sv, FloatTok{{}, +0x4B.0p+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-0x4B.0p+2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-0x4B.0p+2"sv, FloatTok{{}, -0x4B.0p+2}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4_2.0_0p+0_2"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"0x4_2.0_0p+0_2"sv, FloatTok{{}, 0x42.00p+02}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("0x4Bjunk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.0junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.Pjunk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.p-junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.p-10junk"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("+0x"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4Bp"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4BpABC"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4Bp0xABC"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x+0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("+-0x4B"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("-+0x4B"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4Bp+-0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4Bp-+0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.e+0"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("0x4B.E-0"sv); - EXPECT_TRUE(lexer.empty()); - } + + EXPECT_EQ(Lexer("0x4B"sv).takeF32(), 0x4Bp0f); + EXPECT_EQ(Lexer("0x4B"sv).takeF64(), 0x4Bp0); + + EXPECT_EQ(Lexer("0x4B."sv).takeF32(), 0x4B.p0f); + EXPECT_EQ(Lexer("0x4B."sv).takeF64(), 0x4B.p0); + + EXPECT_EQ(Lexer("0x4B.5"sv).takeF32(), 0x4B.5p0f); + EXPECT_EQ(Lexer("0x4B.5"sv).takeF64(), 0x4B.5p0); + + EXPECT_EQ(Lexer("0x4Bp0"sv).takeF32(), 0x4Bp0f); + EXPECT_EQ(Lexer("0x4Bp0"sv).takeF64(), 0x4Bp0); + + EXPECT_EQ(Lexer("0x4B.p1"sv).takeF32(), 0x4B.p1f); + EXPECT_EQ(Lexer("0x4B.p1"sv).takeF64(), 0x4B.p1); + + EXPECT_EQ(Lexer("0x4BP1"sv).takeF32(), 0x4BP1f); + EXPECT_EQ(Lexer("0x4BP1"sv).takeF64(), 0x4BP1); + + EXPECT_EQ(Lexer("0x4Bp+2"sv).takeF32(), 0x4Bp+2f); + EXPECT_EQ(Lexer("0x4Bp+2"sv).takeF64(), 0x4Bp+2); + + EXPECT_EQ(Lexer("0x4B.P-02"sv).takeF32(), 0x4B.P-02f); + EXPECT_EQ(Lexer("0x4B.P-02"sv).takeF64(), 0x4B.P-02); + + EXPECT_EQ(Lexer("0x4B.0p0"sv).takeF32(), 0x4B.0p0f); + EXPECT_EQ(Lexer("0x4B.0p0"sv).takeF64(), 0x4B.0p0); + + EXPECT_EQ(Lexer("0x4B.0P1"sv).takeF32(), 0x4B.0P1f); + EXPECT_EQ(Lexer("0x4B.0P1"sv).takeF64(), 0x4B.0P1); + + EXPECT_EQ(Lexer("0x4B.0p+2"sv).takeF32(), 0x4B.0p+2f); + EXPECT_EQ(Lexer("0x4B.0p+2"sv).takeF64(), 0x4B.0p+2); + + EXPECT_EQ(Lexer("0x4B.0P-2"sv).takeF32(), 0x4B.0P-2f); + EXPECT_EQ(Lexer("0x4B.0P-2"sv).takeF64(), 0x4B.0P-2); + + EXPECT_EQ(Lexer("+0x4B.0p+2"sv).takeF32(), +0x4B.0p+2f); + EXPECT_EQ(Lexer("+0x4B.0p+2"sv).takeF64(), +0x4B.0p+2); + + EXPECT_EQ(Lexer("-0x4B.0p+2"sv).takeF32(), -0x4B.0p+2f); + EXPECT_EQ(Lexer("-0x4B.0p+2"sv).takeF64(), -0x4B.0p+2); + + EXPECT_EQ(Lexer("0x4_2.0_0p+0_2"sv).takeF32(), 0x42.00p+02f); + EXPECT_EQ(Lexer("0x4_2.0_0p+0_2"sv).takeF64(), 0x42.00p+02); + + EXPECT_FALSE(Lexer("0x4Bjunk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4Bjunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.junk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.0junk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.0junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.Pjunk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.Pjunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.p-junk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.p-junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.p-10junk"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.p-10junk"sv).takeF64()); + + EXPECT_FALSE(Lexer("+0x"sv).takeF32()); + EXPECT_FALSE(Lexer("+0x"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4Bp"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4Bp"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4BpABC"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4BpABC"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4Bp0xABC"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4Bp0xABC"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x+0"sv).takeF32()); + EXPECT_FALSE(Lexer("0x+0"sv).takeF64()); + + EXPECT_FALSE(Lexer("+-0x4B"sv).takeF32()); + EXPECT_FALSE(Lexer("+-0x4B"sv).takeF64()); + + EXPECT_FALSE(Lexer("-+0x4B"sv).takeF32()); + EXPECT_FALSE(Lexer("-+0x4B"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4Bp+-0"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4Bp+-0"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4Bp-+0"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4Bp-+0"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.e+0"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.e+0"sv).takeF64()); + + EXPECT_FALSE(Lexer("0x4B.E-0"sv).takeF32()); + EXPECT_FALSE(Lexer("0x4B.E-0"sv).takeF64()); } TEST(LexerTest, LexInfinity) { - { - Lexer lexer("inf"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"inf"sv, FloatTok{{}, INFINITY}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+inf"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+inf"sv, FloatTok{{}, INFINITY}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-inf"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-inf"sv, FloatTok{{}, -INFINITY}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("infjunk"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"infjunk"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("Inf"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("INF"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("infinity"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"infinity"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } + EXPECT_EQ(Lexer("inf"sv).takeF32(), INFINITY); + EXPECT_EQ(Lexer("inf"sv).takeF64(), INFINITY); + + EXPECT_EQ(Lexer("+inf"sv).takeF32(), INFINITY); + EXPECT_EQ(Lexer("+inf"sv).takeF64(), INFINITY); + + EXPECT_EQ(Lexer("-inf"sv).takeF32(), -INFINITY); + EXPECT_EQ(Lexer("-inf"sv).takeF64(), -INFINITY); + + EXPECT_FALSE(Lexer("infjunk"sv).takeF32()); + EXPECT_FALSE(Lexer("infjunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("Inf"sv).takeF32()); + EXPECT_FALSE(Lexer("Inf"sv).takeF64()); + + EXPECT_FALSE(Lexer("INF"sv).takeF32()); + EXPECT_FALSE(Lexer("INF"sv).takeF64()); + + EXPECT_FALSE(Lexer("infinity"sv).takeF32()); + EXPECT_FALSE(Lexer("infinity"sv).takeF64()); } TEST(LexerTest, LexNan) { - const double posNan = std::copysign(NAN, 1.0); - const double negNan = std::copysign(NAN, -1.0); - { - Lexer lexer("nan"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan"sv, FloatTok{{}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+nan"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+nan"sv, FloatTok{{}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-nan"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-nan"sv, FloatTok{{}, negNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x01"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x01"sv, FloatTok{{1}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("+nan:0x01"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"+nan:0x01"sv, FloatTok{{1}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("-nan:0x01"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"-nan:0x01"sv, FloatTok{{1}, negNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x1234"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x1234"sv, FloatTok{{0x1234}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0xf_ffff_ffff_ffff"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0xf_ffff_ffff_ffff"sv, - FloatTok{{0xfffffffffffff}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nanjunk"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nanjunk", KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0xjunk"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0xjunk"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:-0x1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:-0x1"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:+0x1"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:+0x1"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x0"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x0"sv, FloatTok{{0}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x10_0000_0000_0000"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x10_0000_0000_0000"sv, - FloatTok{{0x10000000000000}, posNan}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("nan:0x1_0000_0000_0000_0000"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"nan:0x1_0000_0000_0000_0000"sv, KeywordTok{}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("NAN"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("NaN"sv); - EXPECT_TRUE(lexer.empty()); - } + ASSERT_TRUE(Lexer("nan"sv).takeF32()); + ASSERT_TRUE(Lexer("nan"sv).takeF64()); + + ASSERT_TRUE(Lexer("+nan"sv).takeF32()); + ASSERT_TRUE(Lexer("+nan"sv).takeF64()); + + ASSERT_TRUE(Lexer("-nan"sv).takeF32()); + ASSERT_TRUE(Lexer("-nan"sv).takeF64()); + + ASSERT_TRUE(Lexer("nan:0x01"sv).takeF32()); + ASSERT_TRUE(Lexer("nan:0x01"sv).takeF64()); + + ASSERT_TRUE(Lexer("+nan:0x01"sv).takeF32()); + ASSERT_TRUE(Lexer("+nan:0x01"sv).takeF64()); + + ASSERT_TRUE(Lexer("-nan:0x01"sv).takeF64()); + ASSERT_TRUE(Lexer("-nan:0x01"sv).takeF64()); + + ASSERT_TRUE(Lexer("nan:0x1234"sv).takeF64()); + ASSERT_TRUE(Lexer("nan:0x1234"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0xf_ffff_ffff_ffff"sv).takeF32()); + EXPECT_TRUE(Lexer("nan:0xf_ffff_ffff_ffff"sv).takeF64()); + + EXPECT_FALSE(Lexer("nanjunk"sv).takeF32()); + EXPECT_FALSE(Lexer("nanjunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0x"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:0x"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0xjunk"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:0xjunk"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:-0x1"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:-0x1"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:+0x1"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:+0x1"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0x0"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:0x0"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0x10_0000_0000_0000"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:0x10_0000_0000_0000"sv).takeF64()); + + EXPECT_FALSE(Lexer("nan:0x1_0000_0000_0000_0000"sv).takeF32()); + EXPECT_FALSE(Lexer("nan:0x1_0000_0000_0000_0000"sv).takeF64()); + + EXPECT_FALSE(Lexer("NAN"sv).takeF32()); + EXPECT_FALSE(Lexer("NAN"sv).takeF64()); + + EXPECT_FALSE(Lexer("NaN"sv).takeF32()); + EXPECT_FALSE(Lexer("NaN"sv).takeF64()); } -TEST(LexerTest, ClassifyFloat) { - constexpr int signif64 = 52; - constexpr int signif32 = 23; - constexpr uint64_t payloadMask64 = (1ull << signif64) - 1; - constexpr uint32_t payloadMask32 = (1u << signif32) - 1; - constexpr uint64_t dnanDefault = 1ull << (signif64 - 1); - constexpr uint32_t fnanDefault = 1u << (signif32 - 1); - { - Lexer lexer("340282346638528859811704183484516925440."sv); - ASSERT_FALSE(lexer.empty()); - ASSERT_TRUE(lexer->getF64()); - EXPECT_TRUE(lexer->getF32()); - EXPECT_EQ(*lexer->getF64(), FLT_MAX); - EXPECT_EQ(*lexer->getF32(), FLT_MAX); - } - { - Lexer lexer("17976931348623157081452742373170435679807056752584499659891747" - "68031572607800285387605895586327668781715404589535143824642343" - "21326889464182768467546703537516986049910576551282076245490090" - "38932894407586850845513394230458323690322294816580855933212334" - "8274797826204144723168738177180919299881250404026184124858368" - "."sv); - ASSERT_FALSE(lexer.empty()); - ASSERT_TRUE(lexer->getF64()); - ASSERT_TRUE(lexer->getF32()); - EXPECT_EQ(*lexer->getF64(), DBL_MAX); - EXPECT_EQ(*lexer->getF32(), INFINITY); - } - { - Lexer lexer("nan"); - ASSERT_FALSE(lexer.empty()); +constexpr int signif32 = 23; +constexpr int signif64 = 52; - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); - EXPECT_TRUE(std::isnan(d)); - EXPECT_FALSE(std::signbit(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, dnanDefault); +uint32_t payload(float f) { + uint32_t x; + static_assert(sizeof(f) == sizeof(x)); + memcpy(&x, &f, sizeof(f)); + return x & ((1u << signif32) - 1); +} + +uint64_t payload(double d) { + uint64_t x; + static_assert(sizeof(d) == sizeof(x)); + memcpy(&x, &d, sizeof(d)); + return x & ((1ull << signif64) - 1); +} - ASSERT_TRUE(lexer->getF32()); - float f = *lexer->getF32(); +constexpr uint32_t fnanDefault = 1u << (signif32 - 1); +constexpr uint64_t dnanDefault = 1ull << (signif64 - 1); + +TEST(LexerTest, ClassifyFloat) { + auto flt_max = "340282346638528859811704183484516925440."sv; + EXPECT_EQ(Lexer(flt_max).takeF32(), FLT_MAX); + EXPECT_EQ(Lexer(flt_max).takeF64(), FLT_MAX); + + auto dbl_max = + "17976931348623157081452742373170435679807056752584499659891747" + "68031572607800285387605895586327668781715404589535143824642343" + "21326889464182768467546703537516986049910576551282076245490090" + "38932894407586850845513394230458323690322294816580855933212334" + "8274797826204144723168738177180919299881250404026184124858368" + "."sv; + EXPECT_EQ(Lexer(dbl_max).takeF32(), INFINITY); + EXPECT_EQ(Lexer(dbl_max).takeF64(), DBL_MAX); + + { + auto nan = "nan"sv; + ASSERT_TRUE(Lexer(nan).takeF32()); + float f = *Lexer(nan).takeF32(); EXPECT_TRUE(std::isnan(f)); EXPECT_FALSE(std::signbit(f)); - uint32_t fbits; - memcpy(&fbits, &f, sizeof(fbits)); - EXPECT_EQ(fbits & payloadMask32, fnanDefault); + EXPECT_EQ(payload(f), fnanDefault); + + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); + EXPECT_TRUE(std::isnan(d)); + EXPECT_FALSE(std::signbit(d)); + EXPECT_EQ(payload(d), dnanDefault); } { - Lexer lexer("-nan"); - ASSERT_FALSE(lexer.empty()); + auto nan = "-nan"sv; + ASSERT_TRUE(Lexer(nan).takeF32()); + float f = *Lexer(nan).takeF32(); + EXPECT_TRUE(std::isnan(f)); + EXPECT_TRUE(std::signbit(f)); + EXPECT_EQ(payload(f), fnanDefault); - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); EXPECT_TRUE(std::isnan(d)); EXPECT_TRUE(std::signbit(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, dnanDefault); - - ASSERT_TRUE(lexer->getF32()); - float f = *lexer->getF32(); - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::signbit(f)); - uint32_t fbits; - memcpy(&fbits, &f, sizeof(fbits)); - EXPECT_EQ(fbits & payloadMask32, fnanDefault); + EXPECT_EQ(payload(d), dnanDefault); } { - Lexer lexer("+nan"); - ASSERT_FALSE(lexer.empty()); + auto nan = "+nan"sv; + ASSERT_TRUE(Lexer(nan).takeF32()); + float f = *Lexer(nan).takeF32(); + EXPECT_TRUE(std::isnan(f)); + EXPECT_FALSE(std::signbit(f)); + EXPECT_EQ(payload(f), fnanDefault); - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); EXPECT_TRUE(std::isnan(d)); EXPECT_FALSE(std::signbit(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, dnanDefault); - - ASSERT_TRUE(lexer->getF32()); - float f = *lexer->getF32(); - EXPECT_TRUE(std::isnan(f)); - EXPECT_FALSE(std::signbit(f)); - uint32_t fbits; - memcpy(&fbits, &f, sizeof(fbits)); - EXPECT_EQ(fbits & payloadMask32, fnanDefault); + EXPECT_EQ(payload(d), dnanDefault); } { - Lexer lexer("nan:0x1234"); - ASSERT_FALSE(lexer.empty()); + auto nan = "nan:0x1234"sv; + ASSERT_TRUE(Lexer(nan).takeF32()); + float f = *Lexer(nan).takeF32(); + EXPECT_TRUE(std::isnan(f)); + EXPECT_FALSE(std::signbit(f)); + EXPECT_EQ(payload(f), uint32_t(0x1234)); - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); EXPECT_TRUE(std::isnan(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, 0x1234ull); - - ASSERT_TRUE(lexer->getF32()); - float f = *lexer->getF32(); - EXPECT_TRUE(std::isnan(f)); - uint32_t fbits; - memcpy(&fbits, &f, sizeof(fbits)); - EXPECT_EQ(fbits & payloadMask32, 0x1234u); + EXPECT_FALSE(std::signbit(d)); + EXPECT_EQ(payload(d), uint64_t(0x1234)); } { - Lexer lexer("nan:0x7FFFFF"); - ASSERT_FALSE(lexer.empty()); + auto nan = "nan:0x7FFFFF"sv; + ASSERT_TRUE(Lexer(nan).takeF32()); + float f = *Lexer(nan).takeF32(); + EXPECT_TRUE(std::isnan(f)); + EXPECT_FALSE(std::signbit(f)); + EXPECT_EQ(payload(f), uint32_t(0x7FFFFF)); - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); EXPECT_TRUE(std::isnan(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, 0x7fffffull); - - ASSERT_TRUE(lexer->getF32()); - float f = *lexer->getF32(); - EXPECT_TRUE(std::isnan(f)); - uint32_t fbits; - memcpy(&fbits, &f, sizeof(fbits)); - EXPECT_EQ(fbits & payloadMask32, 0x7fffffu); + EXPECT_FALSE(std::signbit(d)); + EXPECT_EQ(payload(d), uint64_t(0x7FFFFF)); } { - Lexer lexer("nan:0x800000"); - ASSERT_FALSE(lexer.empty()); + auto nan = "nan:0x800000"sv; + EXPECT_FALSE(Lexer(nan).takeF32()); - ASSERT_TRUE(lexer->getF64()); - double d = *lexer->getF64(); + ASSERT_TRUE(Lexer(nan).takeF64()); + double d = *Lexer(nan).takeF64(); EXPECT_TRUE(std::isnan(d)); - uint64_t dbits; - memcpy(&dbits, &d, sizeof(dbits)); - EXPECT_EQ(dbits & payloadMask64, 0x800000ull); - - ASSERT_FALSE(lexer->getF32()); + EXPECT_FALSE(std::signbit(d)); + EXPECT_EQ(payload(d), uint64_t(0x800000)); } - { - Lexer lexer("nan:0x0"); - ASSERT_FALSE(lexer.empty()); - ASSERT_FALSE(lexer->getF64()); - ASSERT_FALSE(lexer->getF32()); + { + auto nan = "nan:0x0"sv; + EXPECT_FALSE(Lexer(nan).takeF32()); + EXPECT_FALSE(Lexer(nan).takeF64()); } } TEST(LexerTest, LexIdent) { - { - Lexer lexer("$09azAZ!#$%&'*+-./:<=>?@\\^_`|~"sv); - ASSERT_FALSE(lexer.empty()); - Token expected{"$09azAZ!#$%&'*+-./:<=>?@\\^_`|~"sv, IdTok{}}; - EXPECT_EQ(*lexer, expected); - EXPECT_TRUE(lexer->getID()); - EXPECT_EQ(*lexer->getID(), "09azAZ!#$%&'*+-./:<=>?@\\^_`|~"sv); - } - { - Lexer lexer("$[]{}"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("$abc[]"sv); - EXPECT_TRUE(lexer.empty()); - } - { - Lexer lexer("$"sv); - EXPECT_TRUE(lexer.empty()); - } + EXPECT_EQ(Lexer("$09azAZ!#$%&'*+-./:<=>?@\\^_`|~"sv).takeID(), + wasm::Name("09azAZ!#$%&'*+-./:<=>?@\\^_`|~"sv)); + EXPECT_FALSE(Lexer("$[]{}"sv).takeID()); + EXPECT_FALSE(Lexer("$abc[]"sv).takeID()); + EXPECT_FALSE(Lexer("$"sv).takeID()); // String IDs - { - Lexer lexer("$\"\""); - ASSERT_FALSE(lexer.empty()); - Token expected{"$\"\""sv, IdTok{true, std::nullopt}}; - EXPECT_EQ(*lexer, expected); - EXPECT_TRUE(lexer->getID()); - EXPECT_EQ(*lexer->getID(), ""sv); - } - { - Lexer lexer("$\"hello\""); - ASSERT_FALSE(lexer.empty()); - Token expected{"$\"hello\""sv, IdTok{true, std::nullopt}}; - EXPECT_EQ(*lexer, expected); - EXPECT_TRUE(lexer->getID()); - EXPECT_EQ(*lexer->getID(), "hello"sv); - } - { - // _$_£_€_𐍈_ - auto unicode = "$\"_\\u{24}_\\u{00a3}_\\u{20AC}_\\u{10348}_\""sv; - Lexer lexer(unicode); - ASSERT_FALSE(lexer.empty()); - std::string escaped{"_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"}; - Token expected{unicode, IdTok{true, {escaped}}}; - EXPECT_EQ(*lexer, expected); - } + EXPECT_EQ(Lexer("$\"\""sv).takeID(), wasm::Name(""sv)); + EXPECT_EQ(Lexer("$\"hello\""sv).takeID(), wasm::Name("hello"sv)); + // _$_£_€_𐍈_ + EXPECT_EQ(Lexer("$\"_\\u{24}_\\u{00a3}_\\u{20AC}_\\u{10348}_\""sv).takeID(), + wasm::Name("_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"sv)); } TEST(LexerTest, LexString) { - { - auto pangram = "\"The quick brown fox jumps over the lazy dog\""sv; - Lexer lexer(pangram); - ASSERT_FALSE(lexer.empty()); - Token expected{pangram, StringTok{{}}}; - EXPECT_EQ(*lexer, expected); - EXPECT_TRUE(lexer->getString()); - EXPECT_EQ(*lexer->getString(), - "The quick brown fox jumps over the lazy dog"sv); - } - { - auto chars = "\"`~!@#$%^&*()_-+0123456789|,.<>/?;:'\""sv; - Lexer lexer(chars); - ASSERT_FALSE(lexer.empty()); - Token expected{chars, StringTok{{}}}; - EXPECT_EQ(*lexer, expected); - } - { - auto escapes = "\"_\\t_\\n_\\r_\\\\_\\\"_\\'_\""sv; - Lexer lexer(escapes); - ASSERT_FALSE(lexer.empty()); - Token expected{escapes, StringTok{{"_\t_\n_\r_\\_\"_'_"}}}; - EXPECT_EQ(*lexer, expected); - EXPECT_TRUE(lexer->getString()); - EXPECT_EQ(*lexer->getString(), "_\t_\n_\r_\\_\"_'_"sv); - } - { - auto escapes = "\"_\\00_\\07_\\20_\\5A_\\7F_\\ff_\\ffff_\""sv; - Lexer lexer(escapes); - ASSERT_FALSE(lexer.empty()); - std::string escaped{"_\0_\7_ _Z_\x7f_\xff_\xff" - "ff_"sv}; - Token expected{escapes, StringTok{{escaped}}}; - EXPECT_EQ(*lexer, expected); - } - { - // _$_£_€_𐍈_ - auto unicode = "\"_\\u{24}_\\u{00a3}_\\u{20AC}_\\u{10348}_\""sv; - Lexer lexer(unicode); - ASSERT_FALSE(lexer.empty()); - std::string escaped{"_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"}; - Token expected{unicode, StringTok{{escaped}}}; - EXPECT_EQ(*lexer, expected); - } - { - // _$_£_€_𐍈_ - auto unicode = "\"_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_\""sv; - Lexer lexer(unicode); - ASSERT_FALSE(lexer.empty()); - Token expected{unicode, StringTok{{}}}; - EXPECT_EQ(*lexer, expected); - } - { - Lexer lexer("\"unterminated"sv); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"unescaped nul\0\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"unescaped U+19\x19\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"unescaped U+7f\x7f\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"\\ stray backslash\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"short \\f hex escape\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"bad hex \\gg\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"empty unicode \\u{}\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"not unicode \\u{abcdefg}\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"extra chars \\u{123(}\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"unpaired surrogate unicode crimes \\u{d800}\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"more surrogate unicode crimes \\u{dfff}\""); - ASSERT_TRUE(lexer.empty()); - } - { - Lexer lexer("\"too big \\u{110000}\""); - ASSERT_TRUE(lexer.empty()); - } + using namespace std::string_literals; + EXPECT_EQ( + Lexer("\"The quick brown fox jumps over the lazy dog\""sv).takeString(), + "The quick brown fox jumps over the lazy dog"); + EXPECT_EQ(Lexer("\"`~!@#$%^&*()_-+0123456789|,.<>/?;:'\""sv).takeString(), + "`~!@#$%^&*()_-+0123456789|,.<>/?;:'"); + EXPECT_EQ(Lexer("\"_\\t_\\n_\\r_\\\\_\\\"_\\'_\""sv).takeString(), + "_\t_\n_\r_\\_\"_\'_"); + EXPECT_EQ(Lexer("\"_\\00_\\07_\\20_\\5A_\\7F_\\ff_\\ffff_\""sv).takeString(), + "_\0_\7_ _Z_\x7f_\xff_\xff"s + "ff_"s); + // _$_£_€_𐍈_ + EXPECT_EQ( + Lexer("\"_\\u{24}_\\u{00a3}_\\u{20AC}_\\u{10348}_\""sv).takeString(), + "_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"s); + EXPECT_EQ( + Lexer("\"_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_\""sv).takeString(), + "_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"s); + + EXPECT_FALSE(Lexer("\"unterminated"sv).takeString()); + EXPECT_FALSE(Lexer("\"unescaped nul\0\""sv).takeString()); + EXPECT_FALSE(Lexer("\"unescaped U+19\x19\""sv).takeString()); + EXPECT_FALSE(Lexer("\"unescaped U+7f\x7f\""sv).takeString()); + EXPECT_FALSE(Lexer("\"\\ stray backslash\""sv).takeString()); + EXPECT_FALSE(Lexer("\"short \\f hex escape\""sv).takeString()); + EXPECT_FALSE(Lexer("\"bad hex \\gg\""sv).takeString()); + EXPECT_FALSE(Lexer("\"empty unicode \\u{}\""sv).takeString()); + EXPECT_FALSE(Lexer("\"not unicode \\u{abcdefg}\""sv).takeString()); + EXPECT_FALSE(Lexer("\"extra chars \\u{123(}\""sv).takeString()); + EXPECT_FALSE( + Lexer("\"unpaired surrogate unicode crimes \\u{d800}\""sv).takeString()); + EXPECT_FALSE( + Lexer("\"more surrogate unicode crimes \\u{dfff}\""sv).takeString()); + EXPECT_FALSE(Lexer("\"too big \\u{110000}\""sv).takeString()); } TEST(LexerTest, LexKeywords) { - Token module{"module"sv, KeywordTok{}}; - Token type{"type"sv, KeywordTok{}}; - Token func{"func"sv, KeywordTok{}}; - Token import{"import"sv, KeywordTok{}}; - Token reserved{"rEsErVeD"sv, KeywordTok{}}; - Lexer lexer("module type func import rEsErVeD"); - - auto it = lexer.begin(); - ASSERT_NE(it, lexer.end()); - Token t1 = *it++; - ASSERT_NE(it, lexer.end()); - Token t2 = *it++; - ASSERT_NE(it, lexer.end()); - Token t3 = *it++; - ASSERT_NE(it, lexer.end()); - Token t4 = *it++; - ASSERT_NE(it, lexer.end()); - Token t5 = *it++; - EXPECT_EQ(it, lexer.end()); - - EXPECT_EQ(t1, module); - EXPECT_EQ(t2, type); - EXPECT_EQ(t3, func); - EXPECT_EQ(t4, import); - EXPECT_EQ(t5, reserved); - - EXPECT_TRUE(t1.getKeyword()); - EXPECT_EQ(*t1.getKeyword(), "module"sv); + ASSERT_EQ(lexer.takeKeyword(), "module"sv); + ASSERT_EQ(lexer.takeKeyword(), "type"sv); + ASSERT_EQ(lexer.takeKeyword(), "func"sv); + ASSERT_EQ(lexer.takeKeyword(), "import"sv); + ASSERT_EQ(lexer.takeKeyword(), "rEsErVeD"sv); + ASSERT_TRUE(lexer.empty()); } |