summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/parser/lexer.cpp81
-rw-r--r--src/parser/lexer.h16
-rw-r--r--test/gtest/wat-lexer.cpp27
-rw-r--r--test/lit/wat-kitchen-sink.wast11
4 files changed, 103 insertions, 32 deletions
diff --git a/src/parser/lexer.cpp b/src/parser/lexer.cpp
index 2625bf8fd..07931d69b 100644
--- a/src/parser/lexer.cpp
+++ b/src/parser/lexer.cpp
@@ -329,6 +329,25 @@ public:
}
};
+struct LexIdResult : LexResult {
+ bool isStr = false;
+ std::optional<std::string> str;
+};
+
+struct LexIdCtx : LexCtx {
+ bool isStr = false;
+ std::optional<std::string> str;
+
+ LexIdCtx(std::string_view in) : LexCtx(in) {}
+
+ std::optional<LexIdResult> lexed() {
+ if (auto basic = LexCtx::lexed()) {
+ return LexIdResult{*basic, isStr, str};
+ }
+ return {};
+ }
+};
+
std::optional<LexResult> lparen(std::string_view in) {
LexCtx ctx(in);
ctx.takePrefix("("sv);
@@ -647,26 +666,6 @@ std::optional<LexResult> idchar(std::string_view in) {
return ctx.lexed();
}
-// id ::= '$' idchar+
-std::optional<LexResult> ident(std::string_view in) {
- LexCtx ctx(in);
- if (!ctx.takePrefix("$"sv)) {
- return {};
- }
- if (auto lexed = idchar(ctx.next())) {
- ctx.take(*lexed);
- } else {
- return {};
- }
- while (auto lexed = idchar(ctx.next())) {
- ctx.take(*lexed);
- }
- if (ctx.canFinish()) {
- return ctx.lexed();
- }
- return {};
-}
-
// string ::= '"' (b*:stringelem)* '"' => concat((b*)*)
// (if |concat((b*)*)| < 2^32)
// stringelem ::= c:stringchar => utf8(c)
@@ -741,6 +740,30 @@ std::optional<LexStrResult> str(std::string_view in) {
return ctx.lexed();
}
+// id ::= '$' idchar+ | '$' str
+std::optional<LexIdResult> ident(std::string_view in) {
+ LexIdCtx ctx(in);
+ if (!ctx.takePrefix("$"sv)) {
+ return {};
+ }
+ if (auto s = str(ctx.next())) {
+ ctx.isStr = true;
+ ctx.str = s->str;
+ ctx.take(*s);
+ } else if (auto lexed = idchar(ctx.next())) {
+ ctx.take(*lexed);
+ while (auto lexed = idchar(ctx.next())) {
+ ctx.take(*lexed);
+ }
+ } else {
+ return {};
+ }
+ if (ctx.canFinish()) {
+ return ctx.lexed();
+ }
+ return {};
+}
+
// keyword ::= ( 'a' | ... | 'z' ) idchar* (if literal terminal in grammar)
// reserved ::= idchar+
//
@@ -889,11 +912,27 @@ std::optional<std::string_view> Token::getString() const {
if (tok->str) {
return std::string_view(*tok->str);
}
+ // Remove quotes.
return span.substr(1, span.size() - 2);
}
return {};
}
+std::optional<std::string_view> Token::getID() const {
+ if (auto* tok = std::get_if<IdTok>(&data)) {
+ if (tok->str) {
+ return std::string_view(*tok->str);
+ }
+ if (tok->isStr) {
+ // Remove '$' and quotes.
+ return span.substr(2, span.size() - 3);
+ }
+ // Remove '$'.
+ return span.substr(1);
+ }
+ return {};
+}
+
void Lexer::skipSpace() {
if (auto ctx = space(next())) {
index += ctx->span.size();
@@ -908,7 +947,7 @@ void Lexer::lexToken() {
} else if (auto t = rparen(next())) {
tok = Token{t->span, RParenTok{}};
} else if (auto t = ident(next())) {
- tok = Token{t->span, IdTok{}};
+ tok = Token{t->span, IdTok{t->isStr, t->str}};
} else if (auto t = integer(next())) {
tok = Token{t->span, IntTok{t->n, t->sign}};
} else if (auto t = float_(next())) {
diff --git a/src/parser/lexer.h b/src/parser/lexer.h
index 42b18508e..f0da151f9 100644
--- a/src/parser/lexer.h
+++ b/src/parser/lexer.h
@@ -53,6 +53,12 @@ struct RParenTok {
};
struct IdTok {
+ // Whether this ID has `$"..."` format
+ bool isStr;
+
+ // If the ID is a string ID and contains escapes, this is its contents.
+ std::optional<std::string> str;
+
bool operator==(const IdTok&) const { return true; }
friend std::ostream& operator<<(std::ostream&, const IdTok&);
};
@@ -81,6 +87,7 @@ struct FloatTok {
};
struct StringTok {
+ // If the string contains escapes, this is its contents.
std::optional<std::string> str;
bool operator==(const StringTok& other) const { return str == other.str; }
@@ -111,14 +118,6 @@ struct Token {
bool isRParen() const { return std::get_if<RParenTok>(&data); }
- std::optional<std::string_view> getID() const {
- if (std::get_if<IdTok>(&data)) {
- // Drop leading '$'.
- return span.substr(1);
- }
- return {};
- }
-
std::optional<std::string_view> getKeyword() const {
if (std::get_if<KeywordTok>(&data)) {
return span;
@@ -132,6 +131,7 @@ struct Token {
std::optional<double> getF64() const;
std::optional<float> getF32() const;
std::optional<std::string_view> getString() const;
+ std::optional<std::string_view> getID() const;
bool operator==(const Token&) const;
friend std::ostream& operator<<(std::ostream& os, const Token&);
diff --git a/test/gtest/wat-lexer.cpp b/test/gtest/wat-lexer.cpp
index b46f9927f..b62644682 100644
--- a/test/gtest/wat-lexer.cpp
+++ b/test/gtest/wat-lexer.cpp
@@ -1377,6 +1377,33 @@ TEST(LexerTest, LexIdent) {
Lexer lexer("$"sv);
EXPECT_TRUE(lexer.empty());
}
+
+ // String IDs
+ {
+ Lexer lexer("$\"\"");
+ ASSERT_FALSE(lexer.empty());
+ Token expected{"$\"\""sv, IdTok{true, std::nullopt}};
+ EXPECT_EQ(*lexer, expected);
+ EXPECT_TRUE(lexer->getID());
+ EXPECT_EQ(*lexer->getID(), ""sv);
+ }
+ {
+ Lexer lexer("$\"hello\"");
+ ASSERT_FALSE(lexer.empty());
+ Token expected{"$\"hello\""sv, IdTok{true, std::nullopt}};
+ EXPECT_EQ(*lexer, expected);
+ EXPECT_TRUE(lexer->getID());
+ EXPECT_EQ(*lexer->getID(), "hello"sv);
+ }
+ {
+ // _$_£_€_𐍈_
+ auto unicode = "$\"_\\u{24}_\\u{00a3}_\\u{20AC}_\\u{10348}_\""sv;
+ Lexer lexer(unicode);
+ ASSERT_FALSE(lexer.empty());
+ std::string escaped{"_$_\xC2\xA3_\xE2\x82\xAC_\xF0\x90\x8D\x88_"};
+ Token expected{unicode, IdTok{true, {escaped}}};
+ EXPECT_EQ(*lexer, expected);
+ }
}
TEST(LexerTest, LexString) {
diff --git a/test/lit/wat-kitchen-sink.wast b/test/lit/wat-kitchen-sink.wast
index 1ab8e7516..32a70d913 100644
--- a/test/lit/wat-kitchen-sink.wast
+++ b/test/lit/wat-kitchen-sink.wast
@@ -380,7 +380,7 @@
;; CHECK: (elem $passive-2 anyref (struct.new_default $s0) (struct.new_default $s0))
(elem $passive-2 anyref (item struct.new $s0) (struct.new $s0))
- ;; CHECK: (elem declare func $ref-func $ref-is-null $table-fill $table-grow $table-set)
+ ;; CHECK: (elem declare func $ref-func $table-fill $table-grow $table-set)
(elem declare func 0 1 2 3)
(elem $declare-2 declare funcref (item ref.func 0) (ref.func 1) (item (ref.func 2)))
@@ -467,6 +467,11 @@
;; CHECK-NEXT: )
(func $f4 (type 18) (local i32 i64) (local $l f32))
+ ;; CHECK: (func $"[quoted_name]" (type $void)
+ ;; CHECK-NEXT: (nop)
+ ;; CHECK-NEXT: )
+ (func $"[quoted_name]")
+
;; CHECK: (func $nop-skate (type $void)
;; CHECK-NEXT: (nop)
;; CHECK-NEXT: (nop)
@@ -3622,13 +3627,13 @@
;; CHECK-NEXT: (ref.func $ref-func)
;; CHECK-NEXT: )
;; CHECK-NEXT: (drop
- ;; CHECK-NEXT: (ref.func $ref-is-null)
+ ;; CHECK-NEXT: (ref.func $ref-func)
;; CHECK-NEXT: )
;; CHECK-NEXT: )
(func $ref-func
ref.func $ref-func
drop
- ref.func 154
+ ref.func 156
drop
)
generated by cgit v1.2.3 (git 2.39.1) at 2025-08-05 08:06:02 +0000