summaryrefslogtreecommitdiff
path: root/src/parser/lexer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/parser/lexer.cpp')
-rw-r--r--src/parser/lexer.cpp81
1 files changed, 60 insertions, 21 deletions
diff --git a/src/parser/lexer.cpp b/src/parser/lexer.cpp
index 2625bf8fd..07931d69b 100644
--- a/src/parser/lexer.cpp
+++ b/src/parser/lexer.cpp
@@ -329,6 +329,25 @@ public:
}
};
+struct LexIdResult : LexResult {
+ bool isStr = false;
+ std::optional<std::string> str;
+};
+
+struct LexIdCtx : LexCtx {
+ bool isStr = false;
+ std::optional<std::string> str;
+
+ LexIdCtx(std::string_view in) : LexCtx(in) {}
+
+ std::optional<LexIdResult> lexed() {
+ if (auto basic = LexCtx::lexed()) {
+ return LexIdResult{*basic, isStr, str};
+ }
+ return {};
+ }
+};
+
std::optional<LexResult> lparen(std::string_view in) {
LexCtx ctx(in);
ctx.takePrefix("("sv);
@@ -647,26 +666,6 @@ std::optional<LexResult> idchar(std::string_view in) {
return ctx.lexed();
}
-// id ::= '$' idchar+
-std::optional<LexResult> ident(std::string_view in) {
- LexCtx ctx(in);
- if (!ctx.takePrefix("$"sv)) {
- return {};
- }
- if (auto lexed = idchar(ctx.next())) {
- ctx.take(*lexed);
- } else {
- return {};
- }
- while (auto lexed = idchar(ctx.next())) {
- ctx.take(*lexed);
- }
- if (ctx.canFinish()) {
- return ctx.lexed();
- }
- return {};
-}
-
// string ::= '"' (b*:stringelem)* '"' => concat((b*)*)
// (if |concat((b*)*)| < 2^32)
// stringelem ::= c:stringchar => utf8(c)
@@ -741,6 +740,30 @@ std::optional<LexStrResult> str(std::string_view in) {
return ctx.lexed();
}
+// id ::= '$' idchar+ | '$' str
+std::optional<LexIdResult> ident(std::string_view in) {
+ LexIdCtx ctx(in);
+ if (!ctx.takePrefix("$"sv)) {
+ return {};
+ }
+ if (auto s = str(ctx.next())) {
+ ctx.isStr = true;
+ ctx.str = s->str;
+ ctx.take(*s);
+ } else if (auto lexed = idchar(ctx.next())) {
+ ctx.take(*lexed);
+ while (auto lexed = idchar(ctx.next())) {
+ ctx.take(*lexed);
+ }
+ } else {
+ return {};
+ }
+ if (ctx.canFinish()) {
+ return ctx.lexed();
+ }
+ return {};
+}
+
// keyword ::= ( 'a' | ... | 'z' ) idchar* (if literal terminal in grammar)
// reserved ::= idchar+
//
@@ -889,11 +912,27 @@ std::optional<std::string_view> Token::getString() const {
if (tok->str) {
return std::string_view(*tok->str);
}
+ // Remove quotes.
return span.substr(1, span.size() - 2);
}
return {};
}
+std::optional<std::string_view> Token::getID() const {
+ if (auto* tok = std::get_if<IdTok>(&data)) {
+ if (tok->str) {
+ return std::string_view(*tok->str);
+ }
+ if (tok->isStr) {
+ // Remove '$' and quotes.
+ return span.substr(2, span.size() - 3);
+ }
+ // Remove '$'.
+ return span.substr(1);
+ }
+ return {};
+}
+
void Lexer::skipSpace() {
if (auto ctx = space(next())) {
index += ctx->span.size();
@@ -908,7 +947,7 @@ void Lexer::lexToken() {
} else if (auto t = rparen(next())) {
tok = Token{t->span, RParenTok{}};
} else if (auto t = ident(next())) {
- tok = Token{t->span, IdTok{}};
+ tok = Token{t->span, IdTok{t->isStr, t->str}};
} else if (auto t = integer(next())) {
tok = Token{t->span, IntTok{t->n, t->sign}};
} else if (auto t = float_(next())) {