karma/interpreter-cpp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

extended lexer to new keywords

Browse Source
This commit is contained in:
Karma Riuk
2025-06-30 00:36:31 +02:00
parent 5cc7147909
commit 7973f7522c
3 changed files with 43 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/token/type.cpp
View File

@@ -6,10 +6,11 @@
namespace token {
// Array mapping enum values to their string representations
constexpr std::array<std::string_view, static_cast<size_t>(type::LET) + 1>
tokenTypeStrings = {
constexpr std::
array<std::string_view, static_cast<size_t>(type::RETURN) + 1>
tokenTypeStrings = {
#define X(name, str) str,
TOKEN_LIST
TOKEN_LIST
#undef X
};
@@ -24,6 +25,11 @@ namespace token {
static std::unordered_map<std::string, type> keywords{
{"fn", type::FUNCTION},
{"let", type::LET},
{"if", type::IF},
{"else", type::ELSE},
{"true", type::TRUE},
{"false", type::FALSE},
{"return", type::RETURN},
};
type lookup_identifier(std::string ident) {

7
src/token/type.hpp
View File

@@ -24,8 +24,13 @@ namespace token {
X(RPAREN, ")") \
X(LBRACE, "{") \
X(RBRACE, "}") \
X(LET, "LET") \
X(FUNCTION, "FUNCTION") \
X(LET, "LET")
X(IF, "IF") \
X(ELSE, "ELSE") \
X(TRUE, "TRUE") \
X(FALSE, "FALSE") \
X(RETURN, "RETURN")
// Define the TokenType enum using the X-macro
enum class type {

95
test/lexer.cpp
View File

@@ -48,75 +48,14 @@ let add = fn(x, y) {\
x + y;\
};\
let result = add(five, ten);\
");
lexer::lexer l{ss};
test tests[] = {
// clang-format off
{token::type::LET, "let"},
{token::type::IDENTIFIER, "five"},
{token::type::ASSIGN, "="},
{token::type::INT, "5"},
{token::type::SEMICOLON, ";"},
{token::type::LET, "let"},
{token::type::IDENTIFIER, "ten"},
{token::type::ASSIGN, "="},
{token::type::INT, "10"},
{token::type::SEMICOLON, ";"},
{token::type::LET, "let"},
{token::type::IDENTIFIER, "add"},
{token::type::ASSIGN, "="},
{token::type::FUNCTION, "fn"},
{token::type::LPAREN, "("},
{token::type::IDENTIFIER, "x"},
{token::type::COMMA, ","},
{token::type::IDENTIFIER, "y"},
{token::type::RPAREN, ")"},
{token::type::LBRACE, "{"},
{token::type::IDENTIFIER, "x"},
{token::type::PLUS, "+"},
{token::type::IDENTIFIER, "y"},
{token::type::SEMICOLON, ";"},
{token::type::RBRACE, "}"},
{token::type::SEMICOLON, ";"},
{token::type::LET, "let"},
{token::type::IDENTIFIER, "result"},
{token::type::ASSIGN, "="},
{token::type::IDENTIFIER, "add"},
{token::type::LPAREN, "("},
{token::type::IDENTIFIER, "five"},
{token::type::COMMA, ","},
{token::type::IDENTIFIER, "ten"},
{token::type::RPAREN, ")"},
{token::type::SEMICOLON, ";"},
// clang-format on
};
for (const auto& t : tests) {
token::token tok = l.next_token();
CHECK(tok.type == t.expectedType);
CHECK(tok.literal == t.expectedLiteral);
}
};
TEST_CASE("Even more tokens (simple)") {
struct test {
token::type expectedType;
std::string expectedLiteral;
};
std::istringstream ss("let five = 5;\
let ten = 10;\
let add = fn(x, y) {\
x + y;\
};\
let result = add(five, ten);\
!-/*5;\
5 < 10 > 5;\
\
if (5 < 10) {\
return true;\
} else {\
return false;\
}\
");
lexer::lexer l{ss};
@@ -176,6 +115,28 @@ let result = add(five, ten);\
{token::type::GT, ">"},
{token::type::INT, "5"},
{token::type::SEMICOLON, ";"},
{token::type::IF, "if"},
{token::type::LPAREN, "("},
{token::type::INT, "5"},
{token::type::LT, "<"},
{token::type::INT, "10"},
{token::type::RPAREN, ")"},
{token::type::LBRACE, "{"},
{token::type::RETURN, "return"},
{token::type::TRUE, "true"},
{token::type::SEMICOLON, ";"},
{token::type::RBRACE, "}"},
{token::type::ELSE, "else"},
{token::type::LBRACE, "{"},
{token::type::RETURN, "return"},
{token::type::FALSE, "false"},
{token::type::SEMICOLON, ";"},
{token::type::RBRACE, "}"},
// clang-format on
};