kayomn
/
ona
1
0
Fork 0
Code Issues 8 Pull Requests Projects 4 Releases Activity

Make tokenization aware of string templates

This commit is contained in:
kayomn 2023-11-07 23:06:11 +00:00
parent 94a13b0d3c
commit 63792ea983
1 changed files with 18 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
source/ona/kym/tokens.zig
View File

@ -31,6 +31,7 @@ pub const Token = union(enum) {
number: []const coral.io.Byte, number: []const coral.io.Byte,
string: []const coral.io.Byte, string: []const coral.io.Byte,
template_string: []const coral.io.Byte,
keyword_nil, keyword_nil,
keyword_false, keyword_false,
@ -81,6 +82,7 @@ pub const Token = union(enum) {
.number => |literal| literal, .number => |literal| literal,
.string => |literal| literal, .string => |literal| literal,
.template_string => |literal| literal,
.keyword_const => "const", .keyword_const => "const",
.keyword_nil => "nil", .keyword_nil => "nil",
@ -305,6 +307,22 @@ pub const Stream = struct {
return; return;
}, },
'`' => {
cursor += 1;
const begin = cursor;
while (cursor < self.source.len) switch (self.source[cursor]) {
'`' => break,
else => cursor += 1,
};
self.token = .{.template_string = self.source[begin .. cursor]};
cursor += 1;
return;
},
'"' => { '"' => {
cursor += 1; cursor += 1;