kayomn
/
ona
1
0
Fork 0
Code Issues 8 Pull Requests Projects 4 Releases Activity

kym-tables-overhaul #30

Merged
kayomn merged 12 commits from kym-tables-overhaul into main 2023-08-12 15:20:18 +02:00
Conversation 0 Commits 12 Files Changed 15 +351 -258
5 changed files with 351 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit b6f7ab1edb - Show all commits

9
debug/app.ona
View File

@ -1,6 +1,13 @@
# Test comment. # Test comment.
@log_info("game is loading")
test = {
.message = "don't you lecture me with your 30 dollar scripting language"
}
# test.message = "game is loading"
@log_info(test.message)
return { return {
.title = "Afterglow", .title = "Afterglow",

46
source/ona/kym.zig
View File

@ -143,19 +143,13 @@ pub const RuntimeEnv = struct {
}); });
defer { defer {
const frame = self.frames.pop(); const frame = self.frames.pop() orelse unreachable;
coral.debug.assert(frame != null);
{ {
var pops_remaining = (self.local_refs.values.len - frame.?.locals_top) + frame.?.arg_count; var pops_remaining = (self.local_refs.values.len - frame.locals_top) + frame.arg_count;
while (pops_remaining != 0) : (pops_remaining -= 1) { while (pops_remaining != 0) : (pops_remaining -= 1) {
const local = self.local_refs.pop(); self.discard(self.local_refs.pop() orelse unreachable);
coral.debug.assert(local != null);
self.discard(local.?);
} }
} }
} }
@ -278,34 +272,18 @@ pub const RuntimeEnv = struct {
} }
pub fn push_boolean(self: *RuntimeEnv, boolean: bool) RuntimeError!void { pub fn push_boolean(self: *RuntimeEnv, boolean: bool) RuntimeError!void {
if (self.frames.is_empty()) {
return self.raise(error.IllegalState, "attempt to push boolean outside a call frame");
}
return self.local_refs.push_one(try self.new_boolean(boolean)); return self.local_refs.push_one(try self.new_boolean(boolean));
} }
pub fn push_fixed(self: *RuntimeEnv, fixed: Fixed) RuntimeError!void { pub fn push_fixed(self: *RuntimeEnv, fixed: Fixed) RuntimeError!void {
if (self.frames.is_empty()) {
return self.raise(error.IllegalState, "attempt to push fixed outside a call frame");
}
return self.local_refs.push_one(try self.new_fixed(fixed)); return self.local_refs.push_one(try self.new_fixed(fixed));
} }
pub fn push_float(self: *RuntimeEnv, float: Float) RuntimeError!void { pub fn push_float(self: *RuntimeEnv, float: Float) RuntimeError!void {
if (self.frames.is_empty()) {
return self.raise(error.IllegalState, "attempt to push float outside a call frame");
}
return self.local_refs.push_one(try self.new_float(float)); return self.local_refs.push_one(try self.new_float(float));
} }
pub fn push_ref(self: *RuntimeEnv, ref: ?*RuntimeRef) RuntimeError!void { pub fn push_ref(self: *RuntimeEnv, ref: ?*RuntimeRef) RuntimeError!void {
if (self.frames.is_empty()) {
return self.raise(error.IllegalState, "attempt to push ref outside a call frame");
}
return self.local_refs.push_one(if (ref) |live_ref| self.acquire(live_ref) else null); return self.local_refs.push_one(if (ref) |live_ref| self.acquire(live_ref) else null);
} }
@ -415,6 +393,18 @@ pub const RuntimeEnv = struct {
return error_value; return error_value;
} }
pub fn set_dynamic(
self: *RuntimeEnv,
indexable: *const DynamicObject,
index_ref: *const RuntimeRef,
value_ref: ?*const RuntimeRef,
) RuntimeError!void {
return indexable.typeinfo.set(.{
.env = self,
.userdata = indexable.userdata,
}, index_ref, value_ref);
}
pub fn syscallable(self: *RuntimeEnv, name: []const coral.io.Byte) RuntimeError!Caller { pub fn syscallable(self: *RuntimeEnv, name: []const coral.io.Byte) RuntimeError!Caller {
return self.syscallers.lookup(name) orelse self.raise(error.BadOperation, "attempt to get undefined syscall"); return self.syscallers.lookup(name) orelse self.raise(error.BadOperation, "attempt to get undefined syscall");
} }
@ -453,11 +443,15 @@ pub const RuntimeRef = opaque {};
pub const Typeinfo = struct { pub const Typeinfo = struct {
name: []const coral.io.Byte, name: []const coral.io.Byte,
call: ?*const fn (method: Method) RuntimeError!?*RuntimeRef = null, call: *const fn (method: Method) RuntimeError!?*RuntimeRef = default_call,
clean: *const fn (method: Method) void = default_clean, clean: *const fn (method: Method) void = default_clean,
get: *const fn (method: Method, index: *const RuntimeRef) RuntimeError!?*RuntimeRef = default_get, get: *const fn (method: Method, index: *const RuntimeRef) RuntimeError!?*RuntimeRef = default_get,
set: *const fn (method: Method, index: *const RuntimeRef, value: ?*const RuntimeRef) RuntimeError!void = default_set, set: *const fn (method: Method, index: *const RuntimeRef, value: ?*const RuntimeRef) RuntimeError!void = default_set,
fn default_call(method: Method) RuntimeError!?*RuntimeRef {
return method.env.raise(error.TypeMismatch, "object is not callable");
}
fn default_clean(_: Method) void { fn default_clean(_: Method) void {
// Nothing to clean by default. // Nothing to clean by default.
} }

197
source/ona/kym/Ast.zig
View File

@ -18,7 +18,18 @@ pub const Expression = union (enum) {
symbol_literal: []const coral.io.Byte, symbol_literal: []const coral.io.Byte,
table_literal: TableLiteral, table_literal: TableLiteral,
grouped_expression: *Expression, grouped_expression: *Expression,
get_local: []const coral.io.Byte, resolve_local: []const coral.io.Byte,
get_field: struct {
object_expression: *Expression,
identifier: []const coral.io.Byte,
},
set_field: struct {
object_expression: *Expression,
identifier: []const coral.io.Byte,
value_expression: *Expression,
},
call_system: struct { call_system: struct {
identifier: []const coral.io.Byte, identifier: []const coral.io.Byte,
@ -80,10 +91,10 @@ pub const ParseError = error {
const Self = @This(); const Self = @This();
pub const Statement = union (enum) { pub const Statement = union (enum) {
return_expression: Expression,
return_nothing, return_nothing,
return_expression: Expression,
set_local: struct { assign_local: struct {
identifier: []const coral.io.Byte, identifier: []const coral.io.Byte,
expression: Expression, expression: Expression,
}, },
@ -113,10 +124,10 @@ fn binary_operation_parser(
inline for (operators) |operator| { inline for (operators) |operator| {
const token = comptime operator.token(); const token = comptime operator.token();
if (self.tokenizer.is_token(coral.io.tag_of(token))) { if (self.tokenizer.token == coral.io.tag_of(token)) {
self.tokenizer.step(); self.tokenizer.step();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("expected other half of expression after `" ++ comptime token.text() ++ "`"); return self.report("expected other half of expression after `" ++ comptime token.text() ++ "`");
} }
@ -178,10 +189,12 @@ pub fn parse(self: *Self, data: []const coral.io.Byte) ParseError!void {
const allocator = self.arena.as_allocator(); const allocator = self.arena.as_allocator();
var has_returned = false; var has_returned = false;
while (true) { self.tokenizer.skip_newlines();
self.tokenizer.skip(.newline);
while (true) {
switch (self.tokenizer.token) {
.end => return,
switch (self.tokenizer.token orelse return) {
.keyword_return => { .keyword_return => {
if (has_returned) { if (has_returned) {
return self.report("multiple returns in function scope but expected only one"); return self.report("multiple returns in function scope but expected only one");
@ -190,12 +203,12 @@ pub fn parse(self: *Self, data: []const coral.io.Byte) ParseError!void {
try self.statements.push_one(get_statement: { try self.statements.push_one(get_statement: {
self.tokenizer.step(); self.tokenizer.step();
if (!self.tokenizer.is_token_null_or(.newline)) { if (self.tokenizer.token != .end and self.tokenizer.token != .newline) {
break: get_statement .{.return_expression = try self.parse_expression()}; break: get_statement .{.return_expression = try self.parse_expression()};
} }
if (!self.tokenizer.is_token_null_or(.newline)) { if (self.tokenizer.token != .end and self.tokenizer.token != .newline) {
return self.report("unexpected token after return"); return self.report("expected end or newline after return statement");
} }
break: get_statement .return_nothing; break: get_statement .return_nothing;
@ -207,28 +220,22 @@ pub fn parse(self: *Self, data: []const coral.io.Byte) ParseError!void {
.identifier => |identifier| { .identifier => |identifier| {
self.tokenizer.step(); self.tokenizer.step();
const no_effect_message = "statement has no effect"; switch (self.tokenizer.token) {
.end, .newline => return self.report("statement has no effect"),
switch (self.tokenizer.token orelse return self.report(no_effect_message)) {
.newline => return self.report(no_effect_message),
.symbol_equals => { .symbol_equals => {
self.tokenizer.step(); self.tokenizer.step();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("expected expression after `=`"); return self.report("expected expression after `=`");
} }
try self.statements.push_one(.{ try self.statements.push_one(.{
.set_local = .{ .assign_local = .{
.expression = try self.parse_expression(), .expression = try self.parse_expression(),
.identifier = identifier, .identifier = identifier,
}, },
}); });
if (!self.tokenizer.is_token_null_or(.newline)) {
return self.report("unexpected token after assignment");
}
}, },
else => return self.report("expected `=` after local"), else => return self.report("expected `=` after local"),
@ -238,10 +245,8 @@ pub fn parse(self: *Self, data: []const coral.io.Byte) ParseError!void {
.special_identifier => |identifier| { .special_identifier => |identifier| {
self.tokenizer.step(); self.tokenizer.step();
const missing_arguments_message = "system call is missing arguments"; switch (self.tokenizer.token) {
.end, .newline => return self.report("system call is missing arguments"),
switch (self.tokenizer.token orelse return self.report(missing_arguments_message)) {
.newline => return self.report(missing_arguments_message),
.symbol_paren_left => { .symbol_paren_left => {
self.tokenizer.step(); self.tokenizer.step();
@ -249,20 +254,20 @@ pub fn parse(self: *Self, data: []const coral.io.Byte) ParseError!void {
var expressions_list = Expression.List.make(allocator); var expressions_list = Expression.List.make(allocator);
while (true) { while (true) {
if (self.tokenizer.is_token(.symbol_paren_right)) { if (self.tokenizer.token == .symbol_paren_right) {
break; break;
} }
try expressions_list.push_one(try self.parse_expression()); try expressions_list.push_one(try self.parse_expression());
switch (self.tokenizer.token orelse return self.report("unexpected end after after `(`")) { switch (self.tokenizer.token) {
.symbol_comma => continue, .symbol_comma => continue,
.symbol_paren_right => break, .symbol_paren_right => break,
else => return self.report("expected `)` or argument after `(`"), else => return self.report("expected `)` or argument after `(`"),
} }
} }
self.tokenizer.step(); self.tokenizer.skip_newlines();
try self.statements.push_one(.{ try self.statements.push_one(.{
.call_system = .{ .call_system = .{
@ -300,66 +305,69 @@ const parse_expression = binary_operation_parser(parse_equality, &.{
fn parse_factor(self: *Self) ParseError!Expression { fn parse_factor(self: *Self) ParseError!Expression {
const allocator = self.arena.as_allocator(); const allocator = self.arena.as_allocator();
switch (self.tokenizer.token orelse return self.report("expected operand after operator")) { var expression = @as(Expression, get: {
switch (self.tokenizer.token) {
.symbol_paren_left => { .symbol_paren_left => {
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("expected an expression after `(`"); return self.report("expected an expression after `(`");
} }
const expression = try self.parse_expression(); const expression = try self.parse_expression();
if (!self.tokenizer.is_token(.symbol_paren_right)) { if (self.tokenizer.token != .symbol_paren_right) {
return self.report("expected a closing `)` after expression"); return self.report("expected a closing `)` after expression");
} }
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.grouped_expression = try coral.io.allocate_one(allocator, expression)}; break: get Expression{.grouped_expression = try coral.io.allocate_one(allocator, expression)};
}, },
.keyword_nil => { .keyword_nil => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return .nil_literal; break: get .nil_literal;
}, },
.keyword_true => { .keyword_true => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return .true_literal; break: get .true_literal;
}, },
.keyword_false => { .keyword_false => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return .false_literal; break: get .false_literal;
}, },
.number => |value| { .number => |value| {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.number_literal = value}; break: get .{.number_literal = value};
}, },
.string => |value| { .string => |value| {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.string_literal = value}; break: get .{.string_literal = value};
}, },
.special_identifier => |identifier| { .special_identifier => |identifier| {
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
var expression_list = Expression.List.make(allocator); var expression_list = Expression.List.make(allocator);
while (true) { while (true) {
switch (self.tokenizer.token orelse return self.report("expected expression or `)` after `(`")) { switch (self.tokenizer.token) {
.symbol_paren_right => { .end => return self.report("expected expression or `)` after `(`"),
self.tokenizer.step();
return Expression{ .symbol_paren_right => {
self.tokenizer.skip_newlines();
break: get .{
.call_system = .{ .call_system = .{
.identifier = identifier, .identifier = identifier,
.argument_expressions = expression_list, .argument_expressions = expression_list,
@ -370,13 +378,14 @@ fn parse_factor(self: *Self) ParseError!Expression {
else => { else => {
try expression_list.push_one(try self.parse_expression()); try expression_list.push_one(try self.parse_expression());
switch (self.tokenizer.token orelse return self.report("expected `,` or `)` after argument")) { switch (self.tokenizer.token) {
.end => return self.report("expected `,` or `)` after argument"),
.symbol_comma => continue, .symbol_comma => continue,
.symbol_paren_right => { .symbol_paren_right => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{ break: get .{
.call_system = .{ .call_system = .{
.identifier = identifier, .identifier = identifier,
.argument_expressions = expression_list, .argument_expressions = expression_list,
@ -392,28 +401,28 @@ fn parse_factor(self: *Self) ParseError!Expression {
}, },
.identifier => |identifier| { .identifier => |identifier| {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.get_local = identifier}; break: get .{.resolve_local = identifier};
}, },
.symbol_brace_left => { .symbol_brace_left => {
var table_literal = Expression.TableLiteral.make(allocator); var table_literal = Expression.TableLiteral.make(allocator);
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
while (true) { while (true) {
switch (self.tokenizer.token orelse return self.report("unexpected end of table literal")) { switch (self.tokenizer.token) {
.symbol_brace_right => { .symbol_brace_right => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.table_literal = table_literal}; break: get .{.table_literal = table_literal};
}, },
.symbol_bracket_left => { .symbol_bracket_left => {
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
if (!self.tokenizer.is_token(.symbol_equals)) { if (self.tokenizer.token != .symbol_equals) {
return self.report("expected expression after identifier"); return self.report("expected expression after identifier");
} }
}, },
@ -421,21 +430,20 @@ fn parse_factor(self: *Self) ParseError!Expression {
.symbol_period => { .symbol_period => {
self.tokenizer.step(); self.tokenizer.step();
if (!self.tokenizer.is_token(.identifier)) { const identifier = switch (self.tokenizer.token) {
return self.report("expected identifier after `.`"); .identifier => |identifier| identifier,
} else => return self.report("expected identifier after `.`"),
};
const identifier = self.tokenizer.token.?.identifier; self.tokenizer.skip_newlines();
self.tokenizer.skip(.newline); if (self.tokenizer.token != .symbol_equals) {
if (!self.tokenizer.is_token(.symbol_equals)) {
return self.report("expected `=` after key"); return self.report("expected `=` after key");
} }
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("unexpected end after `=`"); return self.report("unexpected end after `=`");
} }
@ -444,13 +452,13 @@ fn parse_factor(self: *Self) ParseError!Expression {
.key_expression = .{.symbol_literal = identifier}, .key_expression = .{.symbol_literal = identifier},
}); });
switch (self.tokenizer.token orelse return self.report("unexpected end of table")) { switch (self.tokenizer.token) {
.symbol_comma => self.tokenizer.skip(.newline), .symbol_comma => self.tokenizer.skip_newlines(),
.symbol_brace_right => { .symbol_brace_right => {
self.tokenizer.step(); self.tokenizer.skip_newlines();
return Expression{.table_literal = table_literal}; break: get .{.table_literal = table_literal};
}, },
else => return self.report("expected `,` or `}` after expression"), else => return self.report("expected `,` or `}` after expression"),
@ -463,13 +471,13 @@ fn parse_factor(self: *Self) ParseError!Expression {
}, },
.symbol_minus => { .symbol_minus => {
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("expected expression after numeric negation (`-`)"); return self.report("expected expression after numeric negation (`-`)");
} }
return Expression{ break: get .{
.unary_operation = .{ .unary_operation = .{
.expression = try coral.io.allocate_one(allocator, try self.parse_factor()), .expression = try coral.io.allocate_one(allocator, try self.parse_factor()),
.operator = .numeric_negation, .operator = .numeric_negation,
@ -478,13 +486,13 @@ fn parse_factor(self: *Self) ParseError!Expression {
}, },
.symbol_bang => { .symbol_bang => {
self.tokenizer.skip(.newline); self.tokenizer.skip_newlines();
if (self.tokenizer.token == null) { if (self.tokenizer.token == .end) {
return self.report("expected expression after boolean negation (`!`)"); return self.report("expected expression after boolean negation (`!`)");
} }
return Expression{ break: get .{
.unary_operation = .{ .unary_operation = .{
.expression = try coral.io.allocate_one(allocator, try self.parse_factor()), .expression = try coral.io.allocate_one(allocator, try self.parse_factor()),
.operator = .boolean_negation, .operator = .boolean_negation,
@ -494,6 +502,37 @@ fn parse_factor(self: *Self) ParseError!Expression {
else => return self.report("unexpected token in expression"), else => return self.report("unexpected token in expression"),
} }
});
while (self.tokenizer.token == .symbol_period) {
self.tokenizer.skip_newlines();
const identifier = switch (self.tokenizer.token) {
.identifier => |identifier| identifier,
else => return self.report("expected identifier after `.`"),
};
self.tokenizer.skip_newlines();
expression = switch (self.tokenizer.token) {
.symbol_equals => .{
.set_field = .{
.value_expression = try coral.io.allocate_one(allocator, try self.parse_expression()),
.object_expression = try coral.io.allocate_one(allocator, expression),
.identifier = identifier,
},
},
else => .{
.get_field = .{
.object_expression = try coral.io.allocate_one(allocator, expression),
.identifier = identifier,
},
},
};
}
return expression;
} }
const parse_term = binary_operation_parser(parse_factor, &.{ const parse_term = binary_operation_parser(parse_factor, &.{

71
source/ona/kym/Chunk.zig
View File

@ -99,12 +99,35 @@ const AstCompiler = struct {
try self.compile_expression(grouped_expression.*); try self.compile_expression(grouped_expression.*);
}, },
.get_local => |local| { .resolve_local => |local| {
try self.chunk.append_opcode(.{ try self.chunk.append_opcode(.{
.push_local = self.resolve_local(local) orelse return self.chunk.env.raise(error.OutOfMemory, "undefined local"), .push_local = self.resolve_local(local) orelse {
return self.chunk.env.raise(error.OutOfMemory, "undefined local");
},
}); });
}, },
.get_field => |get_field| {
try self.compile_expression(get_field.object_expression.*);
try self.chunk.append_opcode(.{
.push_const = try self.chunk.declare_constant_symbol(get_field.identifier),
});
try self.chunk.append_opcode(.get_dynamic);
},
.set_field => |set_field| {
try self.compile_expression(set_field.object_expression.*);
try self.chunk.append_opcode(.{
.push_const = try self.chunk.declare_constant_symbol(set_field.identifier),
});
try self.compile_expression(set_field.value_expression.*);
try self.chunk.append_opcode(.set_dynamic);
},
.call_system => |call| { .call_system => |call| {
if (call.argument_expressions.values.len > coral.math.max_int(@typeInfo(u8).Int)) { if (call.argument_expressions.values.len > coral.math.max_int(@typeInfo(u8).Int)) {
return self.chunk.env.raise(error.OutOfMemory, "functions may receive a maximum of 255 locals"); return self.chunk.env.raise(error.OutOfMemory, "functions may receive a maximum of 255 locals");
@ -126,7 +149,7 @@ const AstCompiler = struct {
.return_expression => |return_expression| try self.compile_expression(return_expression), .return_expression => |return_expression| try self.compile_expression(return_expression),
.return_nothing => try self.chunk.append_opcode(.push_nil), .return_nothing => try self.chunk.append_opcode(.push_nil),
.set_local => |local| { .assign_local => |local| {
try self.compile_expression(local.expression); try self.compile_expression(local.expression);
if (self.resolve_local(local.identifier)) |index| { if (self.resolve_local(local.identifier)) |index| {
@ -193,6 +216,8 @@ pub const Opcode = union (enum) {
push_local: u8, push_local: u8,
push_table: u32, push_table: u32,
set_local: u8, set_local: u8,
get_dynamic,
set_dynamic,
call: u8, call: u8,
syscall: u8, syscall: u8,
@ -355,6 +380,46 @@ fn execute(self: *Self, env: *kym.RuntimeEnv) kym.RuntimeError!?*kym.RuntimeRef
try env.set_local(local, ref); try env.set_local(local, ref);
}, },
.get_dynamic => {
const index_ref = try env.pop_local() orelse {
return env.raise(error.TypeMismatch, "nil is not a valid index");
};
defer env.discard(index_ref);
const indexable_ref = try env.pop_local() orelse {
return env.raise(error.TypeMismatch, "nil is not a valid indexable");
};
defer env.discard(indexable_ref);
const value_ref = try env.get_dynamic(try kym.unbox_dynamic(env, indexable_ref), index_ref);
defer env.discard(value_ref);
try env.push_ref(value_ref);
},
.set_dynamic => {
const index_ref = try env.pop_local() orelse {
return env.raise(error.TypeMismatch, "nil is not a valid index");
};
defer env.discard(index_ref);
const indexable_ref = try env.pop_local() orelse {
return env.raise(error.TypeMismatch, "nil is not a valid indexable");
};
defer env.discard(indexable_ref);
const value_ref = try env.pop_local();
defer env.discard(value_ref);
try env.set_dynamic(try kym.unbox_dynamic(env, indexable_ref), index_ref, value_ref);
},
.call => |arg_count| { .call => |arg_count| {
const result_ref = call: { const result_ref = call: {
const callable_ref = try env.pop_local() orelse { const callable_ref = try env.pop_local() orelse {

24
source/ona/kym/tokens.zig
View File

@ -1,6 +1,7 @@
const coral = @import("coral"); const coral = @import("coral");
pub const Token = union(enum) { pub const Token = union(enum) {
end,
unknown: coral.io.Byte, unknown: coral.io.Byte,
newline, newline,
@ -41,6 +42,7 @@ pub const Token = union(enum) {
pub fn text(self: Token) []const coral.io.Byte { pub fn text(self: Token) []const coral.io.Byte {
return switch (self) { return switch (self) {
.end => "end",
.unknown => |unknown| @as([*]const coral.io.Byte, @ptrCast(&unknown))[0 .. 1], .unknown => |unknown| @as([*]const coral.io.Byte, @ptrCast(&unknown))[0 .. 1],
.newline => "newline", .newline => "newline",
@ -85,26 +87,12 @@ pub const Token = union(enum) {
pub const Tokenizer = struct { pub const Tokenizer = struct {
source: []const coral.io.Byte, source: []const coral.io.Byte,
lines_stepped: usize = 1, lines_stepped: usize = 1,
token: ?Token = null, token: Token = .end,
const TokenTag = coral.io.Tag(Token); pub fn skip_newlines(self: *Tokenizer) void {
pub fn is_token(self: *Tokenizer, token_tag: TokenTag) bool {
return if (self.token) |token| token == token_tag else false;
}
pub fn is_token_null_or(self: *Tokenizer, token_tag: TokenTag) bool {
return if (self.token) |token| token == token_tag else true;
}
pub fn skip(self: *Tokenizer, skip_token_tag: TokenTag) void {
self.step(); self.step();
while (self.token) |token| { while (self.token == .newline) {
if (token != skip_token_tag) {
return;
}
self.step(); self.step();
} }
} }
@ -429,7 +417,7 @@ pub const Tokenizer = struct {
} }
} }
self.token = null; self.token = .end;
return; return;
} }