From 792f70ae30ee50c4d5aa974008ae5ad3399d2310 Mon Sep 17 00:00:00 2001 From: "Pascal P." Date: Fri, 17 Jul 2026 09:13:37 +0200 Subject: [PATCH] [wip] get closer to python --- src/lib.zig | 29 +++---- src/parser.zig | 224 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 167 insertions(+), 86 deletions(-) diff --git a/src/lib.zig b/src/lib.zig index 7046186..9a86e62 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -1,29 +1,26 @@ const std = @import("std"); -pub const TokenGroupType = enum(u8) { group, namedGroup, nonCaptureGroup }; -pub const TokenGroupNamed = struct { name: []u8, group: TokenGroupBase }; -pub const TokenGroupNonCapturing = struct { group: TokenGroupBase }; -pub const TokenGroupBase = struct { value: u8 }; -pub const TokenGroup = union(TokenGroupType) { group: TokenGroupBase, namedGroup: TokenGroupNamed, nonCaptureGroup: TokenGroupNonCapturing }; -pub const TokenRange = struct { value: []u8 }; -pub const TokenUnion = struct { value: u8 }; -pub const TokenLiteral = struct { value: u8 }; - -pub const TokenType = enum(u8) { group, literal, range, _union }; -pub const Token = union(TokenType) { group: TokenGroup, literal: TokenLiteral, range: TokenRange, _union: TokenUnion }; +pub const OpCode = enum(u32) { + FAILURE, + SUCCESS, +}; +pub const Token = struct { + code: OpCode, + arg: u32, +}; pub const ParserCtx = struct { const Self = @This(); - _allocator: std.mem.Allocator, + allocator: std.mem.Allocator, + flag: u8 = 0, pos: usize = 0, - tokens: std.MultiArrayList(Token), + tokens: std.ArrayList(Token), pub fn init(allocator: std.mem.Allocator) !Self { - const ctx: ParserCtx = .{ ._allocator = allocator, .tokens = .empty }; + const ctx: ParserCtx = .{ .allocator = allocator, .tokens = .empty }; return ctx; } pub fn deinit(self: *Self) void { - self.tokens.items() - self.tokens.deinit(self._allocator); + self.tokens.deinit(self.allocator); } }; diff --git a/src/parser.zig b/src/parser.zig index c8336e8..e595a46 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -1,97 +1,181 @@ const std = @import("std"); const lib = @import("lib.zig"); -fn parse_range(regex_str: []const u8, ctx: *lib.ParserCtx) !void { - const starting_pos = ctx.pos; - std.debug.print("parsing range from '{s}'\n", .{regex_str[starting_pos..]}); - ctx.pos += 1; +const str = []const u8; - var litterals: std.ArrayList(u8) = .empty; - defer litterals.deinit(ctx._allocator); - var ranges: std.ArrayList([2]u8) = .empty; - defer ranges.deinit(ctx._allocator); +const Tokenizer = struct { + const Self = @This(); + string: str, + index: usize = 0, + next: ?u8 = null, + allocator: std.mem.Allocator, - var last_ch: ?u8 = null; - while (regex_str[ctx.pos] != ']') { - const ch = regex_str[ctx.pos]; - - if (ch == '-' and last_ch != null and ctx.pos + 1 < regex_str.len) { - const next_ch = regex_str[ctx.pos + 1]; - std.debug.print("found range ? {c} to {c}\n", .{ last_ch.?, next_ch }); - try ranges.append(ctx._allocator, .{ last_ch.?, next_ch }); - _ = litterals.pop(); - ctx.pos += 1; - last_ch = null; // character in range cannot be reused - } else { - try litterals.append(ctx._allocator, ch); - last_ch = ch; - } - ctx.pos += 1; - - if (ctx.pos >= regex_str.len) { - std.debug.print("Unclosed '[' pos {d} in '{s}'\n", .{ starting_pos, regex_str[starting_pos..] }); - return error.UnclosedBracket; - } + fn init(string: str, allocator: std.mem.Allocator) !Self { + var tokenizer: Self = .{ .string = string, .allocator = allocator }; + try tokenizer.__next(); + return tokenizer; } - std.debug.print("range literrals: {s}\n", .{litterals.items}); - std.debug.print("ranges: {any}\n", .{ranges.items}); + fn __next(self: *Self) !void { + var index = self.index; - var litteralSet: std.ArrayList(u8) = .empty; - defer litteralSet.deinit(ctx._allocator); - for (ranges.items) |item| { - const start = item[0]; - const end: u8 = item[1]; - if (end <= start) { - std.debug.print("Invalid range {c}-{c}\n", .{ start, end }); - return error.InvalidRange; + var char: u8 = undefined; + if (index < self.string.len) { + char = self.string[index]; + } else { + self.next = null; + return; } - for (start..end) |c| { - for (litteralSet.items) |set_item| { - if (c == set_item) { - break; - } + if (char == '\\') { + // It is an escaped character + index += 1; + if (index < self.string.len) { + char = self.string[index]; // FIXME: this should concatenate !! } else { - try litteralSet.append(ctx._allocator, @intCast(c)); + return error.BadEscapeEndOfPattern; } } + + self.index = index + 1; + self.next = char; } - for (litterals.items) |c| { - for (litteralSet.items) |set_item| { - if (c == set_item) { - break; + + fn match(self: *Self, char: u8) !bool { + if (self.next) |c| { + if (char == c) { + try self.__next(); + return true; + } + } + return false; + } + + fn get(self: *Self) !?u8 { + const c = self.next; + try self.__next(); + return c; + } + + fn get_while(self: *Self, n: usize, charset: str) !std.ArrayList(u8) { + var result: std.ArrayList(u8) = .empty; + try result.ensureTotalCapacity(self.allocator, n); + + var c: u8 = undefined; + for (0..n) |_| { + c = self.next.?; + std.debug.print("trying {c}\n", .{c}); + if (std.mem.findScalar(u8, charset, c) == null) break; + + result.appendAssumeCapacity(c); + try self.__next(); + } + + try result.shrinkToLen(self.allocator); + return result; + } + + fn get_until(self: *Self, terminator: u8, name: str) !std.ArrayList(u8) { + var result: std.ArrayList(u8) = .empty; + var c: ?u8 = undefined; + while (true) { + c = self.next; + try self.__next(); + + if (c) |_c| { + if (_c == terminator) { + if (result.items.len > 0) return result else { + std.debug.print("missing {s}\n", .{name}); + } + } + try result.append(self.allocator, _c); + } else { + if (result.items.len == 0) { + std.debug.print("missing {s} (1)\n", .{name}); + return error.MissingValues; + } else { + std.debug.print("missing {c}, unterminated {s} ({d})\n", .{ terminator, name, result.items.len }); + return error.MissingTerminator; + } } - } else { - try litteralSet.append(ctx._allocator, @intCast(c)); } } - std.debug.print("range covers: {s}\n", .{litteralSet.items}); - try ctx.tokens.append(ctx._allocator, .{ .range = .{ .value = try ctx._allocator.dupe(u8, litteralSet.items) } }); -} -fn parse_regex(regex_str: []const u8, ctx: *lib.ParserCtx) !void { - const ch: u8 = regex_str[ctx.pos]; - switch (ch) { - '(' => {}, - '[' => { - try parse_range(regex_str, ctx); - }, - '*' | '?' | '+' => {}, - '{' => {}, - else => { - // TODO: Handle special chars and escaped chars - try ctx.tokens.append(ctx._allocator, .{ .literal = .{ .value = ch } }); - }, + fn pos(self: *Self) u8 { + return self.index - (self.next orelse 0); } + + fn seek(self: *Self, index: u8) !void { + self.index = index; + try self.__next(); + } +}; + +const State = struct { + flags: u8, + str: str, + allocator: std.mem.Allocator, +}; + +const SubPattern = struct { + const Self = @This(); + + state: State, + data: std.ArrayList(lib.Token) = .empty, + width: ?u8 = 0, +}; + +fn _parse(source: Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void { + var subpattern: SubPattern = .{ .state = state }; + + const this: ?u8 = undefined; + while (true) { + this = source.next; + + if (this == null) break; + if (std.mem.findScalar(u8, "|)", this.?) != null) break; + try source.get(); + } +} + +fn __parse_sub(source: Tokenizer, state: State, verbose: bool, nested: u8) !void { + const allocator = state.allocator; + const items: std.ArrayList(u8) = .empty; + defer items.deinit(allocator); // FIXME: remove from here when returning + + const start = source.pos(); + while (bool) { + // TODO: add to items + try _parse(source, state, verbose, nested, nested == 0 and items.items.len == 0); + if (!source.match('|')) break; + } +} + +fn parse_with_flags(str_regex: str, flags: u8, allocator: std.mem.Allocator, _state: ?State) !void { + const source = try Tokenizer.init(str_regex, allocator); + + var state: State = undefined; + if (_state) |__s| { + state = __s; + } else { + state = .{ .flags = flags, .str = str_regex, .allocator = allocator }; + } +} + +fn parse(str_regex: str, allocator: std.mem.Allocator, state: ?State) !void { + return parse_with_flags(str_regex, 0, allocator, state); } test "parser regex basic" { const alloc = std.testing.allocator; - const regex_to_parse = "[a-zA-Z0-9_]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}"; + const regex_to_parse = "[a-zA-Z0-9_\\]]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}"; std.debug.print("Trying to parse '{s}'\n", .{regex_to_parse}); var ctx = try lib.ParserCtx.init(alloc); defer ctx.deinit(); - try parse_regex(regex_to_parse, &ctx); + //try parse_regex(regex_to_parse, &ctx); - std.debug.print("tokens: {any}", .{ctx.tokens.pop()}); + var tokenizer = try Tokenizer.init(regex_to_parse, alloc); + + var data = try tokenizer.get_until(']', "range"); + defer data.deinit(tokenizer.allocator); + std.debug.print("data: '{s}'\n", .{data.items}); }