From 06284e4196bb4ff8752c6aac8ee5b635d8f85f17 Mon Sep 17 00:00:00 2001 From: "Pascal P." Date: Fri, 17 Jul 2026 22:39:42 +0200 Subject: [PATCH] Fixed the escaped char pb --- src/lib.zig | 35 +++++++++++++++- src/parser.zig | 106 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 105 insertions(+), 36 deletions(-) diff --git a/src/lib.zig b/src/lib.zig index 9a86e62..ae6021a 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -1,14 +1,47 @@ const std = @import("std"); +pub const Flags = packed struct { + verbose: bool = false, +}; pub const OpCode = enum(u32) { FAILURE, SUCCESS, + LITERAL, + CATEGORY, // Category for meta chars }; + +const Categories = enum(u32) { + DIGIT, + NOT_DIGIT, + SPACE, + NOT_SPACE, + WORD, + NOT_WORD, +}; + +const TokenArgTag = enum(u8) { raw, category }; +const TokenArg = union(TokenArgTag) { + raw: u32, + category: Categories, +}; + pub const Token = struct { code: OpCode, - arg: u32, + arg: TokenArg, }; +pub const ESCAPES = "abfnrtv\\"; +pub const CATEGORIES = std.StaticStringMap(Token).initComptime(.{ + .{ "d", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.DIGIT } } }, + .{ "D", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_DIGIT } } }, + .{ "s", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.SPACE } } }, + .{ "S", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_SPACE } } }, + .{ "w", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.WORD } } }, + .{ "W", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_WORD } } }, +}); +pub const HEXDIGITS = "0123456789abcdefABCDEF"; +pub const ASCIILETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + pub const ParserCtx = struct { const Self = @This(); diff --git a/src/parser.zig b/src/parser.zig index e595a46..923bf46 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -8,6 +8,7 @@ const Tokenizer = struct { string: str, index: usize = 0, next: ?u8 = null, + is_next_escaped: bool = false, allocator: std.mem.Allocator, fn init(string: str, allocator: std.mem.Allocator) !Self { @@ -18,6 +19,7 @@ const Tokenizer = struct { fn __next(self: *Self) !void { var index = self.index; + self.is_next_escaped = false; var char: u8 = undefined; if (index < self.string.len) { @@ -30,7 +32,8 @@ const Tokenizer = struct { // It is an escaped character index += 1; if (index < self.string.len) { - char = self.string[index]; // FIXME: this should concatenate !! + self.is_next_escaped = true; + char = self.string[index]; } else { return error.BadEscapeEndOfPattern; } @@ -38,6 +41,7 @@ const Tokenizer = struct { self.index = index + 1; self.next = char; + std.debug.print("next: {c}[{d}]\n", .{ self.next.?, self.index }); } fn match(self: *Self, char: u8) !bool { @@ -62,12 +66,11 @@ const Tokenizer = struct { var c: u8 = undefined; for (0..n) |_| { + try self.__next(); c = self.next.?; - std.debug.print("trying {c}\n", .{c}); if (std.mem.findScalar(u8, charset, c) == null) break; result.appendAssumeCapacity(c); - try self.__next(); } try result.shrinkToLen(self.allocator); @@ -100,8 +103,9 @@ const Tokenizer = struct { } } - fn pos(self: *Self) u8 { - return self.index - (self.next orelse 0); + fn pos(self: *const Self) usize { + const next_len: usize = if (self.next == null) 0 else 1; + return self.index - next_len; } fn seek(self: *Self, index: u8) !void { @@ -111,7 +115,7 @@ const Tokenizer = struct { }; const State = struct { - flags: u8, + flags: lib.Flags, str: str, allocator: std.mem.Allocator, }; @@ -124,58 +128,90 @@ const SubPattern = struct { width: ?u8 = 0, }; -fn _parse(source: Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void { - var subpattern: SubPattern = .{ .state = state }; +fn _escape(source: *Tokenizer, char: u8, state: State) !lib.Token { + if (lib.CATEGORIES.get((&char)[0..1])) |token| return token; + if (std.mem.findScalar(u8, lib.ESCAPES, char) != null) return .{ .code = .LITERAL, .arg = .{ .raw = @intCast(char) } }; - const this: ?u8 = undefined; - while (true) { - this = source.next; - - if (this == null) break; - if (std.mem.findScalar(u8, "|)", this.?) != null) break; - try source.get(); + // Add all conditional escapes + switch (char) { + 'x' => { + var escaped_hex = try source.get_while(2, lib.HEXDIGITS); + defer escaped_hex.deinit(state.allocator); + std.debug.print("escaped hex: {s}\n", .{escaped_hex.items}); + if (escaped_hex.items.len != 2) return error.BadEscapeHex; + return .{ .code = .LITERAL, .arg = .{ .raw = try std.fmt.parseInt(u8, escaped_hex.items, 16) } }; + }, + // TODO: add unicode, octal, backreferences + else => { + if (std.mem.findScalar(u8, lib.ASCIILETTERS, char) == null) { + return .{ .code = .LITERAL, .arg = .{ .raw = @intCast(char) } }; + } + return error.BadEscapeSequence; + }, } } -fn __parse_sub(source: Tokenizer, state: State, verbose: bool, nested: u8) !void { +fn _parse(source: *Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void { + const subpattern: SubPattern = .{ .state = state }; + _ = subpattern; + _ = verbose; + _ = nested; + _ = first; + + var this: u8 = undefined; + while (true) { + if (source.next == null) break; + this = source.next.?; + + if (std.mem.findScalar(u8, "|)", this) != null) break; + + if (source.is_next_escaped) { + const token = try _escape(source, source.next.?, state); + std.debug.print("got escaped char {c}: {any}\n", .{ this, token }); + } + + _ = try source.get(); + } +} + +fn __parse_sub(source: *Tokenizer, state: State, verbose: bool, nested: u8) !void { const allocator = state.allocator; - const items: std.ArrayList(u8) = .empty; + var items: std.ArrayList(u8) = .empty; defer items.deinit(allocator); // FIXME: remove from here when returning const start = source.pos(); - while (bool) { + _ = start; + + while (true) { // TODO: add to items try _parse(source, state, verbose, nested, nested == 0 and items.items.len == 0); - if (!source.match('|')) break; + if (!try source.match('|')) break; } } -fn parse_with_flags(str_regex: str, flags: u8, allocator: std.mem.Allocator, _state: ?State) !void { - const source = try Tokenizer.init(str_regex, allocator); +fn __parse_with_flags(str_regex: str, flags: lib.Flags, allocator: std.mem.Allocator, state: State) !void { + var source = try Tokenizer.init(str_regex, allocator); - var state: State = undefined; - if (_state) |__s| { - state = __s; - } else { - state = .{ .flags = flags, .str = str_regex, .allocator = allocator }; - } + try __parse_sub(&source, state, flags.verbose, 0); } -fn parse(str_regex: str, allocator: std.mem.Allocator, state: ?State) !void { - return parse_with_flags(str_regex, 0, allocator, state); +fn parse_with_flags(str_regex: str, flags: lib.Flags, allocator: std.mem.Allocator) !void { + const state: State = .{ .flags = flags, .str = str_regex, .allocator = allocator }; + + return try __parse_with_flags(str_regex, flags, allocator, state); +} + +fn parse(str_regex: str, allocator: std.mem.Allocator) !void { + return try parse_with_flags(str_regex, .{}, allocator); } test "parser regex basic" { const alloc = std.testing.allocator; - const regex_to_parse = "[a-zA-Z0-9_\\]]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}"; + const regex_to_parse = "\\w+\\x40[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}"; std.debug.print("Trying to parse '{s}'\n", .{regex_to_parse}); var ctx = try lib.ParserCtx.init(alloc); defer ctx.deinit(); //try parse_regex(regex_to_parse, &ctx); - var tokenizer = try Tokenizer.init(regex_to_parse, alloc); - - var data = try tokenizer.get_until(']', "range"); - defer data.deinit(tokenizer.allocator); - std.debug.print("data: '{s}'\n", .{data.items}); + try parse(regex_to_parse, alloc); }