Fixed the escaped char pb

src/lib.zig

@@ -1,14 +1,47 @@
 const std = @import("std");
 
+pub const Flags = packed struct {
+    verbose: bool = false,
+};
 pub const OpCode = enum(u32) {
     FAILURE,
     SUCCESS,
+    LITERAL,
+    CATEGORY, // Category for meta chars
 };
+
+const Categories = enum(u32) {
+    DIGIT,
+    NOT_DIGIT,
+    SPACE,
+    NOT_SPACE,
+    WORD,
+    NOT_WORD,
+};
+
+const TokenArgTag = enum(u8) { raw, category };
+const TokenArg = union(TokenArgTag) {
+    raw: u32,
+    category: Categories,
+};
+
 pub const Token = struct {
     code: OpCode,
-    arg: u32,
+    arg: TokenArg,
 };
 
+pub const ESCAPES = "abfnrtv\\";
+pub const CATEGORIES = std.StaticStringMap(Token).initComptime(.{
+    .{ "d", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.DIGIT } } },
+    .{ "D", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_DIGIT } } },
+    .{ "s", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.SPACE } } },
+    .{ "S", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_SPACE } } },
+    .{ "w", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.WORD } } },
+    .{ "W", Token{ .code = .CATEGORY, .arg = .{ .category = Categories.NOT_WORD } } },
+});
+pub const HEXDIGITS = "0123456789abcdefABCDEF";
+pub const ASCIILETTERS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
 pub const ParserCtx = struct {
     const Self = @This();
 

src/parser.zig

@@ -8,6 +8,7 @@ const Tokenizer = struct {
     string: str,
     index: usize = 0,
     next: ?u8 = null,
+    is_next_escaped: bool = false,
     allocator: std.mem.Allocator,
 
     fn init(string: str, allocator: std.mem.Allocator) !Self {
@@ -18,6 +19,7 @@ const Tokenizer = struct {
 
     fn __next(self: *Self) !void {
         var index = self.index;
+        self.is_next_escaped = false;
 
         var char: u8 = undefined;
         if (index < self.string.len) {
@@ -30,7 +32,8 @@ const Tokenizer = struct {
             // It is an escaped character
             index += 1;
             if (index < self.string.len) {
-                char = self.string[index]; // FIXME: this should concatenate !!
+                self.is_next_escaped = true;
+                char = self.string[index];
             } else {
                 return error.BadEscapeEndOfPattern;
             }
@@ -38,6 +41,7 @@ const Tokenizer = struct {
 
         self.index = index + 1;
         self.next = char;
+        std.debug.print("next: {c}[{d}]\n", .{ self.next.?, self.index });
     }
 
     fn match(self: *Self, char: u8) !bool {
@@ -62,12 +66,11 @@ const Tokenizer = struct {
 
         var c: u8 = undefined;
         for (0..n) |_| {
+            try self.__next();
             c = self.next.?;
-            std.debug.print("trying {c}\n", .{c});
             if (std.mem.findScalar(u8, charset, c) == null) break;
 
             result.appendAssumeCapacity(c);
-            try self.__next();
         }
 
         try result.shrinkToLen(self.allocator);
@@ -100,8 +103,9 @@ const Tokenizer = struct {
         }
     }
 
-    fn pos(self: *Self) u8 {
+    fn pos(self: *const Self) usize {
-        return self.index - (self.next orelse 0);
+        const next_len: usize = if (self.next == null) 0 else 1;
+        return self.index - next_len;
     }
 
     fn seek(self: *Self, index: u8) !void {
@@ -111,7 +115,7 @@ const Tokenizer = struct {
 };
 
 const State = struct {
-    flags: u8,
+    flags: lib.Flags,
     str: str,
     allocator: std.mem.Allocator,
 };
@@ -124,58 +128,90 @@ const SubPattern = struct {
     width: ?u8 = 0,
 };
 
-fn _parse(source: Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void {
+fn _escape(source: *Tokenizer, char: u8, state: State) !lib.Token {
-    var subpattern: SubPattern = .{ .state = state };
+    if (lib.CATEGORIES.get((&char)[0..1])) |token| return token;
+    if (std.mem.findScalar(u8, lib.ESCAPES, char) != null) return .{ .code = .LITERAL, .arg = .{ .raw = @intCast(char) } };
 
-    const this: ?u8 = undefined;
+    // Add all conditional escapes
+    switch (char) {
+        'x' => {
+            var escaped_hex = try source.get_while(2, lib.HEXDIGITS);
+            defer escaped_hex.deinit(state.allocator);
+            std.debug.print("escaped hex: {s}\n", .{escaped_hex.items});
+            if (escaped_hex.items.len != 2) return error.BadEscapeHex;
+            return .{ .code = .LITERAL, .arg = .{ .raw = try std.fmt.parseInt(u8, escaped_hex.items, 16) } };
+        },
+        // TODO: add unicode, octal, backreferences
+        else => {
+            if (std.mem.findScalar(u8, lib.ASCIILETTERS, char) == null) {
+                return .{ .code = .LITERAL, .arg = .{ .raw = @intCast(char) } };
+            }
+            return error.BadEscapeSequence;
+        },
+    }
+}
+
+fn _parse(source: *Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void {
+    const subpattern: SubPattern = .{ .state = state };
+    _ = subpattern;
+    _ = verbose;
+    _ = nested;
+    _ = first;
+
+    var this: u8 = undefined;
     while (true) {
-        this = source.next;
+        if (source.next == null) break;
+        this = source.next.?;
 
-        if (this == null) break;
+        if (std.mem.findScalar(u8, "|)", this) != null) break;
-        if (std.mem.findScalar(u8, "|)", this.?) != null) break;
+
-        try source.get();
+        if (source.is_next_escaped) {
+            const token = try _escape(source, source.next.?, state);
+            std.debug.print("got escaped char {c}: {any}\n", .{ this, token });
+        }
+
+        _ = try source.get();
     }
 }
 
-fn __parse_sub(source: Tokenizer, state: State, verbose: bool, nested: u8) !void {
+fn __parse_sub(source: *Tokenizer, state: State, verbose: bool, nested: u8) !void {
     const allocator = state.allocator;
-    const items: std.ArrayList(u8) = .empty;
+    var items: std.ArrayList(u8) = .empty;
     defer items.deinit(allocator); // FIXME: remove from here when returning
 
     const start = source.pos();
-    while (bool) {
+    _ = start;
+
+    while (true) {
         // TODO: add to items
         try _parse(source, state, verbose, nested, nested == 0 and items.items.len == 0);
-        if (!source.match('|')) break;
+        if (!try source.match('|')) break;
     }
 }
 
-fn parse_with_flags(str_regex: str, flags: u8, allocator: std.mem.Allocator, _state: ?State) !void {
+fn __parse_with_flags(str_regex: str, flags: lib.Flags, allocator: std.mem.Allocator, state: State) !void {
-    const source = try Tokenizer.init(str_regex, allocator);
+    var source = try Tokenizer.init(str_regex, allocator);
 
-    var state: State = undefined;
+    try __parse_sub(&source, state, flags.verbose, 0);
-    if (_state) |__s| {
-        state = __s;
-    } else {
-        state = .{ .flags = flags, .str = str_regex, .allocator = allocator };
-    }
 }
 
-fn parse(str_regex: str, allocator: std.mem.Allocator, state: ?State) !void {
+fn parse_with_flags(str_regex: str, flags: lib.Flags, allocator: std.mem.Allocator) !void {
-    return parse_with_flags(str_regex, 0, allocator, state);
+    const state: State = .{ .flags = flags, .str = str_regex, .allocator = allocator };
+
+    return try __parse_with_flags(str_regex, flags, allocator, state);
+}
+
+fn parse(str_regex: str, allocator: std.mem.Allocator) !void {
+    return try parse_with_flags(str_regex, .{}, allocator);
 }
 
 test "parser regex basic" {
     const alloc = std.testing.allocator;
-    const regex_to_parse = "[a-zA-Z0-9_\\]]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}";
+    const regex_to_parse = "\\w+\\x40[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}";
     std.debug.print("Trying to parse '{s}'\n", .{regex_to_parse});
     var ctx = try lib.ParserCtx.init(alloc);
     defer ctx.deinit();
     //try parse_regex(regex_to_parse, &ctx);
 
-    var tokenizer = try Tokenizer.init(regex_to_parse, alloc);
+    try parse(regex_to_parse, alloc);
-
-    var data = try tokenizer.get_until(']', "range");
-    defer data.deinit(tokenizer.allocator);
-    std.debug.print("data: '{s}'\n", .{data.items});
 }