[wip] get closer to python

src/lib.zig

@@ -1,29 +1,26 @@
 const std = @import("std");
 
-pub const TokenGroupType = enum(u8) { group, namedGroup, nonCaptureGroup };
+pub const OpCode = enum(u32) {
-pub const TokenGroupNamed = struct { name: []u8, group: TokenGroupBase };
+    FAILURE,
-pub const TokenGroupNonCapturing = struct { group: TokenGroupBase };
+    SUCCESS,
-pub const TokenGroupBase = struct { value: u8 };
+};
-pub const TokenGroup = union(TokenGroupType) { group: TokenGroupBase, namedGroup: TokenGroupNamed, nonCaptureGroup: TokenGroupNonCapturing };
+pub const Token = struct {
-pub const TokenRange = struct { value: []u8 };
+    code: OpCode,
-pub const TokenUnion = struct { value: u8 };
+    arg: u32,
-pub const TokenLiteral = struct { value: u8 };
+};
-
-pub const TokenType = enum(u8) { group, literal, range, _union };
-pub const Token = union(TokenType) { group: TokenGroup, literal: TokenLiteral, range: TokenRange, _union: TokenUnion };
 
 pub const ParserCtx = struct {
     const Self = @This();
 
-    _allocator: std.mem.Allocator,
+    allocator: std.mem.Allocator,
+    flag: u8 = 0,
     pos: usize = 0,
-    tokens: std.MultiArrayList(Token),
+    tokens: std.ArrayList(Token),
     pub fn init(allocator: std.mem.Allocator) !Self {
-        const ctx: ParserCtx = .{ ._allocator = allocator, .tokens = .empty };
+        const ctx: ParserCtx = .{ .allocator = allocator, .tokens = .empty };
         return ctx;
     }
     pub fn deinit(self: *Self) void {
-        self.tokens.items()
+        self.tokens.deinit(self.allocator);
-        self.tokens.deinit(self._allocator);
     }
 };

src/parser.zig

@@ -1,97 +1,181 @@
 const std = @import("std");
 const lib = @import("lib.zig");
 
-fn parse_range(regex_str: []const u8, ctx: *lib.ParserCtx) !void {
+const str = []const u8;
-    const starting_pos = ctx.pos;
-    std.debug.print("parsing range from '{s}'\n", .{regex_str[starting_pos..]});
-    ctx.pos += 1;
 
-    var litterals: std.ArrayList(u8) = .empty;
+const Tokenizer = struct {
-    defer litterals.deinit(ctx._allocator);
+    const Self = @This();
-    var ranges: std.ArrayList([2]u8) = .empty;
+    string: str,
-    defer ranges.deinit(ctx._allocator);
+    index: usize = 0,
+    next: ?u8 = null,
+    allocator: std.mem.Allocator,
 
-    var last_ch: ?u8 = null;
+    fn init(string: str, allocator: std.mem.Allocator) !Self {
-    while (regex_str[ctx.pos] != ']') {
+        var tokenizer: Self = .{ .string = string, .allocator = allocator };
-        const ch = regex_str[ctx.pos];
+        try tokenizer.__next();
+        return tokenizer;
+    }
 
-        if (ch == '-' and last_ch != null and ctx.pos + 1 < regex_str.len) {
+    fn __next(self: *Self) !void {
-            const next_ch = regex_str[ctx.pos + 1];
+        var index = self.index;
-            std.debug.print("found range ? {c} to {c}\n", .{ last_ch.?, next_ch });
+
-            try ranges.append(ctx._allocator, .{ last_ch.?, next_ch });
+        var char: u8 = undefined;
-            _ = litterals.pop();
+        if (index < self.string.len) {
-            ctx.pos += 1;
+            char = self.string[index];
-            last_ch = null; // character in range cannot be reused
         } else {
-            try litterals.append(ctx._allocator, ch);
+            self.next = null;
-            last_ch = ch;
+            return;
-        }
-        ctx.pos += 1;
-
-        if (ctx.pos >= regex_str.len) {
-            std.debug.print("Unclosed '[' pos {d} in '{s}'\n", .{ starting_pos, regex_str[starting_pos..] });
-            return error.UnclosedBracket;
-        }
-    }
-
-    std.debug.print("range literrals: {s}\n", .{litterals.items});
-    std.debug.print("ranges: {any}\n", .{ranges.items});
-
-    var litteralSet: std.ArrayList(u8) = .empty;
-    defer litteralSet.deinit(ctx._allocator);
-    for (ranges.items) |item| {
-        const start = item[0];
-        const end: u8 = item[1];
-        if (end <= start) {
-            std.debug.print("Invalid range {c}-{c}\n", .{ start, end });
-            return error.InvalidRange;
-        }
-        for (start..end) |c| {
-            for (litteralSet.items) |set_item| {
-                if (c == set_item) {
-                    break;
         }
+        if (char == '\\') {
+            // It is an escaped character
+            index += 1;
+            if (index < self.string.len) {
+                char = self.string[index]; // FIXME: this should concatenate !!
             } else {
-                try litteralSet.append(ctx._allocator, @intCast(c));
+                return error.BadEscapeEndOfPattern;
-            }
-        }
-    }
-    for (litterals.items) |c| {
-        for (litteralSet.items) |set_item| {
-            if (c == set_item) {
-                break;
-            }
-        } else {
-            try litteralSet.append(ctx._allocator, @intCast(c));
             }
         }
 
-    std.debug.print("range covers: {s}\n", .{litteralSet.items});
+        self.index = index + 1;
-    try ctx.tokens.append(ctx._allocator, .{ .range = .{ .value = try ctx._allocator.dupe(u8, litteralSet.items) } });
+        self.next = char;
     }
-fn parse_regex(regex_str: []const u8, ctx: *lib.ParserCtx) !void {
+
-    const ch: u8 = regex_str[ctx.pos];
+    fn match(self: *Self, char: u8) !bool {
-    switch (ch) {
+        if (self.next) |c| {
-        '(' => {},
+            if (char == c) {
-        '[' => {
+                try self.__next();
-            try parse_range(regex_str, ctx);
+                return true;
-        },
-        '*' | '?' | '+' => {},
-        '{' => {},
-        else => {
-            // TODO: Handle special chars and escaped chars
-            try ctx.tokens.append(ctx._allocator, .{ .literal = .{ .value = ch } });
-        },
             }
         }
+        return false;
+    }
+
+    fn get(self: *Self) !?u8 {
+        const c = self.next;
+        try self.__next();
+        return c;
+    }
+
+    fn get_while(self: *Self, n: usize, charset: str) !std.ArrayList(u8) {
+        var result: std.ArrayList(u8) = .empty;
+        try result.ensureTotalCapacity(self.allocator, n);
+
+        var c: u8 = undefined;
+        for (0..n) |_| {
+            c = self.next.?;
+            std.debug.print("trying {c}\n", .{c});
+            if (std.mem.findScalar(u8, charset, c) == null) break;
+
+            result.appendAssumeCapacity(c);
+            try self.__next();
+        }
+
+        try result.shrinkToLen(self.allocator);
+        return result;
+    }
+
+    fn get_until(self: *Self, terminator: u8, name: str) !std.ArrayList(u8) {
+        var result: std.ArrayList(u8) = .empty;
+        var c: ?u8 = undefined;
+        while (true) {
+            c = self.next;
+            try self.__next();
+
+            if (c) |_c| {
+                if (_c == terminator) {
+                    if (result.items.len > 0) return result else {
+                        std.debug.print("missing {s}\n", .{name});
+                    }
+                }
+                try result.append(self.allocator, _c);
+            } else {
+                if (result.items.len == 0) {
+                    std.debug.print("missing {s} (1)\n", .{name});
+                    return error.MissingValues;
+                } else {
+                    std.debug.print("missing {c}, unterminated {s} ({d})\n", .{ terminator, name, result.items.len });
+                    return error.MissingTerminator;
+                }
+            }
+        }
+    }
+
+    fn pos(self: *Self) u8 {
+        return self.index - (self.next orelse 0);
+    }
+
+    fn seek(self: *Self, index: u8) !void {
+        self.index = index;
+        try self.__next();
+    }
+};
+
+const State = struct {
+    flags: u8,
+    str: str,
+    allocator: std.mem.Allocator,
+};
+
+const SubPattern = struct {
+    const Self = @This();
+
+    state: State,
+    data: std.ArrayList(lib.Token) = .empty,
+    width: ?u8 = 0,
+};
+
+fn _parse(source: Tokenizer, state: State, verbose: bool, nested: u8, first: bool) !void {
+    var subpattern: SubPattern = .{ .state = state };
+
+    const this: ?u8 = undefined;
+    while (true) {
+        this = source.next;
+
+        if (this == null) break;
+        if (std.mem.findScalar(u8, "|)", this.?) != null) break;
+        try source.get();
+    }
+}
+
+fn __parse_sub(source: Tokenizer, state: State, verbose: bool, nested: u8) !void {
+    const allocator = state.allocator;
+    const items: std.ArrayList(u8) = .empty;
+    defer items.deinit(allocator); // FIXME: remove from here when returning
+
+    const start = source.pos();
+    while (bool) {
+        // TODO: add to items
+        try _parse(source, state, verbose, nested, nested == 0 and items.items.len == 0);
+        if (!source.match('|')) break;
+    }
+}
+
+fn parse_with_flags(str_regex: str, flags: u8, allocator: std.mem.Allocator, _state: ?State) !void {
+    const source = try Tokenizer.init(str_regex, allocator);
+
+    var state: State = undefined;
+    if (_state) |__s| {
+        state = __s;
+    } else {
+        state = .{ .flags = flags, .str = str_regex, .allocator = allocator };
+    }
+}
+
+fn parse(str_regex: str, allocator: std.mem.Allocator, state: ?State) !void {
+    return parse_with_flags(str_regex, 0, allocator, state);
+}
 
 test "parser regex basic" {
     const alloc = std.testing.allocator;
-    const regex_to_parse = "[a-zA-Z0-9_]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}";
+    const regex_to_parse = "[a-zA-Z0-9_\\]]+@[a-zA-Z0-9_]+\\.[a-zA-Z]{2,}";
     std.debug.print("Trying to parse '{s}'\n", .{regex_to_parse});
     var ctx = try lib.ParserCtx.init(alloc);
     defer ctx.deinit();
-    try parse_regex(regex_to_parse, &ctx);
+    //try parse_regex(regex_to_parse, &ctx);
 
-    std.debug.print("tokens: {any}", .{ctx.tokens.pop()});
+    var tokenizer = try Tokenizer.init(regex_to_parse, alloc);
+
+    var data = try tokenizer.get_until(']', "range");
+    defer data.deinit(tokenizer.allocator);
+    std.debug.print("data: '{s}'\n", .{data.items});
 }