Merge pull request #1927 from lightpanda-io/feat/fetch-wait-options

Feat/fetch wait options

src/Config.zig

@@ -217,6 +217,13 @@ pub const DumpFormat = enum {
     semantic_tree_text,
 };
 
+pub const WaitUntil = enum {
+    load,
+    domcontentloaded,
+    networkidle,
+    fixed,
+};
+
 pub const Fetch = struct {
     url: [:0]const u8,
     dump_mode: ?DumpFormat = null,
@@ -224,6 +231,8 @@ pub const Fetch = struct {
     with_base: bool = false,
     with_frames: bool = false,
     strip: dump.Opts.Strip = .{},
+    wait_ms: u32 = 5000,
+    wait_until: WaitUntil = .load,
 };
 
 pub const Common = struct {
@@ -387,6 +396,13 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
         \\
         \\--with_frames   Includes the contents of iframes. Defaults to false.
         \\
+        \\--wait_ms       Wait time in milliseconds.
+        \\                Defaults to 5000.
+        \\
+        \\--wait_until    Wait until the specified event.
+        \\                Supported events: load, domcontentloaded, networkidle, fixed.
+        \\                Defaults to 'load'.
+        \\
     ++ common_options ++
         \\
         \\serve command
@@ -619,8 +635,34 @@ fn parseFetchArgs(
     var url: ?[:0]const u8 = null;
     var common: Common = .{};
     var strip: dump.Opts.Strip = .{};
+    var wait_ms: u32 = 5000;
+    var wait_until: WaitUntil = .load;
 
     while (args.next()) |opt| {
+        if (std.mem.eql(u8, "--wait_ms", opt)) {
+            const str = args.next() orelse {
+                log.fatal(.app, "missing argument value", .{ .arg = "--wait_ms" });
+                return error.InvalidArgument;
+            };
+            wait_ms = std.fmt.parseInt(u32, str, 10) catch |err| {
+                log.fatal(.app, "invalid argument value", .{ .arg = "--wait_ms", .err = err });
+                return error.InvalidArgument;
+            };
+            continue;
+        }
+
+        if (std.mem.eql(u8, "--wait_until", opt)) {
+            const str = args.next() orelse {
+                log.fatal(.app, "missing argument value", .{ .arg = "--wait_until" });
+                return error.InvalidArgument;
+            };
+            wait_until = std.meta.stringToEnum(WaitUntil, str) orelse {
+                log.fatal(.app, "invalid argument value", .{ .arg = "--wait_until", .val = str });
+                return error.InvalidArgument;
+            };
+            continue;
+        }
+
         if (std.mem.eql(u8, "--dump", opt)) {
             var peek_args = args.*;
             if (peek_args.next()) |next_arg| {
@@ -709,6 +751,8 @@ fn parseFetchArgs(
         .common = common,
         .with_base = with_base,
         .with_frames = with_frames,
+        .wait_ms = wait_ms,
+        .wait_until = wait_until,
     };
 }
 

src/browser/Session.zig

@@ -319,10 +319,15 @@ fn findPageBy(page: *Page, comptime field: []const u8, id: u32) ?*Page {
     return null;
 }
 
-pub fn wait(self: *Session, wait_ms: u32) WaitResult {
+const WaitOpts = struct {
+    timeout_ms: u32 = 5000,
+    until: lp.Config.WaitUntil = .load,
+};
+
+pub fn wait(self: *Session, opts: WaitOpts) WaitResult {
     var page = &(self.page orelse return .no_page);
     while (true) {
-        const wait_result = self._wait(page, wait_ms) catch |err| {
+        const wait_result = self._wait(page, opts) catch |err| {
             switch (err) {
                 error.JsError => {}, // already logged (with hopefully more context)
                 else => log.err(.browser, "session wait", .{
@@ -346,9 +351,11 @@ pub fn wait(self: *Session, wait_ms: u32) WaitResult {
     }
 }
 
-fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
+fn _wait(self: *Session, page: *Page, opts: WaitOpts) !WaitResult {
+    const wait_until = opts.until;
+
     var timer = try std.time.Timer.start();
-    var ms_remaining = wait_ms;
+    var ms_remaining = opts.timeout_ms;
 
     const browser = self.browser;
     var http_client = browser.http_client;
@@ -372,7 +379,9 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                 // There's no JS to run, and no reason to run the scheduler.
                 if (http_client.active == 0 and exit_when_done) {
                     // haven't started navigating, I guess.
-                    return .done;
+                    if (wait_until != .fixed) {
+                        return .done;
+                    }
                 }
                 // Either we have active http connections, or we're in CDP
                 // mode with an extra socket. Either way, we're waiting
@@ -423,17 +432,14 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                         std.debug.assert(http_client.intercepted == 0);
                     }
 
-                    var ms = blk: {
+                    const is_event_done = switch (wait_until) {
-                        // if (wait_ms - ms_remaining < 100) {
+                        .fixed => false,
-                        //     if (comptime builtin.is_test) {
+                        .domcontentloaded => (page._load_state == .load or page._load_state == .complete),
-                        //         return .done;
+                        .load => (page._load_state == .complete),
-                        //     }
+                        .networkidle => (page._notified_network_idle == .done),
-                        //     // Look, we want to exit ASAP, but we don't want
+                    };
-                        //     // to exit so fast that we've run none of the
-                        //     // background jobs.
-                        //     break :blk 50;
-                        // }
 
+                    var ms = blk: {
                         if (browser.hasBackgroundTasks()) {
                             // _we_ have nothing to run, but v8 is working on
                             // background tasks. We'll wait for them.
@@ -441,19 +447,27 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
                             break :blk 20;
                         }
 
-                        break :blk browser.msToNextMacrotask() orelse return .done;
+                        const next_task = browser.msToNextMacrotask();
+                        if (next_task == null and is_event_done) {
+                            return .done;
+                        }
+                        break :blk next_task orelse 20;
                     };
 
                     if (ms > ms_remaining) {
+                        if (is_event_done) {
+                            return .done;
+                        }
                         // Same as above, except we have a scheduled task,
                         // it just happens to be too far into the future
                         // compared to how long we were told to wait.
-                        if (!browser.hasBackgroundTasks()) {
+                        if (browser.hasBackgroundTasks()) {
-                            return .done;
+                            // _we_ have nothing to run, but v8 is working on
+                            // background tasks. We'll wait for them.
+                            browser.waitForBackgroundTasks();
                         }
-                        // _we_ have nothing to run, but v8 is working on
+                        // We're still wait for our wait_until. Not sure for what
-                        // background tasks. We'll wait for them.
+                        // but let's keep waiting. Worst case, we'll timeout.
-                        browser.waitForBackgroundTasks();
                         ms = 20;
                     }
 

src/cdp/cdp.zig

@@ -131,7 +131,7 @@ pub fn CDPT(comptime TypeProvider: type) type {
         // timeouts (or http events) which are ready to be processed.
         pub fn pageWait(self: *Self, ms: u32) Session.WaitResult {
             const session = &(self.browser.session orelse return .no_page);
-            return session.wait(ms);
+            return session.wait(.{ .timeout_ms = ms });
         }
 
         // Called from above, in processMessage which handles client messages

src/cdp/domains/lp.zig

@@ -288,7 +288,7 @@ test "cdp.lp: action tools" {
     const page = try bc.session.createPage();
     const url = "http://localhost:9582/src/browser/tests/mcp_actions.html";
     try page.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } });
-    _ = bc.session.wait(5000);
+    _ = bc.session.wait(.{});
 
     // Test Click
     const btn = page.document.getElementById("btn", page).?.asNode();

src/cdp/testing.zig

@@ -136,7 +136,7 @@ const TestContext = struct {
                 0,
             );
             try page.navigate(full_url, .{});
-            _ = bc.session.wait(2000);
+            _ = bc.session.wait(.{});
         }
         return bc;
     }

src/lightpanda.zig

@@ -46,6 +46,7 @@ const IS_DEBUG = @import("builtin").mode == .Debug;
 
 pub const FetchOpts = struct {
     wait_ms: u32 = 5000,
+    wait_until: Config.WaitUntil = .load,
     dump: dump.Opts,
     dump_mode: ?Config.DumpFormat = null,
     writer: ?*std.Io.Writer = null,
@@ -107,7 +108,7 @@ pub fn fetch(app: *App, url: [:0]const u8, opts: FetchOpts) !void {
         .reason = .address_bar,
         .kind = .{ .push = null },
     });
-    _ = session.wait(opts.wait_ms);
+    _ = session.wait(.{ .timeout_ms = opts.wait_ms, .until = opts.wait_until });
 
     const writer = opts.writer orelse return;
     if (opts.dump_mode) |mode| {

src/main.zig

@@ -124,7 +124,8 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
             log.debug(.app, "startup", .{ .mode = "fetch", .dump_mode = opts.dump_mode, .url = url, .snapshot = app.snapshot.fromEmbedded() });
 
             var fetch_opts = lp.FetchOpts{
-                .wait_ms = 5000,
+                .wait_ms = opts.wait_ms,
+                .wait_until = opts.wait_until,
                 .dump_mode = opts.dump_mode,
                 .dump = .{
                     .strip = opts.strip,

src/main_legacy_test.zig

@@ -106,7 +106,7 @@ pub fn run(allocator: Allocator, file: []const u8, session: *lp.Session) !void {
     defer try_catch.deinit();
 
     try page.navigate(url, .{});
-    _ = session.wait(2000);
+    _ = session.wait(.{});
 
     ls.local.eval("testing.assertOk()", "testing.assertOk()") catch |err| {
         const caught = try_catch.caughtOrError(allocator, err);

src/mcp/tools.zig

@@ -558,7 +558,7 @@ fn performGoto(server: *Server, url: [:0]const u8, id: std.json.Value) !void {
         return error.NavigationFailed;
     };
 
-    _ = server.session.wait(5000);
+    _ = server.session.wait(.{});
 }
 
 const testing = @import("../testing.zig");
@@ -623,7 +623,7 @@ test "MCP - Actions: click, fill, scroll" {
     const page = try server.session.createPage();
     const url = "http://localhost:9582/src/browser/tests/mcp_actions.html";
     try page.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } });
-    _ = server.session.wait(5000);
+    _ = server.session.wait(.{});
 
     // Test Click
     const btn = page.document.getElementById("btn", page).?.asNode();

src/testing.zig

@@ -415,7 +415,7 @@ fn runWebApiTest(test_file: [:0]const u8) !void {
     defer try_catch.deinit();
 
     try page.navigate(url, .{});
-    _ = test_session.wait(2000);
+    _ = test_session.wait(.{});
 
     test_browser.runMicrotasks();
 
@@ -439,7 +439,7 @@ pub fn pageTest(comptime test_file: []const u8) !*Page {
     );
 
     try page.navigate(url, .{});
-    _ = test_session.wait(2000);
+    _ = test_session.wait(.{});
     return page;
 }