Merge pull request #482 from karlseguin/http_client

Replace zig-async-io with a custom HTTP client

.gitmodules

@@ -23,10 +23,3 @@
 [submodule "vendor/mimalloc"]
 	path = vendor/mimalloc
 	url = https://github.com/microsoft/mimalloc.git/
-[submodule "vendor/tls.zig"]
-	path = vendor/tls.zig
-	url = https://github.com/ianic/tls.zig.git/
-[submodule "vendor/zig-async-io"]
-	path = vendor/zig-async-io
-	url = https://github.com/lightpanda-io/zig-async-io.git/
-	branch = zig-0.14

LICENSING.md

@@ -10,7 +10,6 @@ The default license for this project is [AGPL-3.0-only](LICENSE).
 The following files are licensed under MIT:
 
 ```
-src/http/Client.zig
 src/polyfill/fetch.js
 ```
 

build.zig

@@ -133,8 +133,8 @@ pub fn build(b: *std.Build) !void {
 
     // compile
     const unit_tests = b.addTest(.{
-        .root_source_file = b.path("src/unit_tests.zig"),
-        .test_runner = .{ .path = b.path("src/unit_tests.zig"), .mode = .simple },
+        .root_source_file = b.path("src/main_unit_tests.zig"),
+        .test_runner = .{ .path = b.path("src/test_runner.zig"), .mode = .simple },
         .target = target,
         .optimize = mode,
     });
@@ -177,6 +177,9 @@ fn common(
     options: jsruntime.Options,
 ) !void {
     const target = step.root_module.resolved_target.?;
+    const optimize = step.root_module.optimize.?;
+    const dep_opts = .{ .target = target, .optimize = optimize };
+
     const jsruntimemod = try jsruntime_pkgs.module(
         b,
         options,
@@ -189,15 +192,7 @@ fn common(
     netsurf.addImport("jsruntime", jsruntimemod);
     step.root_module.addImport("netsurf", netsurf);
 
-    const asyncio = b.addModule("asyncio", .{
-        .root_source_file = b.path("vendor/zig-async-io/src/lib.zig"),
-    });
-    step.root_module.addImport("asyncio", asyncio);
-
-    const tlsmod = b.addModule("tls", .{
-        .root_source_file = b.path("vendor/tls.zig/src/root.zig"),
-    });
-    step.root_module.addImport("tls", tlsmod);
+    step.root_module.addImport("tls", b.dependency("tls", dep_opts).module("tls"));
 }
 
 fn moduleNetSurf(b: *std.Build, target: std.Build.ResolvedTarget) !*std.Build.Module {

build.zig.zon

@@ -0,0 +1,12 @@
+.{
+    .name = .browser,
+    .paths = .{""},
+    .version = "0.0.0",
+    .fingerprint = 0xda130f3af836cea0,
+    .dependencies = .{
+        .tls = .{
+            .url = "https://github.com/ianic/tls.zig/archive/b29a8b45fc59fc2d202769c4f54509bb9e17d0a2.tar.gz",
+            .hash = "1220e6fd39920dd6e28b2bc06688787a39430f8856f0597cd77c44ca868c6c54fb86",
+        },
+    },
+}

src/app.zig

@@ -2,7 +2,7 @@ const std = @import("std");
 
 const Loop = @import("jsruntime").Loop;
 const Allocator = std.mem.Allocator;
-const HttpClient = @import("http/Client.zig");
+const HttpClient = @import("http/client.zig").Client;
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 
 const log = std.log.scoped(.app);
@@ -38,7 +38,7 @@ pub const App = struct {
             .allocator = allocator,
             .telemetry = undefined,
             .app_dir_path = app_dir_path,
-            .http_client = .{ .allocator = allocator },
+            .http_client = try HttpClient.init(allocator, 5),
         };
         app.telemetry = Telemetry.init(app, run_mode);
 

src/browser/browser.zig

@@ -24,7 +24,6 @@ const Allocator = std.mem.Allocator;
 const Types = @import("root").Types;
 
 const parser = @import("netsurf");
-const Loader = @import("loader.zig").Loader;
 const Dump = @import("dump.zig");
 const Mime = @import("mime.zig").Mime;
 
@@ -44,10 +43,8 @@ const Location = @import("../html/location.zig").Location;
 
 const storage = @import("../storage/storage.zig");
 
-const FetchResult = @import("../http/Client.zig").Client.FetchResult;
-
+const HttpClient = @import("../http/client.zig").Client;
 const UserContext = @import("../user_context.zig").UserContext;
-const HttpClient = @import("asyncio").Client;
 
 const polyfill = @import("../polyfill/polyfill.zig");
 
@@ -75,7 +72,7 @@ pub const Browser = struct {
             .app = app,
             .session = null,
             .allocator = allocator,
-            .http_client = @ptrCast(&app.http_client),
+            .http_client = &app.http_client,
             .session_pool = SessionPool.init(allocator),
             .page_arena = std.heap.ArenaAllocator.init(allocator),
         };
@@ -121,9 +118,6 @@ pub const Session = struct {
     // all others Session deps use directly self.alloc and not the arena.
     arena: std.heap.ArenaAllocator,
 
-    // TODO handle proxy
-    loader: Loader,
-
     env: Env,
     inspector: jsruntime.Inspector,
 
@@ -132,6 +126,7 @@ pub const Session = struct {
     // TODO move the shed to the browser?
     storage_shed: storage.Shed,
     page: ?Page = null,
+    http_client: *HttpClient,
 
     jstypes: [Types.len]usize = undefined,
 
@@ -143,7 +138,7 @@ pub const Session = struct {
             .env = undefined,
             .browser = browser,
             .inspector = undefined,
-            .loader = Loader.init(allocator),
+            .http_client = browser.http_client,
             .storage_shed = storage.Shed.init(allocator),
             .arena = std.heap.ArenaAllocator.init(allocator),
             .window = Window.create(null, .{ .agent = user_agent }),
@@ -181,7 +176,6 @@ pub const Session = struct {
         }
         self.env.deinit();
         self.arena.deinit();
-        self.loader.deinit();
         self.storage_shed.deinit();
     }
 
@@ -370,32 +364,14 @@ pub const Page = struct {
         } });
 
         // load the data
-        var resp = try self.session.loader.get(arena, self.uri);
-        defer resp.deinit();
+        var request = try self.session.http_client.request(.GET, self.uri);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        const req = resp.req;
+        const header = response.header;
+        log.info("GET {any} {d}", .{ self.uri, header.status });
 
-        log.info("GET {any} {d}", .{ self.uri, @intFromEnum(req.response.status) });
-
-        // TODO handle redirection
-        log.debug("{?} {d} {s}", .{
-            req.response.version,
-            @intFromEnum(req.response.status),
-            req.response.reason,
-            // TODO log headers
-        });
-
-        // TODO handle charset
-        // https://html.spec.whatwg.org/#content-type
-        var it = req.response.iterateHeaders();
-        var ct_: ?[]const u8 = null;
-        while (true) {
-            const h = it.next() orelse break;
-            if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) {
-                ct_ = try arena.dupe(u8, h.value);
-            }
-        }
-        const ct = ct_ orelse {
+        const ct = response.header.get("content-type") orelse {
             // no content type in HTTP headers.
             // TODO try to sniff mime type from the body.
             log.info("no content-type HTTP header", .{});
@@ -404,14 +380,18 @@ pub const Page = struct {
 
         log.debug("header content-type: {s}", .{ct});
         var mime = try Mime.parse(arena, ct);
+        defer mime.deinit();
 
         if (mime.isHTML()) {
-            try self.loadHTMLDoc(req.reader(), mime.charset orelse "utf-8", aux_data);
+            try self.loadHTMLDoc(&response, mime.charset orelse "utf-8", aux_data);
         } else {
             log.info("non-HTML document: {s}", .{ct});
-
+            var arr: std.ArrayListUnmanaged(u8) = .{};
+            while (try response.next()) |data| {
+                try arr.appendSlice(arena, try arena.dupe(u8, data));
+            }
             // save the body into the page.
-            self.raw_data = try req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
+            self.raw_data = arr.items;
         }
     }
 
@@ -453,7 +433,7 @@ pub const Page = struct {
         // replace the user context document with the new one.
         try session.env.setUserContext(.{
             .document = html_doc,
-            .httpClient = self.session.browser.http_client,
+            .http_client = @ptrCast(self.session.http_client),
         });
 
         // browse the DOM tree to retrieve scripts
@@ -625,33 +605,33 @@ pub const Page = struct {
                 res_src = try std.fs.path.resolve(arena, &.{ _dir, src });
             }
         }
-
         const u = try std.Uri.resolve_inplace(self.uri, res_src, &b);
 
-        var fetchres = try self.session.loader.get(arena, u);
-        defer fetchres.deinit();
+        var request = try self.session.http_client.request(.GET, u);
+        defer request.deinit();
+        var response = try request.sendSync(.{});
 
-        const resp = fetchres.req.response;
+        log.info("fetch {any}: {d}", .{ u, response.header.status });
 
-        log.info("fetch {any}: {d}", .{ u, resp.status });
-
-        if (resp.status != .ok) {
+        if (response.header.status != 200) {
             return FetchError.BadStatusCode;
         }
 
+        var arr: std.ArrayListUnmanaged(u8) = .{};
+        while (try response.next()) |data| {
+            try arr.appendSlice(arena, try arena.dupe(u8, data));
+        }
+
         // TODO check content-type
-        const body = try fetchres.req.reader().readAllAlloc(arena, 16 * 1024 * 1024);
 
         // check no body
-        if (body.len == 0) {
+        if (arr.items.len == 0) {
             return FetchError.NoBody;
         }
 
-        return body;
+        return arr.items;
     }
 
-    // fetchScript senf a GET request to the src and execute the script
-    // received.
     fn fetchScript(self: *const Page, s: *const Script) !void {
         const arena = self.arena;
         const body = try self.fetchData(arena, s.src, null);

src/browser/loader.zig

@@ -1,97 +0,0 @@
-// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <francis@lightpanda.io>
-// Pierre Tachoire <pierre@lightpanda.io>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-const std = @import("std");
-const Client = @import("../http/Client.zig");
-
-const user_agent = @import("browser.zig").user_agent;
-
-pub const Loader = struct {
-    client: Client,
-    // use 64KB for headers buffer size.
-    server_header_buffer: [1024 * 64]u8 = undefined,
-
-    pub const Response = struct {
-        alloc: std.mem.Allocator,
-        req: *Client.Request,
-
-        pub fn deinit(self: *Response) void {
-            self.req.deinit();
-            self.alloc.destroy(self.req);
-        }
-    };
-
-    pub fn init(alloc: std.mem.Allocator) Loader {
-        return Loader{
-            .client = Client{
-                .allocator = alloc,
-            },
-        };
-    }
-
-    pub fn deinit(self: *Loader) void {
-        self.client.deinit();
-    }
-
-    // see
-    // https://ziglang.org/documentation/master/std/#A;std:http.Client.fetch
-    // for reference.
-    // The caller is responsible for calling `deinit()` on the `Response`.
-    pub fn get(self: *Loader, alloc: std.mem.Allocator, uri: std.Uri) !Response {
-        var resp = Response{
-            .alloc = alloc,
-            .req = try alloc.create(Client.Request),
-        };
-        errdefer alloc.destroy(resp.req);
-
-        resp.req.* = try self.client.open(.GET, uri, .{
-            .headers = .{
-                .user_agent = .{ .override = user_agent },
-            },
-            .extra_headers = &.{
-                .{ .name = "Accept", .value = "*/*" },
-                .{ .name = "Accept-Language", .value = "en-US,en;q=0.5" },
-            },
-            .server_header_buffer = &self.server_header_buffer,
-        });
-        errdefer resp.req.deinit();
-
-        try resp.req.send();
-        try resp.req.finish();
-        try resp.req.wait();
-
-        return resp;
-    }
-};
-
-test "loader: get" {
-    const alloc = std.testing.allocator;
-    var loader = Loader.init(alloc);
-    defer loader.deinit();
-
-    const uri = try std.Uri.parse("http://localhost:9582/loader");
-    var result = try loader.get(alloc, uri);
-    defer result.deinit();
-
-    try std.testing.expectEqual(.ok, result.req.response.status);
-
-    var res: [128]u8 = undefined;
-    const size = try result.req.readAll(&res);
-    try std.testing.expectEqual(6, size);
-    try std.testing.expectEqualStrings("Hello!", res[0..6]);
-}

src/cdp/testing.zig

@@ -1,3 +1,21 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 const std = @import("std");
 
 const json = std.json;

src/main_tests.zig

@@ -31,7 +31,6 @@ const storage = @import("storage/storage.zig");
 const url = @import("url/url.zig");
 const URL = url.URL;
 const urlquery = @import("url/query.zig");
-const Client = @import("asyncio").Client;
 const Location = @import("html/location.zig").Location;
 
 const documentTestExecFn = @import("dom/document.zig").testExecFn;
@@ -89,12 +88,12 @@ fn testExecFn(
         std.debug.print("documentHTMLClose error: {s}\n", .{@errorName(err)});
     };
 
-    var cli = Client{ .allocator = alloc };
-    defer cli.deinit();
+    var http_client = try @import("http/client.zig").Client.init(alloc, 5);
+    defer http_client.deinit();
 
     try js_env.setUserContext(.{
         .document = doc,
-        .httpClient = &cli,
+        .http_client = &http_client,
     });
 
     // alias global as self and window
@@ -220,6 +219,12 @@ pub fn main() !void {
     if (run == .all or run == .unit) {
         std.debug.print("\n", .{});
         for (builtin.test_functions) |test_fn| {
+            if (std.mem.startsWith(u8, test_fn.name, "http.client.test")) {
+                // covered by unit test, needs a dummy server started, which
+                // main_test doesn't do.
+                continue;
+            }
+
             try parser.init();
             defer parser.deinit();
 

src/main_unit_tests.zig

@@ -0,0 +1,225 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const builtin = @import("builtin");
+const parser = @import("netsurf");
+const tls = @import("tls");
+
+const Allocator = std.mem.Allocator;
+
+test {
+    std.testing.refAllDecls(@import("url/query.zig"));
+    std.testing.refAllDecls(@import("browser/dump.zig"));
+    std.testing.refAllDecls(@import("browser/mime.zig"));
+    std.testing.refAllDecls(@import("css/css.zig"));
+    std.testing.refAllDecls(@import("css/libdom_test.zig"));
+    std.testing.refAllDecls(@import("css/match_test.zig"));
+    std.testing.refAllDecls(@import("css/parser.zig"));
+    std.testing.refAllDecls(@import("generate.zig"));
+    std.testing.refAllDecls(@import("http/client.zig"));
+    std.testing.refAllDecls(@import("storage/storage.zig"));
+    std.testing.refAllDecls(@import("storage/cookie.zig"));
+    std.testing.refAllDecls(@import("iterator/iterator.zig"));
+    std.testing.refAllDecls(@import("server.zig"));
+    std.testing.refAllDecls(@import("cdp/cdp.zig"));
+    std.testing.refAllDecls(@import("log.zig"));
+    std.testing.refAllDecls(@import("datetime.zig"));
+    std.testing.refAllDecls(@import("telemetry/telemetry.zig"));
+    std.testing.refAllDecls(@import("http/client.zig"));
+}
+
+var wg: std.Thread.WaitGroup = .{};
+var gpa = std.heap.GeneralPurposeAllocator(.{}){};
+test "tests:beforeAll" {
+    try parser.init();
+    wg.startMany(3);
+
+    {
+        const address = try std.net.Address.parseIp("127.0.0.1", 9582);
+        const thread = try std.Thread.spawn(.{}, serveHTTP, .{address});
+        thread.detach();
+    }
+
+    {
+        const address = try std.net.Address.parseIp("127.0.0.1", 9581);
+        const thread = try std.Thread.spawn(.{}, serveHTTPS, .{address});
+        thread.detach();
+    }
+
+    {
+        const address = try std.net.Address.parseIp("127.0.0.1", 9583);
+        const thread = try std.Thread.spawn(.{}, serveCDP, .{address});
+        thread.detach();
+    }
+
+    // need to wait for the servers to be listening, else tests will fail because
+    // they aren't able to connect.
+    wg.wait();
+}
+
+test "tests:afterAll" {
+    parser.deinit();
+}
+
+fn serveHTTP(address: std.net.Address) !void {
+    const allocator = gpa.allocator();
+    var arena = std.heap.ArenaAllocator.init(allocator);
+    defer arena.deinit();
+
+    var listener = try address.listen(.{ .reuse_address = true });
+    defer listener.deinit();
+
+    wg.finish();
+
+    var read_buffer: [1024]u8 = undefined;
+    ACCEPT: while (true) {
+        defer _ = arena.reset(.{ .retain_with_limit = 1024 });
+        const aa = arena.allocator();
+
+        var conn = try listener.accept();
+        defer conn.stream.close();
+        var server = std.http.Server.init(conn, &read_buffer);
+
+        while (server.state == .ready) {
+            var request = server.receiveHead() catch |err| switch (err) {
+                error.HttpConnectionClosing => continue :ACCEPT,
+                else => {
+                    std.debug.print("Test HTTP Server error: {}\n", .{err});
+                    return err;
+                },
+            };
+
+            const path = request.head.target;
+            if (std.mem.eql(u8, path, "/loader")) {
+                try request.respond("Hello!", .{});
+            } else if (std.mem.eql(u8, path, "/http_client/simple")) {
+                try request.respond("", .{});
+            } else if (std.mem.eql(u8, path, "/http_client/redirect")) {
+                try request.respond("", .{
+                    .status = .moved_permanently,
+                    .extra_headers = &.{.{ .name = "LOCATION", .value = "../http_client/echo" }},
+                });
+            } else if (std.mem.eql(u8, path, "/http_client/redirect/secure")) {
+                try request.respond("", .{
+                    .status = .moved_permanently,
+                    .extra_headers = &.{.{ .name = "LOCATION", .value = "https://127.0.0.1:9581/http_client/body" }},
+                });
+            } else if (std.mem.eql(u8, path, "/http_client/echo")) {
+                var headers: std.ArrayListUnmanaged(std.http.Header) = .{};
+
+                var it = request.iterateHeaders();
+                while (it.next()) |hdr| {
+                    try headers.append(aa, .{
+                        .name = try std.fmt.allocPrint(aa, "_{s}", .{hdr.name}),
+                        .value = hdr.value,
+                    });
+                }
+
+                try request.respond("over 9000!", .{
+                    .status = .created,
+                    .extra_headers = headers.items,
+                });
+            }
+        }
+    }
+}
+
+// This is a lot of work for testing TLS, but the TLS (async) code is complicated
+// This "server" is written specifically to test the client. It assumes the client
+// isn't a jerk.
+fn serveHTTPS(address: std.net.Address) !void {
+    const allocator = gpa.allocator();
+
+    var listener = try address.listen(.{ .reuse_address = true });
+    defer listener.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(allocator);
+    defer arena.deinit();
+
+    wg.finish();
+
+    var seed: u64 = undefined;
+    std.posix.getrandom(std.mem.asBytes(&seed)) catch unreachable;
+    var r = std.Random.DefaultPrng.init(seed);
+    const rand = r.random();
+
+    var read_buffer: [1024]u8 = undefined;
+    while (true) {
+        // defer _ = arena.reset(.{ .retain_with_limit = 1024 });
+        // const aa = arena.allocator();
+
+        const stream = blk: {
+            const conn = try listener.accept();
+            break :blk conn.stream;
+        };
+        defer stream.close();
+
+        var conn = try tls.server(stream, .{ .auth = null });
+        defer conn.close() catch {};
+
+        var pos: usize = 0;
+        while (true) {
+            const n = try conn.read(read_buffer[pos..]);
+            if (n == 0) {
+                break;
+            }
+            pos += n;
+            const header_end = std.mem.indexOf(u8, read_buffer[0..pos], "\r\n\r\n") orelse {
+                continue;
+            };
+            var it = std.mem.splitScalar(u8, read_buffer[0..header_end], ' ');
+            _ = it.next() orelse unreachable; // method
+            const path = it.next() orelse unreachable;
+
+            var response: []const u8 = undefined;
+            if (std.mem.eql(u8, path, "/http_client/simple")) {
+                response = "HTTP/1.1 200 \r\nContent-Length: 0\r\n\r\n";
+            } else if (std.mem.eql(u8, path, "/http_client/body")) {
+                response = "HTTP/1.1 201 CREATED\r\nContent-Length: 20\r\n   Another :  HEaDer  \r\n\r\n1234567890abcdefhijk";
+            } else if (std.mem.eql(u8, path, "/http_client/redirect/insecure")) {
+                response = "HTTP/1.1 307 GOTO\r\nLocation: http://127.0.0.1:9582/http_client/redirect\r\n\r\n";
+            } else {
+                // should not have an unknown path
+                unreachable;
+            }
+
+            var unsent = response;
+            while (unsent.len > 0) {
+                const to_send = rand.intRangeAtMost(usize, 1, unsent.len);
+                const sent = try conn.write(unsent[0..to_send]);
+                unsent = unsent[sent..];
+                std.time.sleep(std.time.ns_per_us * 5);
+            }
+            break;
+        }
+    }
+}
+
+fn serveCDP(address: std.net.Address) !void {
+    const App = @import("app.zig").App;
+    var app = try App.init(gpa.allocator(), .serve);
+    defer app.deinit();
+
+    const server = @import("server.zig");
+    wg.finish();
+    server.run(app, address, std.time.ns_per_s * 2) catch |err| {
+        std.debug.print("CDP server error: {}", .{err});
+        return err;
+    };
+}

src/netsurf/netsurf.zig

@@ -2181,21 +2181,28 @@ fn parseParams(enc: ?[:0]const u8) c.dom_hubbub_parser_params {
 
 fn parseData(parser: *c.dom_hubbub_parser, reader: anytype) !void {
     var err: c.hubbub_error = undefined;
-    var buffer: [1024]u8 = undefined;
-    var ln = buffer.len;
-    while (ln > 0) {
-        ln = try reader.read(&buffer);
-        err = c.dom_hubbub_parser_parse_chunk(parser, &buffer, ln);
-        // TODO handle encoding change error return.
-        // When the HTML contains a META tag with a different encoding than the
-        // original one, a c.DOM_HUBBUB_HUBBUB_ERR_ENCODINGCHANGE error is
-        // returned.
-        // In this case, we must restart the parsing with the new detected
-        // encoding. The detected encoding is stored in the document and we can
-        // get it with documentGetInputEncoding().
-        try parserErr(err);
+    const TI = @typeInfo(@TypeOf(reader));
+    if (TI == .pointer and @hasDecl(TI.pointer.child, "next")) {
+        while (try reader.next()) |data| {
+            err = c.dom_hubbub_parser_parse_chunk(parser, data.ptr, data.len);
+            try parserErr(err);
+        }
+    } else {
+        var buffer: [1024]u8 = undefined;
+        var ln = buffer.len;
+        while (ln > 0) {
+            ln = try reader.read(&buffer);
+            err = c.dom_hubbub_parser_parse_chunk(parser, &buffer, ln);
+            // TODO handle encoding change error return.
+            // When the HTML contains a META tag with a different encoding than the
+            // original one, a c.DOM_HUBBUB_HUBBUB_ERR_ENCODINGCHANGE error is
+            // returned.
+            // In this case, we must restart the parsing with the new detected
+            // encoding. The detected encoding is stored in the document and we can
+            // get it with documentGetInputEncoding().
+            try parserErr(err);
+        }
     }
-
     err = c.dom_hubbub_parser_completed(parser);
     try parserErr(err);
 }

src/telemetry/lightpanda.zig

@@ -7,6 +7,7 @@ const Allocator = std.mem.Allocator;
 
 const App = @import("../app.zig").App;
 const telemetry = @import("telemetry.zig");
+const HttpClient = @import("../http/client.zig").Client;
 
 const log = std.log.scoped(.telemetry);
 const URL = "https://telemetry.lightpanda.io";
@@ -20,7 +21,7 @@ pub const LightPanda = struct {
     allocator: Allocator,
     mutex: std.Thread.Mutex,
     cond: Thread.Condition,
-    client: *std.http.Client,
+    client: *HttpClient,
     node_pool: std.heap.MemoryPool(List.Node),
 
     const List = std.DoublyLinkedList(LightPandaEvent);
@@ -34,7 +35,7 @@ pub const LightPanda = struct {
             .thread = null,
             .running = true,
             .allocator = allocator,
-            .client = @ptrCast(&app.http_client),
+            .client = &app.http_client,
             .uri = std.Uri.parse(URL) catch unreachable,
             .node_pool = std.heap.MemoryPool(List.Node).init(allocator),
         };
@@ -72,7 +73,6 @@ pub const LightPanda = struct {
     }
 
     fn run(self: *LightPanda) void {
-        const client = self.client;
         var arr: std.ArrayListUnmanaged(u8) = .{};
         defer arr.deinit(self.allocator);
 
@@ -82,7 +82,7 @@ pub const LightPanda = struct {
             while (self.pending.first != null) {
                 const b = self.collectBatch(&batch);
                 self.mutex.unlock();
-                self.postEvent(b, client, &arr) catch |err| {
+                self.postEvent(b, &arr) catch |err| {
                     log.warn("Telementry reporting error: {}", .{err});
                 };
                 self.mutex.lock();
@@ -94,7 +94,7 @@ pub const LightPanda = struct {
         }
     }
 
-    fn postEvent(self: *const LightPanda, events: []LightPandaEvent, client: *std.http.Client, arr: *std.ArrayListUnmanaged(u8)) !void {
+    fn postEvent(self: *const LightPanda, events: []LightPandaEvent, arr: *std.ArrayListUnmanaged(u8)) !void {
         defer arr.clearRetainingCapacity();
         var writer = arr.writer(self.allocator);
         for (events) |event| {
@@ -102,16 +102,15 @@ pub const LightPanda = struct {
             try writer.writeByte('\n');
         }
 
-        var response_header_buffer: [2048]u8 = undefined;
-        const result = try client.fetch(.{
-            .method = .POST,
-            .payload = arr.items,
-            .response_storage = .ignore,
-            .location = .{ .uri = self.uri },
-            .server_header_buffer = &response_header_buffer,
-        });
-        if (result.status != .ok) {
-            log.warn("server error status: {}", .{result.status});
+        var req = try self.client.request(.POST, self.uri);
+        defer req.deinit();
+        req.body = arr.items;
+
+        // drain the response
+        var res = try req.sendSync(.{});
+        while (try res.next()) |_| {}
+        if (res.header.status != 200) {
+            log.warn("server error status: {d}", .{res.header.status});
         }
     }
 

src/test_runner.zig

@@ -18,34 +18,31 @@
 
 const std = @import("std");
 const builtin = @import("builtin");
-const parser = @import("netsurf");
 
 const Allocator = std.mem.Allocator;
 
-const App = @import("app.zig").App;
-const jsruntime = @import("jsruntime");
-pub const Types = jsruntime.reflect(@import("generate.zig").Tuple(.{}){});
-pub const UserContext = @import("user_context.zig").UserContext;
-
-pub const std_options = std.Options{
-    .log_level = .err,
-    .http_disable_tls = true,
-};
-
 const BORDER = "=" ** 80;
 
 // use in custom panic handler
 var current_test: ?[]const u8 = null;
 
+const jsruntime = @import("jsruntime");
+pub const Types = jsruntime.reflect(@import("generate.zig").Tuple(.{}){});
+pub const UserContext = @import("user_context.zig").UserContext;
+
+pub const std_options = std.Options{
+    .log_level = .warn,
+
+    // Crypto in Zig is generally slow, but it's particularly slow in debug mode
+    // this helps a lot (but it's still slow). Not safe to do this in non-test!
+    .side_channels_mitigations = .none,
+};
+
 pub fn main() !void {
-    try parser.init();
-    defer parser.deinit();
+    var mem: [8192]u8 = undefined;
+    var fba = std.heap.FixedBufferAllocator.init(&mem);
 
-    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
-    const allocator = gpa.allocator();
-
-    var app = try App.init(allocator, .serve);
-    defer app.deinit();
+    const allocator = fba.allocator();
 
     const env = Env.init(allocator);
     defer env.deinit(allocator);
@@ -58,30 +55,20 @@ pub fn main() !void {
     var skip: usize = 0;
     var leak: usize = 0;
 
-    const http_thread = blk: {
-        const address = try std.net.Address.parseIp("127.0.0.1", 9582);
-        const thread = try std.Thread.spawn(.{}, serveHTTP, .{address});
-        break :blk thread;
-    };
-    defer http_thread.join();
-
-    const cdp_thread = blk: {
-        const address = try std.net.Address.parseIp("127.0.0.1", 9583);
-        const thread = try std.Thread.spawn(.{}, serveCDP, .{
-            app,
-            address,
-        });
-        break :blk thread;
-    };
-    defer cdp_thread.join();
-
     const printer = Printer.init();
     printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line
 
     for (builtin.test_functions) |t| {
-        if (std.mem.eql(u8, t.name, "unit_tests.test_0")) {
-            // don't display anything for this test
-            try t.func();
+        if (isSetup(t)) {
+            t.func() catch |err| {
+                printer.status(.fail, "\nsetup \"{s}\" failed: {}\n", .{ t.name, err });
+                return err;
+            };
+        }
+    }
+
+    for (builtin.test_functions) |t| {
+        if (isSetup(t) or isTeardown(t)) {
             continue;
         }
 
@@ -151,6 +138,15 @@ pub fn main() !void {
         }
     }
 
+    for (builtin.test_functions) |t| {
+        if (isTeardown(t)) {
+            t.func() catch |err| {
+                printer.status(.fail, "\nteardown \"{s}\" failed: {}\n", .{ t.name, err });
+                return err;
+            };
+        }
+    }
+
     const total_tests = pass + fail;
     const status = if (fail == 0) Status.pass else Status.fail;
     printer.status(status, "\n{d} of {d} test{s} passed\n", .{ pass, total_tests, if (total_tests != 1) "s" else "" });
@@ -315,6 +311,15 @@ const Env = struct {
     }
 };
 
+pub const panic = std.debug.FullPanic(struct {
+    pub fn panicFn(msg: []const u8, first_trace_addr: ?usize) noreturn {
+        if (current_test) |ct| {
+            std.debug.print("\x1b[31m{s}\npanic running \"{s}\"\n{s}\x1b[0m\n", .{ BORDER, ct, BORDER });
+        }
+        std.debug.defaultPanic(msg, first_trace_addr);
+    }
+}.panicFn);
+
 fn isUnnamed(t: std.builtin.TestFn) bool {
     const marker = ".test_";
     const test_name = t.name;
@@ -323,69 +328,10 @@ fn isUnnamed(t: std.builtin.TestFn) bool {
     return true;
 }
 
-fn serveHTTP(address: std.net.Address) !void {
-    var listener = try address.listen(.{ .reuse_address = true });
-    defer listener.deinit();
-
-    var read_buffer: [1024]u8 = undefined;
-    ACCEPT: while (true) {
-        var conn = try listener.accept();
-        defer conn.stream.close();
-        var server = std.http.Server.init(conn, &read_buffer);
-
-        while (server.state == .ready) {
-            var request = server.receiveHead() catch |err| switch (err) {
-                error.HttpConnectionClosing => continue :ACCEPT,
-                else => {
-                    std.debug.print("Test HTTP Server error: {}\n", .{err});
-                    return err;
-                },
-            };
-
-            const path = request.head.target;
-            if (std.mem.eql(u8, path, "/loader")) {
-                try writeResponse(&request, .{
-                    .body = "Hello!",
-                });
-            }
-        }
-    }
+fn isSetup(t: std.builtin.TestFn) bool {
+    return std.mem.endsWith(u8, t.name, "tests:beforeAll");
 }
 
-fn serveCDP(app: *App, address: std.net.Address) !void {
-    const server = @import("server.zig");
-    server.run(app, address, std.time.ns_per_s * 2) catch |err| {
-        std.debug.print("CDP server error: {}", .{err});
-        return err;
-    };
-}
-
-const Response = struct {
-    body: []const u8 = "",
-    status: std.http.Status = .ok,
-};
-
-fn writeResponse(req: *std.http.Server.Request, res: Response) !void {
-    try req.respond(res.body, .{ .status = res.status });
-}
-
-test {
-    std.testing.refAllDecls(@import("url/query.zig"));
-    std.testing.refAllDecls(@import("browser/dump.zig"));
-    std.testing.refAllDecls(@import("browser/loader.zig"));
-    std.testing.refAllDecls(@import("browser/mime.zig"));
-    std.testing.refAllDecls(@import("css/css.zig"));
-    std.testing.refAllDecls(@import("css/libdom_test.zig"));
-    std.testing.refAllDecls(@import("css/match_test.zig"));
-    std.testing.refAllDecls(@import("css/parser.zig"));
-    std.testing.refAllDecls(@import("generate.zig"));
-    std.testing.refAllDecls(@import("http/Client.zig"));
-    std.testing.refAllDecls(@import("storage/storage.zig"));
-    std.testing.refAllDecls(@import("storage/cookie.zig"));
-    std.testing.refAllDecls(@import("iterator/iterator.zig"));
-    std.testing.refAllDecls(@import("server.zig"));
-    std.testing.refAllDecls(@import("cdp/cdp.zig"));
-    std.testing.refAllDecls(@import("log.zig"));
-    std.testing.refAllDecls(@import("datetime.zig"));
-    std.testing.refAllDecls(@import("telemetry/telemetry.zig"));
+fn isTeardown(t: std.builtin.TestFn) bool {
+    return std.mem.endsWith(u8, t.name, "tests:afterAll");
 }

src/testing.zig

@@ -1,3 +1,21 @@
+// Copyright (C) 2023-2024  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 const std = @import("std");
 
 pub const allocator = std.testing.allocator;
@@ -30,8 +48,8 @@ pub fn expectEqual(expected: anytype, actual: anytype) !void {
             return;
         },
         .optional => {
-            if (actual == null) {
-                return std.testing.expectEqual(null, expected);
+            if (@typeInfo(@TypeOf(expected)) == .null) {
+                return std.testing.expectEqual(null, actual);
             }
             return expectEqual(expected, actual.?);
         },
@@ -141,3 +159,34 @@ pub fn print(comptime fmt: []const u8, args: anytype) void {
 pub fn app(_: anytype) *App {
     return App.init(allocator, .serve) catch unreachable;
 }
+
+pub const Random = struct {
+    var instance: ?std.Random.DefaultPrng = null;
+
+    pub fn fill(buf: []u8) void {
+        var r = random();
+        r.bytes(buf);
+    }
+
+    pub fn fillAtLeast(buf: []u8, min: usize) []u8 {
+        var r = random();
+        const l = r.intRangeAtMost(usize, min, buf.len);
+        r.bytes(buf[0..l]);
+        return buf;
+    }
+
+    pub fn intRange(comptime T: type, min: T, max: T) T {
+        var r = random();
+        return r.intRangeAtMost(T, min, max);
+    }
+
+    pub fn random() std.Random {
+        if (instance == null) {
+            var seed: u64 = undefined;
+            std.posix.getrandom(std.mem.asBytes(&seed)) catch unreachable;
+            instance = std.Random.DefaultPrng.init(seed);
+            // instance = std.Random.DefaultPrng.init(0);
+        }
+        return instance.?.random();
+    }
+};

src/user_context.zig

@@ -1,8 +1,8 @@
 const std = @import("std");
 const parser = @import("netsurf");
-const Client = @import("asyncio").Client;
+const Client = @import("http/client.zig").Client;
 
 pub const UserContext = struct {
     document: *parser.DocumentHTML,
-    httpClient: *Client,
+    http_client: *Client,
 };

src/wpt/run.zig

@@ -28,7 +28,7 @@ const Loop = jsruntime.Loop;
 const Env = jsruntime.Env;
 const Window = @import("../html/window.zig").Window;
 const storage = @import("../storage/storage.zig");
-const Client = @import("asyncio").Client;
+const HttpClient = @import("../http/client.zig").Client;
 
 const Types = @import("../main_wpt.zig").Types;
 const UserContext = @import("../main_wpt.zig").UserContext;
@@ -55,13 +55,13 @@ pub fn run(arena: *std.heap.ArenaAllocator, comptime dir: []const u8, f: []const
     var loop = try Loop.init(alloc);
     defer loop.deinit();
 
-    var cli = Client{ .allocator = alloc };
-    defer cli.deinit();
+    var http_client = try HttpClient.init(alloc, 2);
+    defer http_client.deinit();
 
     var js_env: Env = undefined;
     Env.init(&js_env, alloc, &loop, UserContext{
         .document = html_doc,
-        .httpClient = &cli,
+        .http_client = &http_client,
     });
     defer js_env.deinit();
 

src/xhr/xhr.zig

@@ -31,7 +31,7 @@ const XMLHttpRequestEventTarget = @import("event_target.zig").XMLHttpRequestEven
 const Mime = @import("../browser/mime.zig").Mime;
 
 const Loop = jsruntime.Loop;
-const Client = @import("asyncio").Client;
+const http = @import("../http/client.zig");
 
 const parser = @import("netsurf");
 
@@ -95,14 +95,12 @@ pub const XMLHttpRequestBodyInit = union(XMLHttpRequestBodyInitTag) {
 pub const XMLHttpRequest = struct {
     proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{},
     alloc: std.mem.Allocator,
-    cli: *Client,
-    io: Client.IO,
+    client: *http.Client,
+    request: ?http.Request = null,
 
     priv_state: PrivState = .new,
-    req: ?Client.Request = null,
-    ctx: ?Client.Ctx = null,
 
-    method: std.http.Method,
+    method: http.Request.Method,
     state: State,
     url: ?[]const u8,
     uri: std.Uri,
@@ -110,6 +108,7 @@ pub const XMLHttpRequest = struct {
     headers: Headers,
     sync: bool = true,
     err: ?anyerror = null,
+    last_dispatch: i64 = 0,
 
     // TODO uncomment this field causes casting issue with
     // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
@@ -125,7 +124,7 @@ pub const XMLHttpRequest = struct {
 
     withCredentials: bool = false,
     // TODO: response readonly attribute any response;
-    response_bytes: ?[]const u8 = null,
+    response_bytes: std.ArrayListUnmanaged(u8) = .{},
     response_type: ResponseType = .Empty,
     response_headers: Headers,
 
@@ -133,7 +132,7 @@ pub const XMLHttpRequest = struct {
     // use 16KB for headers buffer size.
     response_header_buffer: [1024 * 16]u8 = undefined,
 
-    response_status: u10 = 0,
+    response_status: u16 = 0,
 
     // TODO uncomment this field causes casting issue with
     // XMLHttpRequestEventTarget. I think it's dueto an alignement issue, but
@@ -145,8 +144,6 @@ pub const XMLHttpRequest = struct {
     response_obj: ?ResponseObj = null,
     send_flag: bool = false,
 
-    payload: ?[]const u8 = null,
-
     pub const prototype = *XMLHttpRequestEventTarget;
     pub const mem_guarantied = true;
 
@@ -246,13 +243,6 @@ pub const XMLHttpRequest = struct {
         fn all(self: Headers) []std.http.Header {
             return self.list.items;
         }
-
-        fn load(self: *Headers, it: *std.http.HeaderIterator) !void {
-            while (true) {
-                const h = it.next() orelse break;
-                _ = try self.append(h.name, h.value);
-            }
-        }
     };
 
     const Response = union(ResponseType) {
@@ -290,17 +280,16 @@ pub const XMLHttpRequest = struct {
 
     const min_delay: u64 = 50000000; // 50ms
 
-    pub fn constructor(alloc: std.mem.Allocator, loop: *Loop, userctx: UserContext) !XMLHttpRequest {
+    pub fn constructor(alloc: std.mem.Allocator, userctx: UserContext) !XMLHttpRequest {
         return .{
             .alloc = alloc,
             .headers = Headers.init(alloc),
             .response_headers = Headers.init(alloc),
-            .io = Client.IO.init(loop),
             .method = undefined,
             .url = null,
             .uri = undefined,
             .state = .unsent,
-            .cli = userctx.httpClient,
+            .client = userctx.http_client,
         };
     }
 
@@ -308,10 +297,6 @@ pub const XMLHttpRequest = struct {
         if (self.url) |v| alloc.free(v);
         self.url = null;
 
-        if (self.payload) |v| alloc.free(v);
-        self.payload = null;
-
-        if (self.response_bytes) |v| alloc.free(v);
         if (self.response_obj) |v| v.deinit();
 
         self.response_obj = null;
@@ -329,12 +314,6 @@ pub const XMLHttpRequest = struct {
         self.send_flag = false;
 
         self.priv_state = .new;
-
-        if (self.ctx) |*c| c.deinit();
-        self.ctx = null;
-
-        if (self.req) |*r| r.deinit();
-        self.req = null;
     }
 
     pub fn deinit(self: *XMLHttpRequest, alloc: std.mem.Allocator) void {
@@ -449,7 +428,7 @@ pub const XMLHttpRequest = struct {
     }
 
     const methods = [_]struct {
-        tag: std.http.Method,
+        tag: http.Request.Method,
         name: []const u8,
     }{
         .{ .tag = .DELETE, .name = "DELETE" },
@@ -461,7 +440,7 @@ pub const XMLHttpRequest = struct {
     };
     const methods_forbidden = [_][]const u8{ "CONNECT", "TRACE", "TRACK" };
 
-    pub fn validMethod(m: []const u8) DOMError!std.http.Method {
+    pub fn validMethod(m: []const u8) DOMError!http.Request.Method {
         for (methods) |method| {
             if (std.ascii.eqlIgnoreCase(method.name, m)) {
                 return method.tag;
@@ -485,183 +464,97 @@ pub const XMLHttpRequest = struct {
     }
 
     // TODO body can be either a XMLHttpRequestBodyInit or a document
-    pub fn _send(self: *XMLHttpRequest, alloc: std.mem.Allocator, body: ?[]const u8) !void {
+    pub fn _send(self: *XMLHttpRequest, loop: *Loop, alloc: std.mem.Allocator, body: ?[]const u8) !void {
         if (self.state != .opened) return DOMError.InvalidState;
         if (self.send_flag) return DOMError.InvalidState;
 
+        log.debug("{any} {any}", .{ self.method, self.uri });
+
+        self.send_flag = true;
+        self.priv_state = .open;
+
+        self.request = try self.client.request(self.method, self.uri);
+        var request = &self.request.?;
+        errdefer request.deinit();
+
+        for (self.headers.list.items) |hdr| {
+            try request.addHeader(hdr.name, hdr.value, .{});
+        }
+
         //  The body argument provides the request body, if any, and is ignored
         //  if the request method is GET or HEAD.
         //  https://xhr.spec.whatwg.org/#the-send()-method
         // var used_body: ?XMLHttpRequestBodyInit = null;
-        if (body != null and self.method != .GET and self.method != .HEAD) {
-            // TODO If body is a Document, then set this’s request body to body, serialized, converted, and UTF-8 encoded.
-
-            const body_init = XMLHttpRequestBodyInit{ .String = body.? };
-
-            // keep the user content type from request headers.
-            if (self.headers.has("Content-Type")) {
-                // https://fetch.spec.whatwg.org/#bodyinit-safely-extract
-                try self.headers.append("Content-Type", try body_init.contentType());
+        if (body) |b| {
+            if (self.method != .GET and self.method != .HEAD) {
+                request.body = try alloc.dupe(u8, b);
+                try request.addHeader("Content-Type", "text/plain; charset=UTF-8", .{});
             }
-
-            // copy the payload
-            if (self.payload) |v| alloc.free(v);
-            self.payload = try body_init.dupe(alloc);
         }
 
-        log.debug("{any} {any}", .{ self.method, self.uri });
-
-        self.send_flag = true;
-
-        self.priv_state = .open;
-
-        self.req = try self.cli.create(self.method, self.uri, .{
-            .server_header_buffer = &self.response_header_buffer,
-            .extra_headers = self.headers.all(),
-        });
-        errdefer {
-            self.req.?.deinit();
-            self.req = null;
-        }
-
-        self.ctx = try Client.Ctx.init(&self.io, &self.req.?);
-        errdefer {
-            self.ctx.?.deinit();
-            self.ctx = null;
-        }
-        self.ctx.?.userData = self;
-
-        try self.cli.async_open(
-            self.method,
-            self.uri,
-            .{ .server_header_buffer = &self.response_header_buffer },
-            &self.ctx.?,
-            onRequestConnect,
-        );
+        try request.sendAsync(loop, self, .{});
     }
 
-    fn onRequestWait(ctx: *Client.Ctx, res: anyerror!void) !void {
-        var self = selfCtx(ctx);
-        res catch |err| return self.onErr(err);
+    pub fn onHttpResponse(self: *XMLHttpRequest, progress_: anyerror!http.Progress) !void {
+        const progress = progress_ catch |err| {
+            self.onErr(err);
+            return err;
+        };
 
-        log.info("{any} {any} {d}", .{ self.method, self.uri, self.req.?.response.status });
+        if (progress.first) {
+            const header = progress.header;
+            log.info("{any} {any} {d}", .{ self.method, self.uri, header.status });
 
-        self.priv_state = .done;
-        var it = self.req.?.response.iterateHeaders();
-        self.response_headers.load(&it) catch |e| return self.onErr(e);
+            self.priv_state = .done;
 
-        // extract a mime type from headers.
-        const ct = self.response_headers.getFirstValue("Content-Type") orelse "text/xml";
-        self.response_mime = Mime.parse(self.alloc, ct) catch |e| return self.onErr(e);
+            for (header.headers.items) |hdr| {
+                try self.response_headers.append(hdr.name, hdr.value);
+            }
 
-        // TODO handle override mime type
+            // extract a mime type from headers.
+            const ct = header.get("content-type") orelse "text/xml";
+            self.response_mime = Mime.parse(self.alloc, ct) catch |e| return self.onErr(e);
 
-        self.state = .headers_received;
-        self.dispatchEvt("readystatechange");
+            // TODO handle override mime type
+            self.state = .headers_received;
+            self.dispatchEvt("readystatechange");
 
-        self.response_status = @intFromEnum(self.req.?.response.status);
+            self.response_status = header.status;
 
-        var buf: std.ArrayListUnmanaged(u8) = .{};
-
-        // TODO set correct length
-        const total = 0;
-        var loaded: u64 = 0;
-
-        // dispatch a progress event loadstart.
-        self.dispatchProgressEvent("loadstart", .{ .loaded = loaded, .total = total });
-
-        // TODO read async
-        const reader = self.req.?.reader();
-        var buffer: [1024]u8 = undefined;
-        var ln = buffer.len;
-        var prev_dispatch: ?std.time.Instant = null;
-        while (ln > 0) {
-            ln = reader.read(&buffer) catch |e| {
-                buf.deinit(self.alloc);
-                return self.onErr(e);
-            };
-            buf.appendSlice(self.alloc, buffer[0..ln]) catch |e| {
-                buf.deinit(self.alloc);
-                return self.onErr(e);
-            };
-            loaded = loaded + ln;
-
-            // Dispatch only if 50ms have passed.
-            const now = std.time.Instant.now() catch |e| {
-                buf.deinit(self.alloc);
-                return self.onErr(e);
-            };
-            if (prev_dispatch != null and now.since(prev_dispatch.?) < min_delay) continue;
-            defer prev_dispatch = now;
+            // TODO correct total
+            self.dispatchProgressEvent("loadstart", .{ .loaded = 0, .total = 0 });
 
             self.state = .loading;
             self.dispatchEvt("readystatechange");
-
-            // dispatch a progress event progress.
-            self.dispatchProgressEvent("progress", .{
-                .loaded = loaded,
-                .total = total,
-            });
         }
-        self.response_bytes = buf.items;
-        self.send_flag = false;
+
+        if (progress.data) |data| {
+            try self.response_bytes.appendSlice(self.alloc, data);
+        }
+
+        const loaded = self.response_bytes.items.len;
+        const now = std.time.milliTimestamp();
+        if (now - self.last_dispatch > 50) {
+            // don't send this more than once every 50ms
+            self.dispatchProgressEvent("progress", .{
+                .total = loaded,
+                .loaded = loaded,
+            });
+            self.last_dispatch = now;
+        }
+
+        if (progress.done == false) {
+            return;
+        }
 
         self.state = .done;
+        self.send_flag = false;
         self.dispatchEvt("readystatechange");
 
         // dispatch a progress event load.
-        self.dispatchProgressEvent("load", .{ .loaded = loaded, .total = total });
+        self.dispatchProgressEvent("load", .{ .loaded = loaded, .total = loaded });
         // dispatch a progress event loadend.
-        self.dispatchProgressEvent("loadend", .{ .loaded = loaded, .total = total });
-
-        if (self.ctx) |*c| c.deinit();
-        self.ctx = null;
-
-        if (self.req) |*r| r.deinit();
-        self.req = null;
-    }
-
-    fn onRequestFinish(ctx: *Client.Ctx, res: anyerror!void) !void {
-        var self = selfCtx(ctx);
-        res catch |err| return self.onErr(err);
-
-        self.priv_state = .wait;
-        return ctx.req.async_wait(ctx, onRequestWait) catch |e| return self.onErr(e);
-    }
-
-    fn onRequestSend(ctx: *Client.Ctx, res: anyerror!void) !void {
-        var self = selfCtx(ctx);
-        res catch |err| return self.onErr(err);
-
-        if (self.payload) |payload| {
-            self.priv_state = .write;
-            return ctx.req.async_writeAll(payload, ctx, onRequestWrite) catch |e| return self.onErr(e);
-        }
-
-        self.priv_state = .finish;
-        return ctx.req.async_finish(ctx, onRequestFinish) catch |e| return self.onErr(e);
-    }
-
-    fn onRequestWrite(ctx: *Client.Ctx, res: anyerror!void) !void {
-        var self = selfCtx(ctx);
-        res catch |err| return self.onErr(err);
-        self.priv_state = .finish;
-        return ctx.req.async_finish(ctx, onRequestFinish) catch |e| return self.onErr(e);
-    }
-
-    fn onRequestConnect(ctx: *Client.Ctx, res: anyerror!void) anyerror!void {
-        var self = selfCtx(ctx);
-        res catch |err| return self.onErr(err);
-
-        // prepare payload transfert.
-        if (self.payload) |v| self.req.?.transfer_encoding = .{ .content_length = v.len };
-
-        self.priv_state = .send;
-        return ctx.req.async_send(ctx, onRequestSend) catch |err| return self.onErr(err);
-    }
-
-    fn selfCtx(ctx: *Client.Ctx) *XMLHttpRequest {
-        return @ptrCast(@alignCast(ctx.userData));
+        self.dispatchProgressEvent("loadend", .{ .loaded = loaded, .total = loaded });
     }
 
     fn onErr(self: *XMLHttpRequest, err: anyerror) void {
@@ -675,12 +568,6 @@ pub const XMLHttpRequest = struct {
         self.dispatchProgressEvent("loadend", .{});
 
         log.debug("{any} {any} {any}", .{ self.method, self.uri, self.err });
-
-        if (self.ctx) |*c| c.deinit();
-        self.ctx = null;
-
-        if (self.req) |*r| r.deinit();
-        self.req = null;
     }
 
     pub fn _abort(self: *XMLHttpRequest) void {
@@ -803,7 +690,7 @@ pub const XMLHttpRequest = struct {
         }
 
         if (self.response_type == .JSON) {
-            if (self.response_bytes == null) return null;
+            if (self.response_bytes.items.len == 0) return null;
 
             // TODO Let jsonObject be the result of running parse JSON from bytes
             // on this’s received bytes. If that threw an exception, then return
@@ -841,7 +728,7 @@ pub const XMLHttpRequest = struct {
         };
         defer alloc.free(ccharset);
 
-        var fbs = std.io.fixedBufferStream(self.response_bytes.?);
+        var fbs = std.io.fixedBufferStream(self.response_bytes.items);
         const doc = parser.documentHTMLParse(fbs.reader(), ccharset) catch {
             self.response_obj = .{ .Failure = true };
             return;
@@ -862,7 +749,7 @@ pub const XMLHttpRequest = struct {
         const p = std.json.parseFromSlice(
             JSONValue,
             alloc,
-            self.response_bytes.?,
+            self.response_bytes.items,
             .{},
         ) catch |e| {
             log.err("parse JSON: {}", .{e});
@@ -875,8 +762,7 @@ pub const XMLHttpRequest = struct {
 
     pub fn get_responseText(self: *XMLHttpRequest) ![]const u8 {
         if (self.response_type != .Empty and self.response_type != .Text) return DOMError.InvalidState;
-
-        return if (self.response_bytes) |v| v else "";
+        return self.response_bytes.items;
     }
 
     pub fn _getResponseHeader(self: *XMLHttpRequest, name: []const u8) ?[]const u8 {