add set_pathname on URL

ws: increase max http message from 2kb to 4kb

.github/workflows/build.yml

@@ -98,7 +98,7 @@ jobs:
       ARCH: aarch64
       OS: macos
 
-    runs-on: macos-latest
+    runs-on: macos-14
     timeout-minutes: 15
 
     steps:
@@ -136,7 +136,7 @@ jobs:
       ARCH: x86_64
       OS: macos
 
-    runs-on: macos-13
+    runs-on: macos-14
     timeout-minutes: 15
 
     steps:

.github/workflows/e2e-test.yml

@@ -45,6 +45,9 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
+    # Don't run the CI with draft PR.
+    if: github.event.pull_request.draft == false
+
     steps:
       - uses: actions/checkout@v4
         with:

README.md

@@ -18,7 +18,7 @@ Lightpanda is the open-source browser made for headless usage:
 
 - Javascript execution
 - Support of Web APIs (partial, WIP)
-- Compatible with Playwright[^1], Puppeteer through CDP (WIP)
+- Compatible with Playwright[^1], Puppeteer, chromedp through CDP
 
 Fast web automation for AI agents, LLM training, scraping and testing:
 
@@ -41,7 +41,8 @@ Due to the nature of Playwright, a script that works with the current version of
 
 ## Quick start
 
-### Install from the nightly builds
+### Install 
+**Install from the nightly builds**
 
 You can download the last binary from the [nightly
 builds](https://github.com/lightpanda-io/browser/releases/tag/nightly) for
@@ -64,6 +65,17 @@ chmod a+x ./lightpanda
 The Lightpanda browser is compatible to run on windows inside WSL. Follow the Linux instruction for installation from a WSL terminal.
 It is recommended to install clients like Puppeteer on the Windows host.
 
+**Install from Docker**
+
+Lightpanda provides [official Docker
+images](https://hub.docker.com/r/lightpanda/browser) for both Linux amd64 and
+arm64 architectures.
+The following command fetches the Docker image and starts a new container exposing Lightpanda's CDP server on port `9222`.
+The `--privileged` option is required because the browser requires `io_uring` syscalls which are blocked by default by Docker.
+```console
+docker run -d --name lightpanda -p 9222:9222 --privileged lightpanda/browser:nightly
+```
+
 ### Dump a URL
 
 ```console
@@ -124,21 +136,27 @@ By default, Lightpanda collects and sends usage telemetry. This can be disabled
 
 ## Status
 
-Lightpanda is still a work in progress and is currently at a Beta stage.
-
-:warning: You should expect most websites to fail or crash.
+Lightpanda is in Beta and currently a work in progress. Stability and coverage are improving and many websites now work.
+You may still encounter errors or crashes. Please open an issue with specifics if so.
 
 Here are the key features we have implemented:
 
+- [x] HTTP loader
 - [x] HTTP loader
 - [x] HTML parser and DOM tree (based on Netsurf libs)
 - [x] Javascript support (v8)
-- [x] Basic DOM APIs
+- [x] DOM APIs
 - [x] Ajax
   - [x] XHR API
-  - [x] Fetch API
+  - [x] Fetch API (polyfill)
 - [x] DOM dump
-- [x] Basic CDP/websockets server
+- [x] CDP/websockets server
+- [x] Click
+- [x] Input form
+- [x] Cookies
+- [x] Custom HTTP headers
+- [ ] Proxy support
+- [ ] Network interception 
 
 NOTE: There are hundreds of Web APIs. Developing a browser (even just for headless mode) is a huge task. Coverage will increase over time.
 

build.zig.zon

@@ -5,8 +5,8 @@
     .fingerprint = 0xda130f3af836cea0,
     .dependencies = .{
         .tls = .{
-            .url = "https://github.com/ianic/tls.zig/archive/8250aa9184fbad99983b32411bbe1a5d2fd6f4b7.tar.gz",
-            .hash = "tls-0.1.0-ER2e0pU3BQB-UD2_s90uvppceH_h4KZxtHCrCct8L054",
+            .url = "https://github.com/ianic/tls.zig/archive/55845f755d9e2e821458ea55693f85c737cd0c7a.tar.gz",
+            .hash = "tls-0.1.0-ER2e0m43BQAshi8ixj1qf3w2u2lqKtXtkrxUJ4AGZDcl",
         },
         .tigerbeetle_io = .{
             .url = "https://github.com/lightpanda-io/tigerbeetle-io/archive/61d9652f1a957b7f4db723ea6aa0ce9635e840ce.tar.gz",

src/browser/dom/attribute.zig

@@ -15,7 +15,6 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
 const parser = @import("../netsurf.zig");
 
 const Node = @import("node.zig").Node;
@@ -47,7 +46,14 @@ pub const Attr = struct {
     }
 
     pub fn set_value(self: *parser.Attribute, v: []const u8) !?[]const u8 {
-        try parser.attributeSetValue(self, v);
+        if (try parser.attributeGetOwnerElement(self)) |el| {
+            // if possible, go through the element, as that triggers a
+            // DOMAttrModified event (which MutationObserver cares about)
+            const name = try parser.attributeGetName(self);
+            try parser.elementSetAttribute(el, name, v);
+        } else {
+            try parser.attributeSetValue(self, v);
+        }
         return v;
     }
 

src/browser/dom/document.zig

@@ -237,7 +237,7 @@ pub const Document = struct {
     pub fn _querySelector(self: *parser.Document, selector: []const u8, page: *Page) !?ElementUnion {
         if (selector.len == 0) return null;
 
-        const n = try css.querySelector(page.arena, parser.documentToNode(self), selector);
+        const n = try css.querySelector(page.call_arena, parser.documentToNode(self), selector);
 
         if (n == null) return null;
 

src/browser/dom/document_fragment.zig

@@ -16,8 +16,12 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+const css = @import("css.zig");
 const parser = @import("../netsurf.zig");
 const Page = @import("../page.zig").Page;
+const NodeList = @import("nodelist.zig").NodeList;
+const Element = @import("element.zig").Element;
+const ElementUnion = @import("element.zig").Union;
 
 const Node = @import("node.zig").Node;
 
@@ -53,6 +57,20 @@ pub const DocumentFragment = struct {
     pub fn _replaceChildren(self: *parser.DocumentFragment, nodes: []const Node.NodeOrText) !void {
         return Node.replaceChildren(parser.documentFragmentToNode(self), nodes);
     }
+
+    pub fn _querySelector(self: *parser.DocumentFragment, selector: []const u8, page: *Page) !?ElementUnion {
+        if (selector.len == 0) return null;
+
+        const n = try css.querySelector(page.call_arena, parser.documentFragmentToNode(self), selector);
+
+        if (n == null) return null;
+
+        return try Element.toInterface(parser.nodeToElement(n.?));
+    }
+
+    pub fn _querySelectorAll(self: *parser.DocumentFragment, selector: []const u8, page: *Page) !NodeList {
+        return css.querySelectorAll(page.arena, parser.documentFragmentToNode(self), selector);
+    }
 };
 
 const testing = @import("../../testing.zig");
@@ -83,5 +101,11 @@ test "Browser.DOM.DocumentFragment" {
 
         .{ "document.getElementsByTagName('body')[0].append(f.cloneNode(true));", null },
         .{ "document.getElementById('x') != null;", "true" },
+
+        .{ "document.querySelector('.hello')", "null" },
+        .{ "document.querySelectorAll('.hello').length", "0" },
+
+        .{ "document.querySelector('#x').id", "x" },
+        .{ "document.querySelectorAll('#x')[0].id", "x" },
     }, .{});
 }

src/browser/dom/dom.zig

@@ -23,6 +23,7 @@ const NamedNodeMap = @import("namednodemap.zig").NamedNodeMap;
 const DOMTokenList = @import("token_list.zig");
 const NodeList = @import("nodelist.zig");
 const Node = @import("node.zig");
+const ResizeObserver = @import("resize_observer.zig");
 const MutationObserver = @import("mutation_observer.zig");
 const IntersectionObserver = @import("intersection_observer.zig");
 const DOMParser = @import("dom_parser.zig").DOMParser;
@@ -40,6 +41,7 @@ pub const Interfaces = .{
     NodeList.Interfaces,
     Node.Node,
     Node.Interfaces,
+    ResizeObserver.Interfaces,
     MutationObserver.Interfaces,
     IntersectionObserver.Interfaces,
     DOMParser,

src/browser/dom/element.zig

@@ -335,7 +335,7 @@ pub const Element = struct {
     pub fn _querySelector(self: *parser.Element, selector: []const u8, page: *Page) !?Union {
         if (selector.len == 0) return null;
 
-        const n = try css.querySelector(page.arena, parser.elementToNode(self), selector);
+        const n = try css.querySelector(page.call_arena, parser.elementToNode(self), selector);
 
         if (n == null) return null;
 

src/browser/dom/event_target.zig

@@ -23,10 +23,12 @@ const Page = @import("../page.zig").Page;
 const EventHandler = @import("../events/event.zig").EventHandler;
 
 const DOMException = @import("exceptions.zig").DOMException;
-const Nod = @import("node.zig");
+const nod = @import("node.zig");
 
-// EventTarget interfaces
-pub const Union = Nod.Union;
+pub const Union = union(enum) {
+    node: nod.Union,
+    xhr: *@import("../xhr/xhr.zig").XMLHttpRequest,
+};
 
 // EventTarget implementation
 pub const EventTarget = struct {
@@ -39,18 +41,22 @@ pub const EventTarget = struct {
         // The window is a common non-node target, but it's easy to handle as
         // its a singleton.
         if (@intFromPtr(et) == @intFromPtr(&page.window.base)) {
-            return .{ .Window = &page.window };
+            return .{ .node = .{ .Window = &page.window } };
         }
 
         // AbortSignal is another non-node target. It has a distinct usage though
         // so we hijack the event internal type to identity if.
         switch (try parser.eventGetInternalType(e)) {
             .abort_signal => {
-                return .{ .AbortSignal = @fieldParentPtr("proto", @as(*parser.EventTargetTBase, @ptrCast(et))) };
+                return .{ .node = .{ .AbortSignal = @fieldParentPtr("proto", @as(*parser.EventTargetTBase, @ptrCast(et))) } };
+            },
+            .xhr_event => {
+                const XMLHttpRequestEventTarget = @import("../xhr/event_target.zig").XMLHttpRequestEventTarget;
+                const base: *XMLHttpRequestEventTarget = @fieldParentPtr("base", @as(*parser.EventTargetTBase, @ptrCast(et)));
+                return .{ .xhr = @fieldParentPtr("proto", base) };
             },
             else => {
-                // some of these probably need to be special-cased like abort_signal
-                return Nod.Node.toInterface(@as(*parser.Node, @ptrCast(et)));
+                return .{ .node = try nod.Node.toInterface(@as(*parser.Node, @ptrCast(et))) };
             },
         }
     }

src/browser/dom/mutation_observer.zig

@@ -22,6 +22,7 @@ const Allocator = std.mem.Allocator;
 const log = @import("../../log.zig");
 const parser = @import("../netsurf.zig");
 const Page = @import("../page.zig").Page;
+const Loop = @import("../../runtime/loop.zig").Loop;
 
 const Env = @import("../env.zig").Env;
 const NodeList = @import("nodelist.zig").NodeList;
@@ -35,25 +36,37 @@ const Walker = @import("../dom/walker.zig").WalkerChildren;
 
 // WEB IDL https://dom.spec.whatwg.org/#interface-mutationobserver
 pub const MutationObserver = struct {
+    loop: *Loop,
     cbk: Env.Function,
     arena: Allocator,
+    connected: bool,
+    scheduled: bool,
+    loop_node: Loop.CallbackNode,
 
     // List of records which were observed. When the call scope ends, we need to
     // execute our callback with it.
-    observed: std.ArrayListUnmanaged(*MutationRecord),
+    observed: std.ArrayListUnmanaged(MutationRecord),
 
     pub fn constructor(cbk: Env.Function, page: *Page) !MutationObserver {
         return .{
             .cbk = cbk,
+            .loop = page.loop,
             .observed = .{},
+            .connected = true,
+            .scheduled = false,
             .arena = page.arena,
+            .loop_node = .{ .func = callback },
         };
     }
 
-    pub fn _observe(self: *MutationObserver, node: *parser.Node, options_: ?MutationObserverInit) !void {
-        const options = options_ orelse MutationObserverInit{};
+    pub fn _observe(self: *MutationObserver, node: *parser.Node, options_: ?Options) !void {
+        const arena = self.arena;
+        var options = options_ orelse Options{};
+        if (options.attributeFilter.len > 0) {
+            options.attributeFilter = try arena.dupe([]const u8, options.attributeFilter);
+        }
 
-        const observer = try self.arena.create(Observer);
+        const observer = try arena.create(Observer);
         observer.* = .{
             .node = node,
             .options = options,
@@ -102,30 +115,34 @@ pub const MutationObserver = struct {
         }
     }
 
-    pub fn jsCallScopeEnd(self: *MutationObserver) void {
-        const record = self.observed.items;
-        if (record.len == 0) {
+    fn callback(node: *Loop.CallbackNode, _: *?u63) void {
+        const self: *MutationObserver = @fieldParentPtr("loop_node", node);
+        if (self.connected == false) {
+            self.scheduled = true;
+            return;
+        }
+        self.scheduled = false;
+
+        const records = self.observed.items;
+        if (records.len == 0) {
             return;
         }
 
         defer self.observed.clearRetainingCapacity();
 
-        for (record) |r| {
-            const records = [_]MutationRecord{r.*};
-            var result: Env.Function.Result = undefined;
-            self.cbk.tryCall(void, .{records}, &result) catch {
-                log.debug(.user_script, "callback error", .{
-                    .err = result.exception,
-                    .stack = result.stack,
-                    .source = "mutation observer",
-                });
-            };
-        }
+        var result: Env.Function.Result = undefined;
+        self.cbk.tryCall(void, .{records}, &result) catch {
+            log.debug(.user_script, "callback error", .{
+                .err = result.exception,
+                .stack = result.stack,
+                .source = "mutation observer",
+            });
+        };
     }
 
     // TODO
-    pub fn _disconnect(_: *MutationObserver) !void {
-        // TODO unregister listeners.
+    pub fn _disconnect(self: *MutationObserver) !void {
+        self.connected = false;
     }
 
     // TODO
@@ -182,31 +199,27 @@ pub const MutationRecord = struct {
     }
 };
 
-const MutationObserverInit = struct {
+const Options = struct {
     childList: bool = false,
     attributes: bool = false,
     characterData: bool = false,
     subtree: bool = false,
     attributeOldValue: bool = false,
     characterDataOldValue: bool = false,
-    // TODO
-    // attributeFilter: [][]const u8,
+    attributeFilter: [][]const u8 = &.{},
 
-    fn attr(self: MutationObserverInit) bool {
-        return self.attributes or self.attributeOldValue;
+    fn attr(self: Options) bool {
+        return self.attributes or self.attributeOldValue or self.attributeFilter.len > 0;
     }
 
-    fn cdata(self: MutationObserverInit) bool {
+    fn cdata(self: Options) bool {
         return self.characterData or self.characterDataOldValue;
     }
 };
 
 const Observer = struct {
     node: *parser.Node,
-    options: MutationObserverInit,
-
-    // record of the mutation, all observed changes in 1 call are batched
-    record: ?MutationRecord = null,
+    options: Options,
 
     // reference back to the MutationObserver so that we can access the arena
     // and batch the mutation records.
@@ -214,19 +227,34 @@ const Observer = struct {
 
     event_node: parser.EventNode,
 
-    fn appliesTo(o: *const Observer, target: *parser.Node) bool {
+    fn appliesTo(
+        self: *const Observer,
+        target: *parser.Node,
+        event_type: MutationEventType,
+        event: *parser.MutationEvent,
+    ) !bool {
+        if (event_type == .DOMAttrModified and self.options.attributeFilter.len > 0) {
+            const attribute_name = try parser.mutationEventAttributeName(event);
+            for (self.options.attributeFilter) |needle| blk: {
+                if (std.mem.eql(u8, attribute_name, needle)) {
+                    break :blk;
+                }
+            }
+            return false;
+        }
+
         // mutation on any target is always ok.
-        if (o.options.subtree) {
+        if (self.options.subtree) {
             return true;
         }
 
         // if target equals node, alway ok.
-        if (target == o.node) {
+        if (target == self.node) {
             return true;
         }
 
         // no subtree, no same target and no childlist, always noky.
-        if (!o.options.childList) {
+        if (!self.options.childList) {
             return false;
         }
 
@@ -234,7 +262,7 @@ const Observer = struct {
         const walker = Walker{};
         var next: ?*parser.Node = null;
         while (true) {
-            next = walker.get_next(o.node, next) catch break orelse break;
+            next = walker.get_next(self.node, next) catch break orelse break;
             if (next.? == target) {
                 return true;
             }
@@ -258,27 +286,22 @@ const Observer = struct {
             break :blk parser.eventTargetToNode(event_target);
         };
 
-        if (self.appliesTo(node) == false) {
-            return;
-        }
-
+        const mutation_event = parser.eventToMutationEvent(event);
         const event_type = blk: {
             const t = try parser.eventType(event);
             break :blk std.meta.stringToEnum(MutationEventType, t) orelse return;
         };
 
-        const arena = mutation_observer.arena;
-        if (self.record == null) {
-            self.record = .{
-                .target = self.node,
-                .type = event_type.recordType(),
-            };
-            try mutation_observer.observed.append(arena, &self.record.?);
+        if (try self.appliesTo(node, event_type, mutation_event) == false) {
+            return;
         }
 
-        var record = &self.record.?;
-        const mutation_event = parser.eventToMutationEvent(event);
+        var record = MutationRecord{
+            .target = self.node,
+            .type = event_type.recordType(),
+        };
 
+        const arena = mutation_observer.arena;
         switch (event_type) {
             .DOMAttrModified => {
                 record.attribute_name = parser.mutationEventAttributeName(mutation_event) catch null;
@@ -302,6 +325,13 @@ const Observer = struct {
                 }
             },
         }
+
+        try mutation_observer.observed.append(arena, record);
+
+        if (mutation_observer.scheduled == false) {
+            mutation_observer.scheduled = true;
+            _ = try mutation_observer.loop.timeout(0, &mutation_observer.loop_node);
+        }
     }
 };
 
@@ -341,10 +371,10 @@ test "Browser.DOM.MutationObserver" {
             \\ document.firstElementChild.setAttribute("foo", "bar");
             \\ // ignored b/c it's about another target.
             \\ document.firstElementChild.firstChild.setAttribute("foo", "bar");
-            \\ nb;
             ,
-            "1",
+            null,
         },
+        .{ "nb", "1" },
         .{ "mrs[0].type", "attributes" },
         .{ "mrs[0].target == document.firstElementChild", "true" },
         .{ "mrs[0].target.getAttribute('foo')", "bar" },
@@ -362,10 +392,10 @@ test "Browser.DOM.MutationObserver" {
             \\     nb2++;
             \\ }).observe(node, { characterData: true, characterDataOldValue: true });
             \\ node.data = "foo";
-            \\ nb2;
             ,
-            "1",
+            null,
         },
+        .{ "nb2", "1" },
         .{ "mrs2[0].type", "characterData" },
         .{ "mrs2[0].target == node", "true" },
         .{ "mrs2[0].target.data", "foo" },
@@ -383,7 +413,24 @@ test "Browser.DOM.MutationObserver" {
             \\ }).observe(document, { subtree:true,childList:true });
             \\ node.innerText = "2";
             ,
-            "2",
+            null,
         },
+        .{ "node.innerText", "a" },
+    }, .{});
+
+    try runner.testCases(&.{
+        .{
+            \\ var node = document.getElementById("para");
+            \\ var attrWatch = 0;
+            \\ new MutationObserver(() => {
+            \\     attrWatch++;
+            \\ }).observe(document, { attributeFilter: ["name"], subtree: true });
+            \\ node.setAttribute("id", "1");
+            ,
+            null,
+        },
+        .{ "attrWatch", "0" },
+        .{ "node.setAttribute('name', 'other');", null },
+        .{ "attrWatch", "1" },
     }, .{});
 }

src/browser/dom/namednodemap.zig

@@ -134,5 +134,7 @@ test "Browser.DOM.NamedNodeMap" {
         .{ "a['id'].name", "id" },
         .{ "a['id'].value", "content" },
         .{ "a['other']", "undefined" },
+        .{ "a[0].value = 'abc123'", null },
+        .{ "a[0].value", "abc123" },
     }, .{});
 }

src/browser/dom/performance.zig

@@ -162,7 +162,7 @@ test "Performance: get_timeOrigin" {
     try testing.expect(time_origin >= 0);
 
     // Check resolution
-    try testing.expectDelta(@rem(time_origin * std.time.us_per_ms, 100.0), 0.0, 0.1);
+    try testing.expectDelta(@rem(time_origin * std.time.us_per_ms, 100.0), 0.0, 0.2);
 }
 
 test "Performance: now" {

src/browser/dom/resize_observer.zig

@@ -0,0 +1,54 @@
+// Copyright (C) 2023-2025  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const Env = @import("../env.zig").Env;
+const parser = @import("../netsurf.zig");
+
+pub const Interfaces = .{
+    ResizeObserver,
+};
+
+// WEB IDL https://drafts.csswg.org/resize-observer/#resize-observer-interface
+pub const ResizeObserver = struct {
+    pub fn constructor(cbk: Env.Function) ResizeObserver {
+        _ = cbk;
+        return .{};
+    }
+
+    pub fn _observe(self: *const ResizeObserver, element: *parser.Element, options_: ?Options) void {
+        _ = self;
+        _ = element;
+        _ = options_;
+        return;
+    }
+
+    pub fn _unobserve(self: *const ResizeObserver, element: *parser.Element) void {
+        _ = self;
+        _ = element;
+        return;
+    }
+
+    // TODO
+    pub fn _disconnect(self: *ResizeObserver) void {
+        _ = self;
+    }
+};
+
+const Options = struct {
+    box: []const u8,
+};

src/browser/events/event.zig

@@ -27,6 +27,7 @@ const Page = @import("../page.zig").Page;
 const DOMException = @import("../dom/exceptions.zig").DOMException;
 const EventTarget = @import("../dom/event_target.zig").EventTarget;
 const EventTargetUnion = @import("../dom/event_target.zig").Union;
+const AbortSignal = @import("../html/AbortController.zig").AbortSignal;
 
 const CustomEvent = @import("custom_event.zig").CustomEvent;
 const ProgressEvent = @import("../xhr/progress_event.zig").ProgressEvent;
@@ -54,7 +55,7 @@ pub const Event = struct {
 
     pub fn toInterface(evt: *parser.Event) !Union {
         return switch (try parser.eventGetInternalType(evt)) {
-            .event, .abort_signal => .{ .Event = evt },
+            .event, .abort_signal, .xhr_event => .{ .Event = evt },
             .custom_event => .{ .CustomEvent = @as(*CustomEvent, @ptrCast(evt)).* },
             .progress_event => .{ .ProgressEvent = @as(*ProgressEvent, @ptrCast(evt)).* },
             .mouse_event => .{ .MouseEvent = @as(*parser.MouseEvent, @ptrCast(evt)) },
@@ -175,7 +176,7 @@ pub const EventHandler = struct {
             // that the listener won't call preventDefault() and thus can safely
             // run the default as needed).
             passive: ?bool,
-            signal: ?bool, // currently does nothing
+            signal: ?*AbortSignal, // currently does nothing
         };
     };
 
@@ -188,18 +189,14 @@ pub const EventHandler = struct {
     ) !?*EventHandler {
         var once = false;
         var capture = false;
+        var signal: ?*AbortSignal = null;
+
         if (opts_) |opts| {
             switch (opts) {
                 .capture => |c| capture = c,
                 .flags => |f| {
-                    // Done this way so that, for common cases that _only_ set
-                    // capture, i.e. {captrue: true}, it works.
-                    // But for any case that sets any of the other flags, we
-                    // error. If we don't error, this function call would succeed
-                    // but the behavior might be wrong. At this point, it's
-                    // better to be explicit and error.
-                    if (f.signal orelse false) return error.NotImplemented;
                     once = f.once orelse false;
+                    signal = f.signal orelse null;
                     capture = f.capture orelse false;
                 },
             }
@@ -207,6 +204,28 @@ pub const EventHandler = struct {
 
         const callback = (try listener.callback(target)) orelse return null;
 
+        if (signal) |s| {
+            const signal_target = parser.toEventTarget(AbortSignal, s);
+
+            const scb = try allocator.create(SignalCallback);
+            scb.* = .{
+                .target = target,
+                .capture = capture,
+                .callback_id = callback.id,
+                .typ = try allocator.dupe(u8, typ),
+                .signal_target = signal_target,
+                .signal_listener = undefined,
+                .node = .{ .func = SignalCallback.handle },
+            };
+
+            scb.signal_listener = try parser.eventTargetAddEventListener(
+                signal_target,
+                "abort",
+                &scb.node,
+                false,
+            );
+        }
+
         // check if event target has already this listener
         if (try parser.eventTargetHasListener(target, typ, capture, callback.id) != null) {
             return null;
@@ -262,6 +281,50 @@ pub const EventHandler = struct {
     }
 };
 
+const SignalCallback = struct {
+    typ: []const u8,
+    capture: bool,
+    callback_id: usize,
+    node: parser.EventNode,
+    target: *parser.EventTarget,
+    signal_target: *parser.EventTarget,
+    signal_listener: *parser.EventListener,
+
+    fn handle(node: *parser.EventNode, _: *parser.Event) void {
+        const self: *SignalCallback = @fieldParentPtr("node", node);
+        self._handle() catch |err| {
+            log.err(.app, "event signal handler", .{ .err = err });
+        };
+    }
+
+    fn _handle(self: *SignalCallback) !void {
+        const lst = try parser.eventTargetHasListener(
+            self.target,
+            self.typ,
+            self.capture,
+            self.callback_id,
+        );
+        if (lst == null) {
+            return;
+        }
+
+        try parser.eventTargetRemoveEventListener(
+            self.target,
+            self.typ,
+            lst.?,
+            self.capture,
+        );
+
+        // remove the abort signal listener itself
+        try parser.eventTargetRemoveEventListener(
+            self.signal_target,
+            "abort",
+            self.signal_listener,
+            false,
+        );
+    }
+};
+
 const testing = @import("../../testing.zig");
 test "Browser.Event" {
     var runner = try testing.jsRunner(testing.tracking_allocator, .{});
@@ -367,5 +430,18 @@ test "Browser.Event" {
         .{ "document.dispatchEvent(new Event('count'))", "true" },
         .{ "document.dispatchEvent(new Event('count'))", "true" },
         .{ "nb", "1" },
+        .{ "document.removeEventListener('count', cbk)", "undefined" },
+    }, .{});
+
+    try runner.testCases(&.{
+        .{ "nb = 0; function cbk(event) { nb ++; }", null },
+        .{ "let ac = new AbortController()", null },
+        .{ "document.addEventListener('count', cbk, {signal: ac.signal})", null },
+        .{ "document.dispatchEvent(new Event('count'))", "true" },
+        .{ "document.dispatchEvent(new Event('count'))", "true" },
+        .{ "ac.abort()", null },
+        .{ "document.dispatchEvent(new Event('count'))", "true" },
+        .{ "nb", "2" },
+        .{ "document.removeEventListener('count', cbk)", "undefined" },
     }, .{});
 }

src/browser/html/document.zig

@@ -233,19 +233,23 @@ pub const HTMLDocument = struct {
     // Since LightPanda requires the client to know what they are clicking on we do not return the underlying element at this moment
     // This can currenty only happen if the first pixel is clicked without having rendered any element. This will change when css properties are supported.
     // This returns an ElementUnion instead of a *Parser.Element in case the element somehow hasn't passed through the js runtime yet.
-    pub fn _elementFromPoint(_: *parser.DocumentHTML, x: f32, y: f32, page: *Page) !?ElementUnion {
-        const ix: i32 = @intFromFloat(@floor(x));
-        const iy: i32 = @intFromFloat(@floor(y));
-        const element = page.renderer.getElementAtPosition(ix, iy) orelse return null;
+    // While x and y should be f32, here we take i32 since that's what our
+    // "renderer" uses. By specifying i32 here, rather than f32 and doing the
+    // conversion ourself, we rely on v8's type conversion which is both more
+    // flexible (e.g. handles NaN) and will be more consistent with a browser.
+    pub fn _elementFromPoint(_: *parser.DocumentHTML, x: i32, y: i32, page: *Page) !?ElementUnion {
+        const element = page.renderer.getElementAtPosition(x, y) orelse return null;
         // TODO if pointer-events set to none the underlying element should be returned (parser.documentGetDocumentElement(self.document);?)
         return try Element.toInterface(element);
     }
 
     // Returns an array of all elements at the specified coordinates (relative to the viewport). The elements are ordered from the topmost to the bottommost box of the viewport.
-    pub fn _elementsFromPoint(_: *parser.DocumentHTML, x: f32, y: f32, page: *Page) ![]ElementUnion {
-        const ix: i32 = @intFromFloat(@floor(x));
-        const iy: i32 = @intFromFloat(@floor(y));
-        const element = page.renderer.getElementAtPosition(ix, iy) orelse return &.{};
+    // While x and y should be f32, here we take i32 since that's what our
+    // "renderer" uses. By specifying i32 here, rather than f32 and doing the
+    // conversion ourself, we rely on v8's type conversion which is both more
+    // flexible (e.g. handles NaN) and will be more consistent with a browser.
+    pub fn _elementsFromPoint(_: *parser.DocumentHTML, x: i32, y: i32, page: *Page) ![]ElementUnion {
+        const element = page.renderer.getElementAtPosition(x, y) orelse return &.{};
         // TODO if pointer-events set to none the underlying element should be returned (parser.documentGetDocumentElement(self.document);?)
 
         var list: std.ArrayListUnmanaged(ElementUnion) = .empty;

src/browser/html/elements.zig

@@ -271,8 +271,18 @@ pub const HTMLAnchorElement = struct {
         return try parser.nodeSetTextContent(parser.anchorToNode(self), v);
     }
 
-    inline fn url(self: *parser.Anchor, page: *Page) !URL {
-        return URL.constructor(.{ .element = @alignCast(@ptrCast(self)) }, null, page); // TODO inject base url
+    fn url(self: *parser.Anchor, page: *Page) !URL {
+        // Although the URL.constructor union accepts an .{.element = X}, we
+        // can't use this here because the behavior is different.
+        //    URL.constructor(document.createElement('a')
+        // should fail (a.href isn't a valid URL)
+        // But
+        //     document.createElement('a').host
+        // should not fail, it should return an empty string
+        if (try parser.elementGetAttribute(@alignCast(@ptrCast(self)), "href")) |href| {
+            return URL.constructor(.{ .string = href }, null, page); // TODO inject base url
+        }
+        return .empty;
     }
 
     // TODO return a disposable string
@@ -885,6 +895,15 @@ pub const HTMLLinkElement = struct {
     pub fn constructor(page: *Page, js_this: Env.JsThis) !*parser.Element {
         return constructHtmlElement(page, js_this);
     }
+
+    pub fn get_href(self: *parser.Link) ![]const u8 {
+        return try parser.linkGetHref(self);
+    }
+
+    pub fn set_href(self: *parser.Link, href: []const u8, page: *const Page) !void {
+        const full = try urlStitch(page.call_arena, href, page.url.raw, .{});
+        return try parser.linkSetHref(self, full);
+    }
 };
 
 pub const HTMLMapElement = struct {
@@ -1559,6 +1578,8 @@ test "Browser.HTML.Element" {
 
     try runner.testCases(&.{
         .{ "let a = document.createElement('a');", null },
+        .{ "a.href", "" },
+        .{ "a.host", "" },
         .{ "a.href = 'about'", null },
         .{ "a.href", "https://lightpanda.io/opensource-browser/about" },
     }, .{});
@@ -1569,6 +1590,16 @@ test "Browser.HTML.Element" {
         .{ "document.createElement('a').focus()", null },
         .{ "document.activeElement === focused", "true" },
     }, .{});
+
+    try runner.testCases(&.{
+        .{ "let l2 = document.createElement('link');", null },
+        .{ "l2.href", "" },
+        .{ "l2.href = 'https://lightpanda.io/opensource-browser/15'", null },
+        .{ "l2.href", "https://lightpanda.io/opensource-browser/15" },
+
+        .{ "l2.href = '/over/9000'", null },
+        .{ "l2.href", "https://lightpanda.io/over/9000" },
+    }, .{});
 }
 
 test "Browser.HTML.Element.DataSet" {

src/browser/html/window.zig

@@ -298,9 +298,31 @@ pub const Window = struct {
             behavior: []const u8,
         };
     };
-    pub fn _scrollTo(_: *const Window, opts: ScrollToOpts, y: ?u32) void {
+    pub fn _scrollTo(self: *Window, opts: ScrollToOpts, y: ?u32) !void {
         _ = opts;
         _ = y;
+
+        {
+            const scroll_event = try parser.eventCreate();
+            defer parser.eventDestroy(scroll_event);
+
+            try parser.eventInit(scroll_event, "scroll", .{});
+            _ = try parser.eventTargetDispatchEvent(
+                parser.toEventTarget(Window, self),
+                scroll_event,
+            );
+        }
+
+        {
+            const scroll_end = try parser.eventCreate();
+            defer parser.eventDestroy(scroll_end);
+
+            try parser.eventInit(scroll_end, "scrollend", .{});
+            _ = try parser.eventTargetDispatchEvent(
+                parser.toEventTarget(parser.DocumentHTML, self.document),
+                scroll_end,
+            );
+        }
     }
 };
 
@@ -437,4 +459,13 @@ test "Browser.HTML.Window" {
         .{ "str", "https://ziglang.org/documentation/master/std/#std.base64.Base64Decoder" },
         .{ "try { atob('b') } catch (e) { e } ", "Error: InvalidCharacterError" },
     }, .{});
+
+    try runner.testCases(&.{
+        .{ "let scroll = false; let scrolend = false", null },
+        .{ "window.addEventListener('scroll', () => {scroll = true});", null },
+        .{ "document.addEventListener('scrollend', () => {scrollend = true});", null },
+        .{ "window.scrollTo(0)", null },
+        .{ "scroll", "true" },
+        .{ "scrollend", "true" },
+    }, .{});
 }

src/browser/netsurf.zig

@@ -527,6 +527,7 @@ pub const EventType = enum(u8) {
     mouse_event = 3,
     error_event = 4,
     abort_signal = 5,
+    xhr_event = 6,
 };
 
 pub const MutationEvent = c.dom_mutation_event;
@@ -1829,6 +1830,21 @@ pub fn anchorSetRel(a: *Anchor, rel: []const u8) !void {
     try DOMErr(err);
 }
 
+// HTMLLinkElement
+
+pub fn linkGetHref(link: *Link) ![]const u8 {
+    var res: ?*String = undefined;
+    const err = c.dom_html_link_element_get_href(link, &res);
+    try DOMErr(err);
+    if (res == null) return "";
+    return strToData(res.?);
+}
+
+pub fn linkSetHref(link: *Link, href: []const u8) !void {
+    const err = c.dom_html_link_element_set_href(link, try strFromData(href));
+    try DOMErr(err);
+}
+
 // ElementsHTML
 
 pub const MediaElement = struct { base: *c.dom_html_element };

src/browser/page.zig

@@ -122,10 +122,10 @@ pub const Page = struct {
         // load polyfills
         try polyfill.load(self.arena, self.main_context);
 
-        _ = try session.browser.app.loop.timeout(1 * std.time.ns_per_ms, &self.microtask_node);
         // message loop must run only non-test env
         if (comptime !builtin.is_test) {
-            _ = try session.browser.app.loop.timeout(1 * std.time.ns_per_ms, &self.messageloop_node);
+            _ = try session.browser.app.loop.timeout(1 * std.time.ns_per_ms, &self.microtask_node);
+            _ = try session.browser.app.loop.timeout(100 * std.time.ns_per_ms, &self.messageloop_node);
         }
     }
 
@@ -1017,7 +1017,7 @@ const Script = struct {
 
         const src: []const u8 = blk: {
             const s = self.src orelse break :blk page.url.raw;
-            break :blk try URL.stitch(page.arena, s, page.url.raw, .{.alloc = .if_needed});
+            break :blk try URL.stitch(page.arena, s, page.url.raw, .{ .alloc = .if_needed });
         };
 
         // if self.src is null, then this is an inline script, and it should

src/browser/url/url.zig

@@ -54,6 +54,11 @@ pub const URL = struct {
     uri: std.Uri,
     search_params: URLSearchParams,
 
+    pub const empty = URL{
+        .uri = .{ .scheme = "" },
+        .search_params = .{},
+    };
+
     const URLArg = union(enum) {
         url: *URL,
         element: *parser.ElementHTML,
@@ -224,6 +229,19 @@ pub const URL = struct {
     pub fn _toJSON(self: *URL, page: *Page) ![]const u8 {
         return self.get_href(page);
     }
+
+    pub fn set_pathname(self: *URL, fragment: []const u8, page: *Page) !void {
+        // pathname must always start with a '/';
+        const real_path = blk: {
+            if (std.mem.startsWith(u8, fragment, "/")) {
+                break :blk try page.arena.dupe(u8, fragment);
+            } else {
+                break :blk try std.fmt.allocPrint(page.arena, "/{s}", .{fragment});
+            }
+        };
+
+        self.uri.path = .{ .percent_encoded = real_path };
+    }
 };
 
 // uriComponentNullStr converts an optional std.Uri.Component to string value.

src/browser/xhr/event_target.zig

@@ -39,6 +39,7 @@ pub const XMLHttpRequestEventTarget = struct {
     onload_cbk: ?Function = null,
     ontimeout_cbk: ?Function = null,
     onloadend_cbk: ?Function = null,
+    onreadystatechange_cbk: ?Function = null,
 
     fn register(
         self: *XMLHttpRequestEventTarget,
@@ -86,6 +87,9 @@ pub const XMLHttpRequestEventTarget = struct {
     pub fn get_onloadend(self: *XMLHttpRequestEventTarget) ?Function {
         return self.onloadend_cbk;
     }
+    pub fn get_onreadystatechange(self: *XMLHttpRequestEventTarget) ?Function {
+        return self.onreadystatechange_cbk;
+    }
 
     pub fn set_onloadstart(self: *XMLHttpRequestEventTarget, listener: EventHandler.Listener, page: *Page) !void {
         if (self.onloadstart_cbk) |cbk| try self.unregister("loadstart", cbk.id);
@@ -111,4 +115,8 @@ pub const XMLHttpRequestEventTarget = struct {
         if (self.onloadend_cbk) |cbk| try self.unregister("loadend", cbk.id);
         self.onloadend_cbk = try self.register(page.arena, "loadend", listener);
     }
+    pub fn set_onreadystatechange(self: *XMLHttpRequestEventTarget, listener: EventHandler.Listener, page: *Page) !void {
+        if (self.onreadystatechange_cbk) |cbk| try self.unregister("readystatechange", cbk.id);
+        self.onreadystatechange_cbk = try self.register(page.arena, "readystatechange", listener);
+    }
 };

src/browser/xhr/xhr.zig

@@ -138,6 +138,13 @@ pub const XMLHttpRequest = struct {
         done = 4,
     };
 
+    // class attributes
+    pub const _UNSENT = @intFromEnum(State.unsent);
+    pub const _OPENED = @intFromEnum(State.opened);
+    pub const _HEADERS_RECEIVED = @intFromEnum(State.headers_received);
+    pub const _LOADING = @intFromEnum(State.loading);
+    pub const _DONE = @intFromEnum(State.done);
+
     // https://xhr.spec.whatwg.org/#response-type
     const ResponseType = enum {
         Empty,
@@ -360,6 +367,8 @@ pub const XMLHttpRequest = struct {
         // We can we defer event destroy once the event is dispatched.
         defer parser.eventDestroy(evt);
 
+        try parser.eventSetInternalType(evt, .xhr_event);
+
         try parser.eventInit(evt, typ, .{ .bubbles = true, .cancelable = true });
         _ = try parser.eventTargetDispatchEvent(@as(*parser.EventTarget, @ptrCast(self)), evt);
     }
@@ -579,11 +588,27 @@ pub const XMLHttpRequest = struct {
     }
 
     fn onErr(self: *XMLHttpRequest, err: anyerror) void {
-        self.state = .done;
         self.send_flag = false;
-        self.dispatchEvt("readystatechange");
-        self.dispatchProgressEvent("error", .{});
-        self.dispatchProgressEvent("loadend", .{});
+
+        // capture the state before we change it
+        const s = self.state;
+
+        const is_abort = err == DOMError.Abort;
+
+        if (is_abort) {
+            self.state = .unsent;
+        } else {
+            self.state = .done;
+            self.dispatchEvt("error");
+        }
+
+        if (s != .done or s != .unsent) {
+            self.dispatchEvt("readystatechange");
+            if (is_abort) {
+                self.dispatchProgressEvent("abort", .{});
+            }
+            self.dispatchProgressEvent("loadend", .{});
+        }
 
         const level: log.Level = if (err == DOMError.Abort) .debug else .err;
         log.log(.http, level, "error", .{
@@ -922,4 +947,26 @@ test "Browser.XHR.XMLHttpRequest" {
         // So the url has been retrieved.
         .{ "status", "200" },
     }, .{});
+
+    try runner.testCases(&.{
+        .{ "const req6 = new XMLHttpRequest()", null },
+        .{
+            \\ var readyStates = [];
+            \\ var currentTarget = null;
+            \\ req6.onreadystatechange = (e) => {
+            \\  currentTarget = e.currentTarget;
+            \\  readyStates.push(req6.readyState);
+            \\ }
+            ,
+            null,
+        },
+        .{ "req6.open('GET', 'https://127.0.0.1:9581/xhr')", null },
+        .{ "req6.send()", null },
+        .{ "readyStates.length", "4" },
+        .{ "readyStates[0] === XMLHttpRequest.OPENED", "true" },
+        .{ "readyStates[1] === XMLHttpRequest.HEADERS_RECEIVED", "true" },
+        .{ "readyStates[2] === XMLHttpRequest.LOADING", "true" },
+        .{ "readyStates[3] === XMLHttpRequest.DONE", "true" },
+        .{ "currentTarget == req6", "true" },
+    }, .{});
 }

src/http/client.zig

@@ -236,7 +236,7 @@ pub const Client = struct {
         return proxy_type == .connect;
     }
 
-    fn isSimpleProxy(self: *const Client) bool {
+    fn isForwardProxy(self: *const Client) bool {
         const proxy_type = self.proxy_type orelse return false;
         return proxy_type == .forward;
     }
@@ -322,11 +322,19 @@ const Connection = struct {
 
     const TLSClient = union(enum) {
         blocking: tls.Connection(std.net.Stream),
+        blocking_tlsproxy: struct {
+            proxy: tls.Connection(std.net.Stream), // Note, self-referential field. Proxy should be pinned in memory.
+            destination: tls.Connection(*tls.Connection(std.net.Stream)),
+        },
         nonblocking: tls.nonblock.Connection,
 
         fn close(self: *TLSClient) void {
             switch (self.*) {
                 .blocking => |*tls_client| tls_client.close() catch {},
+                .blocking_tlsproxy => |*tls_in_tls| {
+                    tls_in_tls.destination.close() catch {};
+                    tls_in_tls.proxy.close() catch {};
+                },
                 .nonblocking => {},
             }
         }
@@ -375,9 +383,6 @@ pub const Request = struct {
     // List of request headers
     headers: std.ArrayListUnmanaged(std.http.Header),
 
-    // whether or not we expect this connection to be secure
-    _secure: bool,
-
     // whether or not we should keep the underlying socket open and and usable
     // for other requests
     _keepalive: bool,
@@ -385,6 +390,10 @@ pub const Request = struct {
     // extracted from request_uri
     _request_port: u16,
     _request_host: []const u8,
+    // Whether or not we expect this connection to be secure, connection may still be secure due to proxy
+    _request_secure: bool,
+    // Whether or not we expect the SIMPLE/CONNECT proxy connection to be secure
+    _proxy_secure: bool,
 
     // extracted from connect_uri
     _connect_port: u16,
@@ -470,11 +479,12 @@ pub const Request = struct {
             .method = method,
             .notification = null,
             .arena = state.arena.allocator(),
-            ._secure = decomposed.secure,
             ._connect_host = decomposed.connect_host,
             ._connect_port = decomposed.connect_port,
+            ._proxy_secure = decomposed.proxy_secure,
             ._request_host = decomposed.request_host,
             ._request_port = decomposed.request_port,
+            ._request_secure = decomposed.request_secure,
             ._state = state,
             ._client = client,
             ._aborter = null,
@@ -506,12 +516,13 @@ pub const Request = struct {
     }
 
     const DecomposedURL = struct {
-        secure: bool,
         connect_port: u16,
         connect_host: []const u8,
         connect_uri: *const std.Uri,
+        proxy_secure: bool,
         request_port: u16,
         request_host: []const u8,
+        request_secure: bool,
     };
     fn decomposeURL(client: *const Client, uri: *const Uri) !DecomposedURL {
         if (uri.host == null) {
@@ -526,27 +537,31 @@ pub const Request = struct {
             connect_host = proxy.host.?.percent_encoded;
         }
 
-        const is_connect_proxy = client.isConnectProxy();
-
-        var secure: bool = undefined;
-        const scheme = if (is_connect_proxy) uri.scheme else connect_uri.scheme;
-        if (std.ascii.eqlIgnoreCase(scheme, "https")) {
-            secure = true;
-        } else if (std.ascii.eqlIgnoreCase(scheme, "http")) {
-            secure = false;
+        var request_secure: bool = undefined;
+        if (std.ascii.eqlIgnoreCase(uri.scheme, "https")) {
+            request_secure = true;
+        } else if (std.ascii.eqlIgnoreCase(uri.scheme, "http")) {
+            request_secure = false;
         } else {
             return error.UnsupportedUriScheme;
         }
-        const request_port: u16 = uri.port orelse if (secure) 443 else 80;
-        const connect_port: u16 = connect_uri.port orelse (if (is_connect_proxy) 80 else request_port);
+        const proxy_secure = client.http_proxy != null and std.ascii.eqlIgnoreCase(client.http_proxy.?.scheme, "https");
+
+        const request_port: u16 = uri.port orelse if (request_secure) 443 else 80;
+        const connect_port: u16 = connect_uri.port orelse blk: {
+            if (client.isConnectProxy()) {
+                if (proxy_secure) break :blk 443 else break :blk 80;
+            } else break :blk request_port;
+        };
 
         return .{
-            .secure = secure,
             .connect_port = connect_port,
             .connect_host = connect_host,
             .connect_uri = connect_uri,
+            .proxy_secure = proxy_secure,
             .request_port = request_port,
             .request_host = request_host,
+            .request_secure = request_secure,
         };
     }
 
@@ -655,19 +670,50 @@ pub const Request = struct {
             };
             self._connection = connection;
 
-            const is_connect_proxy = self._client.isConnectProxy();
-            if (is_connect_proxy) {
-                try SyncHandler.connect(self);
-            }
+            const tls_config = tls.config.Client{
+                .host = self._request_host,
+                .root_ca = self._client.root_ca,
+                .insecure_skip_verify = self._tls_verify_host == false,
+                // .key_log_callback = tls.config.key_log.callback,
+            };
 
-            if (self._secure) {
+            // proxy
+            const is_connect_proxy = self._client.isConnectProxy();
+
+            if (is_connect_proxy) {
+                var proxy_conn: SyncHandler.Conn = .{ .plain = self._connection.?.socket };
+
+                if (self._proxy_secure) {
+                    // Create an underlying TLS stream with the proxy
+                    var proxy_tls_config = tls_config;
+                    proxy_tls_config.host = self._connect_host;
+                    var proxy_conn_tls = try tls.client(std.net.Stream{ .handle = socket }, proxy_tls_config);
+                    proxy_conn = .{ .tls = &proxy_conn_tls };
+                }
+
+                // Connect to the proxy
+                try SyncHandler.connect(self, &proxy_conn);
+
+                if (self._proxy_secure) {
+                    if (self._request_secure) {
+                        // If secure endpoint, create the main TLS stream encapsulated into the TLS stream proxy
+                        self._connection.?.tls = .{
+                            .blocking_tlsproxy = .{
+                                .proxy = proxy_conn.tls.*,
+                                .destination = undefined,
+                            },
+                        };
+                        const proxy = &self._connection.?.tls.?.blocking_tlsproxy.proxy;
+                        self._connection.?.tls.?.blocking_tlsproxy.destination = try tls.client(proxy, tls_config);
+                    } else {
+                        // Otherwise, just use the TLS stream proxy
+                        self._connection.?.tls = .{ .blocking = proxy_conn.tls.* };
+                    }
+                }
+            }
+            if (self._request_secure and !self._proxy_secure and !self._client.isForwardProxy()) {
                 self._connection.?.tls = .{
-                    .blocking = try tls.client(std.net.Stream{ .handle = socket }, .{
-                        .host = if (is_connect_proxy) self._request_host else self._connect_host,
-                        .root_ca = self._client.root_ca,
-                        .insecure_skip_verify = self._tls_verify_host == false,
-                        // .key_log_callback = tls.config.key_log.callback,
-                    }),
+                    .blocking = try tls.client(std.net.Stream{ .handle = socket }, tls_config),
                 };
             }
 
@@ -746,7 +792,8 @@ pub const Request = struct {
             .conn = .{ .handler = async_handler, .protocol = .{ .plain = {} } },
         };
 
-        if (self._secure) {
+        if (self._client.isConnectProxy() and self._proxy_secure) log.warn(.http, "ASYNC TLS CONNECT no impl.", .{});
+        if (self._request_secure) {
             if (self._connection_from_keepalive) {
                 // If the connection came from the keepalive pool, than we already
                 // have a TLS Connection.
@@ -755,7 +802,7 @@ pub const Request = struct {
                 std.debug.assert(connection.tls == null);
                 async_handler.conn.protocol = .{
                     .handshake = tls.nonblock.Client.init(.{
-                        .host = if (self._client.isConnectProxy()) self._request_host else self._connect_host,
+                        .host = if (self._client.isConnectProxy()) self._request_host else self._connect_host, // looks wrong
                         .root_ca = self._client.root_ca,
                         .insecure_skip_verify = self._tls_verify_host == false,
                         .key_log_callback = tls.config.key_log.callback,
@@ -804,7 +851,7 @@ pub const Request = struct {
         try self.headers.append(arena, .{ .name = "User-Agent", .value = "Lightpanda/1.0" });
         try self.headers.append(arena, .{ .name = "Accept", .value = "*/*" });
 
-        if (self._client.isSimpleProxy()) {
+        if (self._client.isForwardProxy()) {
             if (self._client.proxy_auth) |proxy_auth| {
                 try self.headers.append(arena, .{ .name = "Proxy-Authorization", .value = proxy_auth });
             }
@@ -835,9 +882,10 @@ pub const Request = struct {
         const decomposed = try decomposeURL(self._client, self.request_uri);
         self.connect_uri = decomposed.connect_uri;
         self._request_host = decomposed.request_host;
+        self._request_secure = decomposed.request_secure;
         self._connect_host = decomposed.connect_host;
         self._connect_port = decomposed.connect_port;
-        self._secure = decomposed.secure;
+        self._proxy_secure = decomposed.proxy_secure;
         self._keepalive = false;
         self._redirect_count = redirect_count + 1;
 
@@ -885,7 +933,9 @@ pub const Request = struct {
             return null;
         }
 
-        return self._client.connection_manager.get(self._secure, self._connect_host, self._connect_port, blocking);
+        // A simple http proxy to an https destination is made into tls by the proxy, we see it as a plain connection
+        const expect_tls = self._proxy_secure or (self._request_secure and !self._client.isForwardProxy());
+        return self._client.connection_manager.get(expect_tls, self._connect_host, self._connect_port, blocking);
     }
 
     fn createSocket(self: *Request, blocking: bool) !struct { posix.socket_t, std.net.Address } {
@@ -908,7 +958,7 @@ pub const Request = struct {
     }
 
     fn buildHeader(self: *Request) ![]const u8 {
-        const proxied = self._client.isSimpleProxy();
+        const proxied = self._client.isForwardProxy();
 
         const buf = self._state.header_buf;
         var fbs = std.io.fixedBufferStream(buf);
@@ -1723,7 +1773,15 @@ const SyncHandler = struct {
         var conn: Conn = blk: {
             const c = request._connection.?;
             if (c.tls) |*tls_client| {
-                break :blk .{ .tls = &tls_client.blocking };
+                switch (tls_client.*) {
+                    .nonblocking => unreachable,
+                    .blocking => |*blocking| {
+                        break :blk .{ .tls = blocking };
+                    },
+                    .blocking_tlsproxy => |*blocking_tlsproxy| {
+                        break :blk .{ .tls_in_tls = &blocking_tlsproxy.destination };
+                    },
+                }
             }
             break :blk .{ .plain = c.socket };
         };
@@ -1806,11 +1864,9 @@ const SyncHandler = struct {
 
     // Unfortunately, this is called from the Request doSendSync since we need
     // to do this before setting up our TLS connection.
-    fn connect(request: *Request) !void {
-        const socket = request._connection.?.socket;
-
+    fn connect(request: *Request, conn: *Conn) !void {
         const header = try request.buildConnectHeader();
-        try Conn.writeAll(socket, header);
+        try conn.writeAll(header);
 
         var pos: usize = 0;
         var reader = request.newReader();
@@ -1821,7 +1877,7 @@ const SyncHandler = struct {
             // we only send CONNECT requests on newly established connections
             // and maybeRetryOrErr is only for connections that might have been
             // closed while being kept-alive
-            const n = try posix.read(socket, read_buf[pos..]);
+            const n = try conn.read(read_buf[pos..]);
             if (n == 0) {
                 return error.ConnectionResetByPeer;
             }
@@ -1833,6 +1889,7 @@ const SyncHandler = struct {
 
             // we don't have enough data yet.
         }
+        return;
     }
 
     fn maybeRetryOrErr(self: *SyncHandler, err: anyerror) !Response {
@@ -1882,12 +1939,13 @@ const SyncHandler = struct {
     }
 
     const Conn = union(enum) {
+        tls_in_tls: *tls.Connection(*tls.Connection(std.net.Stream)),
         tls: *tls.Connection(std.net.Stream),
         plain: posix.socket_t,
 
         fn sendRequest(self: *Conn, header: []const u8, body: ?[]const u8) !void {
             switch (self.*) {
-                .tls => |tls_client| {
+                inline .tls, .tls_in_tls => |tls_client| {
                     try tls_client.writeAll(header);
                     if (body) |b| {
                         try tls_client.writeAll(b);
@@ -1901,7 +1959,7 @@ const SyncHandler = struct {
                         };
                         return writeAllIOVec(socket, &vec);
                     }
-                    return writeAll(socket, header);
+                    return self.writeAll(header);
                 },
             }
         }
@@ -1909,6 +1967,7 @@ const SyncHandler = struct {
         fn read(self: *Conn, buf: []u8) !usize {
             const n = switch (self.*) {
                 .tls => |tls_client| try tls_client.read(buf),
+                .tls_in_tls => |tls_client| try tls_client.read(buf),
                 .plain => |socket| try posix.read(socket, buf),
             };
             if (n == 0) {
@@ -1917,6 +1976,19 @@ const SyncHandler = struct {
             return n;
         }
 
+        fn writeAll(self: *Conn, data: []const u8) !void {
+            switch (self.*) {
+                .tls => |tls_client| try tls_client.writeAll(data),
+                .tls_in_tls => |tls_client| try tls_client.writeAll(data),
+                .plain => |socket| {
+                    var i: usize = 0;
+                    while (i < data.len) {
+                        i += try posix.write(socket, data[i..]);
+                    }
+                },
+            }
+        }
+
         fn writeAllIOVec(socket: posix.socket_t, vec: []posix.iovec_const) !void {
             var i: usize = 0;
             while (true) {
@@ -1932,13 +2004,6 @@ const SyncHandler = struct {
                 vec[i].len -= n;
             }
         }
-
-        fn writeAll(socket: posix.socket_t, data: []const u8) !void {
-            var i: usize = 0;
-            while (i < data.len) {
-                i += try posix.write(socket, data[i..]);
-            }
-        }
     };
 
     // We don't ask for encoding, but some providers (CloudFront!!)
@@ -2083,6 +2148,7 @@ const Reader = struct {
         if (result.done == false) {
             // CONNECT responses should not have a body. If the header is
             // done, then the entire response should be done.
+            log.info(.http_client, "InvalidConnectResponse", .{ .status = self.response.status, .unprocessed = result.unprocessed });
             return error.InvalidConnectResponse;
         }
 
@@ -2909,14 +2975,14 @@ const ConnectionManager = struct {
         self.connection_pool.deinit();
     }
 
-    fn get(self: *ConnectionManager, secure: bool, host: []const u8, port: u16, blocking: bool) ?*Connection {
+    fn get(self: *ConnectionManager, expect_tls: bool, host: []const u8, port: u16, blocking: bool) ?*Connection {
         self.mutex.lock();
         defer self.mutex.unlock();
 
         var node = self.idle.first;
         while (node) |n| {
             const connection = n.data;
-            if (std.ascii.eqlIgnoreCase(connection.host, host) and connection.port == port and connection.blocking == blocking and ((connection.tls == null) == !secure)) {
+            if (std.ascii.eqlIgnoreCase(connection.host, host) and connection.port == port and connection.blocking == blocking and ((connection.tls == null) == !expect_tls)) {
                 self.count -= 1;
                 self.idle.remove(n);
                 self.node_pool.destroy(n);

src/runtime/js.zig

@@ -81,14 +81,14 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
     // that looks like:
     //
     // const TypeLookup = struct {
-    //     comptime cat: usize = TypeMeta{.index = 0, ...},
-    //     comptime owner: usize = TypeMeta{.index = 1, ...},
+    //     comptime cat: usize = 0,
+    //     comptime owner: usize = 1,
     //     ...
     // }
     //
     // So to get the template index of `owner`, we can do:
     //
-    //  const index_id = @field(type_lookup, @typeName(@TypeOf(res)).index;
+    //  const index_id = @field(type_lookup, @typeName(@TypeOf(res));
     //
     const TypeLookup = comptime blk: {
         var fields: [Types.len]std.builtin.Type.StructField = undefined;
@@ -103,15 +103,12 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 @compileError(std.fmt.comptimePrint("Prototype '{s}' for type '{s} must be a pointer", .{ @typeName(Struct.prototype), @typeName(Struct) }));
             }
 
-            const subtype: ?SubType = if (@hasDecl(Struct, "subtype")) Struct.subtype else null;
-
-            const R = Receiver(Struct);
             fields[i] = .{
-                .name = @typeName(R),
-                .type = TypeMeta,
+                .name = @typeName(Receiver(Struct)),
+                .type = usize,
                 .is_comptime = true,
                 .alignment = @alignOf(usize),
-                .default_value_ptr = &TypeMeta{ .index = i, .subtype = subtype },
+                .default_value_ptr = &i,
             };
         }
         break :blk @Type(.{ .@"struct" = .{
@@ -146,7 +143,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             if (@hasDecl(Struct, "prototype")) {
                 const TI = @typeInfo(Struct.prototype);
                 const proto_name = @typeName(Receiver(TI.pointer.child));
-                prototype_index = @field(TYPE_LOOKUP, proto_name).index;
+                prototype_index = @field(TYPE_LOOKUP, proto_name);
             }
             table[i] = prototype_index;
         }
@@ -168,7 +165,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
         // access to its TunctionTemplate (the thing we need to create an instance
         // of it)
         // I.e.:
-        // const index = @field(TYPE_LOOKUP, @typeName(type_name)).index
+        // const index = @field(TYPE_LOOKUP, @typeName(type_name))
         // const template = templates[index];
         templates: [Types.len]v8.FunctionTemplate,
 
@@ -177,6 +174,8 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
         // index.
         prototype_lookup: [Types.len]u16,
 
+        meta_lookup: [Types.len]TypeMeta,
+
         const Self = @This();
 
         const TYPE_LOOKUP = TypeLookup{};
@@ -222,13 +221,14 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 .templates = undefined,
                 .allocator = allocator,
                 .isolate_params = params,
+                .meta_lookup = undefined,
                 .prototype_lookup = undefined,
             };
 
             // Populate our templates lookup. generateClass creates the
             // v8.FunctionTemplate, which we store in our env.templates.
             // The ordering doesn't matter. What matters is that, given a type
-            // we can get its index via: @field(TYPE_LOOKUP, type_name).index
+            // we can get its index via: @field(TYPE_LOOKUP, type_name)
             const templates = &env.templates;
             inline for (Types, 0..) |s, i| {
                 @setEvalBranchQuota(10_000);
@@ -237,6 +237,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
 
             // Above, we've created all our our FunctionTemplates. Now that we
             // have them all, we can hook up the prototypes.
+            const meta_lookup = &env.meta_lookup;
             inline for (Types, 0..) |s, i| {
                 const Struct = s.defaultValue().?;
                 if (@hasDecl(Struct, "prototype")) {
@@ -249,9 +250,32 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                     // Just like we said above, given a type, we can get its
                     // template index.
 
-                    const proto_index = @field(TYPE_LOOKUP, proto_name).index;
+                    const proto_index = @field(TYPE_LOOKUP, proto_name);
                     templates[i].inherit(templates[proto_index]);
                 }
+
+                // while we're here, let's populate our meta lookup
+                const subtype: ?SubType = if (@hasDecl(Struct, "subtype")) Struct.subtype else null;
+
+                const proto_offset = comptime blk: {
+                    if (!@hasField(Struct, "proto")) {
+                        break :blk 0;
+                    }
+                    const proto_info = std.meta.fieldInfo(Struct, .proto);
+                    if (@typeInfo(proto_info.type) == .pointer) {
+                        // we store the offset as a negative, to so that,
+                        // when we reverse this, we know that it's
+                        // behind a pointer that we need to resolve.
+                        break :blk -@offsetOf(Struct, "proto");
+                    }
+                    break :blk @offsetOf(Struct, "proto");
+                };
+
+                meta_lookup[i] = .{
+                    .index = i,
+                    .subtype = subtype,
+                    .proto_offset = proto_offset,
+                };
             }
 
             return env;
@@ -391,7 +415,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                     if (@hasDecl(Global, "prototype")) {
                         const proto_type = Receiver(@typeInfo(Global.prototype).pointer.child);
                         const proto_name = @typeName(proto_type);
-                        const proto_index = @field(TYPE_LOOKUP, proto_name).index;
+                        const proto_index = @field(TYPE_LOOKUP, proto_name);
                         js_global.inherit(templates[proto_index]);
                     }
 
@@ -414,7 +438,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                                 @compileError("Type '" ++ @typeName(Struct) ++ "' defines an unknown prototype: " ++ proto_name);
                             }
 
-                            const proto_index = @field(TYPE_LOOKUP, proto_name).index;
+                            const proto_index = @field(TYPE_LOOKUP, proto_name);
                             const proto_obj = templates[proto_index].getFunction(v8_context).toObject();
 
                             const self_obj = templates[i].getFunction(v8_context).toObject();
@@ -449,6 +473,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                     .isolate = isolate,
                     .v8_context = v8_context,
                     .templates = &env.templates,
+                    .meta_lookup = &env.meta_lookup,
                     .handle_scope = handle_scope,
                     .call_arena = self.call_arena.allocator(),
                     .context_arena = self.context_arena.allocator(),
@@ -551,9 +576,12 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             v8_context: v8.Context,
             handle_scope: ?v8.HandleScope,
 
-            // references the Env.template array
+            // references Env.templates
             templates: []v8.FunctionTemplate,
 
+            // references the Env.meta_lookup
+            meta_lookup: []TypeMeta,
+
             // An arena for the lifetime of a call-group. Gets reset whenever
             // call_depth reaches 0.
             call_arena: Allocator,
@@ -595,9 +623,6 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             // Some Zig types have code to execute to cleanup
             destructor_callbacks: std.ArrayListUnmanaged(DestructorCallback) = .empty,
 
-            // Some Zig types have code to execute when the call scope ends
-            call_scope_end_callbacks: std.ArrayListUnmanaged(CallScopeEndCallback) = .empty,
-
             // Our module cache: normalized module specifier => module.
             module_cache: std.StringHashMapUnmanaged(PersistentModule) = .empty,
 
@@ -828,10 +853,6 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                             try self.destructor_callbacks.append(context_arena, DestructorCallback.init(value));
                         }
 
-                        if (comptime @hasDecl(ptr.child, "jsCallScopeEnd")) {
-                            try self.call_scope_end_callbacks.append(context_arena, CallScopeEndCallback.init(value));
-                        }
-
                         // Sometimes we're creating a new v8.Object, like when
                         // we're returning a value from a function. In those cases
                         // we have the FunctionTemplate, and we can get an object
@@ -852,12 +873,13 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                             // well as any meta data we'll need to use it later.
                             // See the TaggedAnyOpaque struct for more details.
                             const tao = try context_arena.create(TaggedAnyOpaque);
-                            const meta = @field(TYPE_LOOKUP, @typeName(ptr.child));
+                            const meta_index = @field(TYPE_LOOKUP, @typeName(ptr.child));
+                            const meta = self.meta_lookup[meta_index];
+
                             tao.* = .{
                                 .ptr = value,
                                 .index = meta.index,
                                 .subtype = meta.subtype,
-                                .offset = if (@typeInfo(ptr.child) != .@"opaque" and @hasField(ptr.child, "proto")) @offsetOf(ptr.child, "proto") else -1,
                             };
 
                             js_obj.setInternalField(0, v8.External.init(isolate, tao));
@@ -913,7 +935,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                             }
                             if (@hasField(TypeLookup, @typeName(ptr.child))) {
                                 const js_obj = js_value.castTo(v8.Object);
-                                return typeTaggedAnyOpaque(named_function, *Receiver(ptr.child), js_obj);
+                                return self.typeTaggedAnyOpaque(named_function, *Receiver(ptr.child), js_obj);
                             }
                         },
                         .slice => {
@@ -1208,7 +1230,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                                 // of having a version of typeTaggedAnyOpaque which
                                 // returns a boolean or an optional, we rely on the
                                 // main implementation and just handle the error.
-                                const attempt = typeTaggedAnyOpaque(named_function, *Receiver(ptr.child), js_obj);
+                                const attempt = self.typeTaggedAnyOpaque(named_function, *Receiver(ptr.child), js_obj);
                                 if (attempt) |value| {
                                     return .{ .value = value };
                                 } else |_| {
@@ -1394,6 +1416,78 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 try self.module_identifier.putNoClobber(arena, m.getIdentityHash(), owned_specifier);
                 return m.handle;
             }
+
+            // Reverses the mapZigInstanceToJs, making sure that our TaggedAnyOpaque
+            // contains a ptr to the correct type.
+            fn typeTaggedAnyOpaque(self: *const JsContext, comptime named_function: NamedFunction, comptime R: type, js_obj: v8.Object) !R {
+                const ti = @typeInfo(R);
+                if (ti != .pointer) {
+                    @compileError(named_function.full_name ++ "has a non-pointer Zig parameter type: " ++ @typeName(R));
+                }
+
+                const T = ti.pointer.child;
+                if (comptime isEmpty(T)) {
+                    // Empty structs aren't stored as TOAs and there's no data
+                    // stored in the JSObject's IntenrnalField. Why bother when
+                    // we can just return an empty struct here?
+                    return @constCast(@as(*const T, &.{}));
+                }
+
+                // if it isn't an empty struct, then the v8.Object should have an
+                // InternalFieldCount > 0, since our toa pointer should be embedded
+                // at index 0 of the internal field count.
+                if (js_obj.internalFieldCount() == 0) {
+                    return error.InvalidArgument;
+                }
+
+                const type_name = @typeName(T);
+                if (@hasField(TypeLookup, type_name) == false) {
+                    @compileError(named_function.full_name ++ "has an unknown Zig type: " ++ @typeName(R));
+                }
+
+                const op = js_obj.getInternalField(0).castTo(v8.External).get();
+                const toa: *TaggedAnyOpaque = @alignCast(@ptrCast(op));
+                const expected_type_index = @field(TYPE_LOOKUP, type_name);
+
+                var type_index = toa.index;
+                if (type_index == expected_type_index) {
+                    return @alignCast(@ptrCast(toa.ptr));
+                }
+
+                const meta_lookup = self.meta_lookup;
+
+                // If we have N levels deep of prototypes, then the offset is the
+                // sum at each level...
+                var total_offset: usize = 0;
+
+                // ...unless, the proto is behind a pointer, then total_offset will
+                // get reset to 0, and our base_ptr will move to the address
+                // referenced by the proto field.
+                var base_ptr: usize = @intFromPtr(toa.ptr);
+
+                // search through the prototype tree
+                while (true) {
+                    const proto_offset = meta_lookup[type_index].proto_offset;
+                    if (proto_offset < 0) {
+                        base_ptr = @as(*align(1) usize, @ptrFromInt(base_ptr + total_offset + @as(usize, @intCast(-proto_offset)))).*;
+                        total_offset = 0;
+                    } else {
+                        total_offset += @intCast(proto_offset);
+                    }
+
+                    const prototype_index = PROTOTYPE_TABLE[type_index];
+                    if (prototype_index == expected_type_index) {
+                        return @ptrFromInt(base_ptr + total_offset);
+                    }
+
+                    if (prototype_index == type_index) {
+                        // When a type has itself as the prototype, then we've
+                        // reached the end of the chain.
+                        return error.InvalidArgument;
+                    }
+                    type_index = prototype_index;
+                }
+            }
         };
 
         pub const Function = struct {
@@ -1428,7 +1522,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 const this_obj = if (@TypeOf(value) == JsObject)
                     value.js_obj
                 else
-                    try self.js_context.valueToExistingObject(value);
+                    (try self.js_context.zigValueToJs(value)).castTo(v8.Object);
 
                 return .{
                     .id = self.id,
@@ -2007,7 +2101,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             const template = v8.FunctionTemplate.initCallback(isolate, struct {
                 fn callback(raw_info: ?*const v8.C_FunctionCallbackInfo) callconv(.c) void {
                     const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
-                    var caller = Caller(Self, State).init(info);
+                    var caller = Caller(JsContext, State).init(info);
                     defer caller.deinit();
 
                     // See comment above. We generateConstructor on all types
@@ -2052,7 +2146,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             const function_template = v8.FunctionTemplate.initCallback(isolate, struct {
                 fn callback(raw_info: ?*const v8.C_FunctionCallbackInfo) callconv(.c) void {
                     const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
-                    var caller = Caller(Self, State).init(info);
+                    var caller = Caller(JsContext, State).init(info);
                     defer caller.deinit();
 
                     const named_function = comptime NamedFunction.init(Struct, name);
@@ -2069,7 +2163,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             const function_template = v8.FunctionTemplate.initCallback(isolate, struct {
                 fn callback(raw_info: ?*const v8.C_FunctionCallbackInfo) callconv(.c) void {
                     const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
-                    var caller = Caller(Self, State).init(info);
+                    var caller = Caller(JsContext, State).init(info);
                     defer caller.deinit();
 
                     const named_function = comptime NamedFunction.init(Struct, "static_" ++ name);
@@ -2105,7 +2199,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
             const getter_callback = v8.FunctionTemplate.initCallback(isolate, struct {
                 fn callback(raw_info: ?*const v8.C_FunctionCallbackInfo) callconv(.c) void {
                     const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
-                    var caller = Caller(Self, State).init(info);
+                    var caller = Caller(JsContext, State).init(info);
                     defer caller.deinit();
 
                     const named_function = comptime NamedFunction.init(Struct, "get_" ++ name);
@@ -2126,7 +2220,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                     const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
                     std.debug.assert(info.length() == 1);
 
-                    var caller = Caller(Self, State).init(info);
+                    var caller = Caller(JsContext, State).init(info);
                     defer caller.deinit();
 
                     const named_function = comptime NamedFunction.init(Struct, "set_" ++ name);
@@ -2147,7 +2241,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 .getter = struct {
                     fn callback(idx: u32, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 {
                         const info = v8.PropertyCallbackInfo.initFromV8(raw_info);
-                        var caller = Caller(Self, State).init(info);
+                        var caller = Caller(JsContext, State).init(info);
                         defer caller.deinit();
 
                         const named_function = comptime NamedFunction.init(Struct, "indexed_get");
@@ -2183,7 +2277,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 .getter = struct {
                     fn callback(c_name: ?*const v8.C_Name, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 {
                         const info = v8.PropertyCallbackInfo.initFromV8(raw_info);
-                        var caller = Caller(Self, State).init(info);
+                        var caller = Caller(JsContext, State).init(info);
                         defer caller.deinit();
 
                         const named_function = comptime NamedFunction.init(Struct, "named_get");
@@ -2205,7 +2299,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 configuration.setter = struct {
                     fn callback(c_name: ?*const v8.C_Name, c_value: ?*const v8.C_Value, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 {
                         const info = v8.PropertyCallbackInfo.initFromV8(raw_info);
-                        var caller = Caller(Self, State).init(info);
+                        var caller = Caller(JsContext, State).init(info);
                         defer caller.deinit();
 
                         const named_function = comptime NamedFunction.init(Struct, "named_set");
@@ -2221,7 +2315,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 configuration.deleter = struct {
                     fn callback(c_name: ?*const v8.C_Name, raw_info: ?*const v8.C_PropertyCallbackInfo) callconv(.c) u8 {
                         const info = v8.PropertyCallbackInfo.initFromV8(raw_info);
-                        var caller = Caller(Self, State).init(info);
+                        var caller = Caller(JsContext, State).init(info);
                         defer caller.deinit();
 
                         const named_function = comptime NamedFunction.init(Struct, "named_delete");
@@ -2267,7 +2361,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 template.setCallAsFunctionHandler(struct {
                     fn callback(raw_info: ?*const v8.C_FunctionCallbackInfo) callconv(.c) void {
                         const info = v8.FunctionCallbackInfo.initFromV8(raw_info);
-                        var caller = Caller(Self, State).init(info);
+                        var caller = Caller(JsContext, State).init(info);
                         defer caller.deinit();
 
                         const named_function = comptime NamedFunction.init(Struct, "jsCallAsFunction");
@@ -2322,7 +2416,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                     .one => {
                         const type_name = @typeName(ptr.child);
                         if (@hasField(TypeLookup, type_name)) {
-                            const template = templates[@field(TYPE_LOOKUP, type_name).index];
+                            const template = templates[@field(TYPE_LOOKUP, type_name)];
                             const js_obj = try JsContext.mapZigInstanceToJs(v8_context, template, value);
                             return js_obj.toValue();
                         }
@@ -2358,7 +2452,7 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
                 .@"struct" => |s| {
                     const type_name = @typeName(T);
                     if (@hasField(TypeLookup, type_name)) {
-                        const template = templates[@field(TYPE_LOOKUP, type_name).index];
+                        const template = templates[@field(TYPE_LOOKUP, type_name)];
                         const js_obj = try JsContext.mapZigInstanceToJs(v8_context, template, value);
                         return js_obj.toValue();
                     }
@@ -2427,69 +2521,6 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
 
             @compileError("A function returns an unsupported type: " ++ @typeName(T));
         }
-        // Reverses the mapZigInstanceToJs, making sure that our TaggedAnyOpaque
-        // contains a ptr to the correct type.
-        fn typeTaggedAnyOpaque(comptime named_function: NamedFunction, comptime R: type, js_obj: v8.Object) !R {
-            const ti = @typeInfo(R);
-            if (ti != .pointer) {
-                @compileError(named_function.full_name ++ "has a non-pointer Zig parameter type: " ++ @typeName(R));
-            }
-
-            const T = ti.pointer.child;
-            if (comptime isEmpty(T)) {
-                // Empty structs aren't stored as TOAs and there's no data
-                // stored in the JSObject's IntenrnalField. Why bother when
-                // we can just return an empty struct here?
-                return @constCast(@as(*const T, &.{}));
-            }
-
-            // if it isn't an empty struct, then the v8.Object should have an
-            // InternalFieldCount > 0, since our toa pointer should be embedded
-            // at index 0 of the internal field count.
-            if (js_obj.internalFieldCount() == 0) {
-                return error.InvalidArgument;
-            }
-
-            const type_name = @typeName(T);
-            if (@hasField(TypeLookup, type_name) == false) {
-                @compileError(named_function.full_name ++ "has an unknown Zig type: " ++ @typeName(R));
-            }
-
-            const op = js_obj.getInternalField(0).castTo(v8.External).get();
-            const toa: *TaggedAnyOpaque = @alignCast(@ptrCast(op));
-            const expected_type_index = @field(TYPE_LOOKUP, type_name).index;
-
-            var type_index = toa.index;
-            if (type_index == expected_type_index) {
-                return @alignCast(@ptrCast(toa.ptr));
-            }
-
-            // search through the prototype tree
-            while (true) {
-                const prototype_index = PROTOTYPE_TABLE[type_index];
-                if (prototype_index == expected_type_index) {
-                    // -1 is a sentinel value used for non-composition prototype
-                    // This is used with netsurf and we just unsafely cast one
-                    // type to another
-                    const offset = toa.offset;
-                    if (offset == -1) {
-                        return @alignCast(@ptrCast(toa.ptr));
-                    }
-
-                    // A non-negative offset means we're using composition prototype
-                    // (i.e. our struct has a "proto" field). the offset
-                    // reresents the byte offset of the field. We can use that
-                    // + the toa.ptr to get the field
-                    return @ptrFromInt(@intFromPtr(toa.ptr) + @as(usize, @intCast(offset)));
-                }
-                if (prototype_index == type_index) {
-                    // When a type has itself as the prototype, then we've
-                    // reached the end of the chain.
-                    return error.InvalidArgument;
-                }
-                type_index = prototype_index;
-            }
-        }
 
         // An interface for types that want to have their jsDeinit function to be
         // called when the call context ends
@@ -2549,24 +2580,24 @@ pub fn Env(comptime State: type, comptime WebApis: type) type {
     };
 }
 
-// We'll create a struct with all the types we want to bind to JavaScript. The
-// fields for this struct will be the type names. The values, will be an
-// instance of this struct.
-// const TypeLookup = struct {
-//     comptime cat: usize = TypeMeta{.index = 0, subtype = null},
-//     comptime owner: usize = TypeMeta{.index = 1, subtype = .array}.
-//     ...
-// }
-// This is essentially meta data for each type.
+// This is essentially meta data for each type. Each is stored in env.meta_lookup
+// The index for a type can be retrieved via:
+//   const index = @field(TYPE_LOOKUP, @typeName(Receiver(Struct)));
+//   const meta = env.meta_lookup[index];
 const TypeMeta = struct {
     // Every type is given a unique index. That index is used to lookup various
     // things, i.e. the prototype chain.
-    index: usize,
+    index: u16,
 
     // We store the type's subtype here, so that when we create an instance of
     // the type, and bind it to JavaScript, we can store the subtype along with
     // the created TaggedAnyOpaque.s
     subtype: ?SubType,
+
+    // If this type has composition-based prototype, represents the byte-offset
+    // from ptr where the `proto` field is located. A negative offsets is used
+    // to indicate that the prototype field is behind a pointer.
+    proto_offset: i32,
 };
 
 // When we map a Zig instance into a JsObject, we'll normally store the a
@@ -2597,9 +2628,9 @@ fn isComplexAttributeType(ti: std.builtin.Type) bool {
 // probably just contained in ExecutionWorld, but having this specific logic, which
 // is somewhat repetitive between constructors, functions, getters, etc contained
 // here does feel like it makes it clenaer.
-fn Caller(comptime E: type, comptime State: type) type {
+fn Caller(comptime JsContext: type, comptime State: type) type {
     return struct {
-        js_context: *E.JsContext,
+        js_context: *JsContext,
         v8_context: v8.Context,
         isolate: v8.Isolate,
         call_arena: Allocator,
@@ -2612,7 +2643,7 @@ fn Caller(comptime E: type, comptime State: type) type {
         fn init(info: anytype) Self {
             const isolate = info.getIsolate();
             const v8_context = isolate.getCurrentContext();
-            const js_context: *E.JsContext = @ptrFromInt(v8_context.getEmbedderData(1).castTo(v8.BigInt).getUint64());
+            const js_context: *JsContext = @ptrFromInt(v8_context.getEmbedderData(1).castTo(v8.BigInt).getUint64());
 
             js_context.call_depth += 1;
             return .{
@@ -2639,10 +2670,6 @@ fn Caller(comptime E: type, comptime State: type) type {
             // Therefore, we keep a call_depth, and only reset the call_arena
             // when a top-level (call_depth == 0) function ends.
             if (call_depth == 0) {
-                for (js_context.call_scope_end_callbacks.items) |cb| {
-                    cb.callScopeEnd();
-                }
-
                 const arena: *ArenaAllocator = @alignCast(@ptrCast(js_context.call_arena.ptr));
                 _ = arena.reset(.{ .retain_with_limit = CALL_ARENA_RETAIN });
             }
@@ -2664,9 +2691,9 @@ fn Caller(comptime E: type, comptime State: type) type {
             const this = info.getThis();
             if (@typeInfo(ReturnType) == .error_union) {
                 const non_error_res = res catch |err| return err;
-                _ = try E.JsContext.mapZigInstanceToJs(self.v8_context, this, non_error_res);
+                _ = try JsContext.mapZigInstanceToJs(self.v8_context, this, non_error_res);
             } else {
-                _ = try E.JsContext.mapZigInstanceToJs(self.v8_context, this, res);
+                _ = try JsContext.mapZigInstanceToJs(self.v8_context, this, res);
             }
             info.getReturnValue().set(this);
         }
@@ -2679,7 +2706,7 @@ fn Caller(comptime E: type, comptime State: type) type {
             const js_context = self.js_context;
             const func = @field(Struct, named_function.name);
             var args = try self.getArgs(Struct, named_function, 1, info);
-            const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
+            const zig_instance = try js_context.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
 
             // inject 'self' as the first parameter
             @field(args, "0") = zig_instance;
@@ -2711,7 +2738,7 @@ fn Caller(comptime E: type, comptime State: type) type {
             switch (arg_fields.len) {
                 0, 1, 2 => @compileError(named_function.full_name ++ " must take at least a u32 and *bool parameter"),
                 3, 4 => {
-                    const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
+                    const zig_instance = try js_context.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
                     comptime assertSelfReceiver(Struct, named_function);
                     @field(args, "0") = zig_instance;
                     @field(args, "1") = idx;
@@ -2733,12 +2760,13 @@ fn Caller(comptime E: type, comptime State: type) type {
         }
 
         fn getNamedIndex(self: *Self, comptime Struct: type, comptime named_function: NamedFunction, name: v8.Name, info: v8.PropertyCallbackInfo) !u8 {
+            const js_context = self.js_context;
             const func = @field(Struct, named_function.name);
             comptime assertSelfReceiver(Struct, named_function);
 
             var has_value = true;
             var args = try self.getArgs(Struct, named_function, 3, info);
-            const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
+            const zig_instance = try js_context.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
             @field(args, "0") = zig_instance;
             @field(args, "1") = try self.nameToString(name);
             @field(args, "2") = &has_value;
@@ -2758,7 +2786,7 @@ fn Caller(comptime E: type, comptime State: type) type {
 
             var has_value = true;
             var args = try self.getArgs(Struct, named_function, 4, info);
-            const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
+            const zig_instance = try js_context.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
             @field(args, "0") = zig_instance;
             @field(args, "1") = try self.nameToString(name);
             @field(args, "2") = try js_context.jsValueToZig(named_function, @TypeOf(@field(args, "2")), js_value);
@@ -2769,12 +2797,13 @@ fn Caller(comptime E: type, comptime State: type) type {
         }
 
         fn deleteNamedIndex(self: *Self, comptime Struct: type, comptime named_function: NamedFunction, name: v8.Name, info: v8.PropertyCallbackInfo) !u8 {
+            const js_context = self.js_context;
             const func = @field(Struct, named_function.name);
             comptime assertSelfReceiver(Struct, named_function);
 
             var has_value = true;
             var args = try self.getArgs(Struct, named_function, 3, info);
-            const zig_instance = try E.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
+            const zig_instance = try js_context.typeTaggedAnyOpaque(named_function, *Receiver(Struct), info.getThis());
             @field(args, "0") = zig_instance;
             @field(args, "1") = try self.nameToString(name);
             @field(args, "2") = &has_value;
@@ -3390,12 +3419,6 @@ const TaggedAnyOpaque = struct {
     // PROTOTYPE_TABLE
     index: u16,
 
-    // If this type has composition-based prototype, represents the byte-offset
-    // from ptr where the `proto` field is located. The value -1 represents
-    // unsafe prototype where we can just cast ptr to the destination type
-    // (this is used extensively with netsurf)
-    offset: i32,
-
     // Ptr to the Zig instance. Between the context where it's called (i.e.
     // we have the comptime parameter info for all functions), and the index field
     // we can figure out what type this is.
@@ -3431,7 +3454,7 @@ fn valueToDetailString(arena: Allocator, value: v8.Value, isolate: v8.Isolate, v
             if (debugValueToString(arena, value.castTo(v8.Object), isolate, v8_context)) |ds| {
                 return ds;
             } else |err| {
-                log.err(.js, "debug serialize value", .{.err = err});
+                log.err(.js, "debug serialize value", .{ .err = err });
             }
         }
     }

src/runtime/test_object_types.zig

@@ -77,6 +77,117 @@ pub const MyAPI = struct {
     }
 };
 
+pub const Parent = packed struct {
+    parent_id: i32 = 0,
+
+    pub fn get_parent(self: *const Parent) i32 {
+        return self.parent_id;
+    }
+    pub fn set_parent(self: *Parent, id: i32) void {
+        self.parent_id = id;
+    }
+};
+
+pub const Middle = struct {
+    pub const prototype = *Parent;
+
+    middle_id: i32 = 0,
+    _padding_1: u8 = 0,
+    _padding_2: u8 = 1,
+    _padding_3: u8 = 2,
+    proto: Parent,
+
+    pub fn constructor() Middle {
+        return .{
+            .middle_id = 0,
+            .proto = .{ .parent_id = 0 },
+        };
+    }
+
+    pub fn get_middle(self: *const Middle) i32 {
+        return self.middle_id;
+    }
+    pub fn set_middle(self: *Middle, id: i32) void {
+        self.middle_id = id;
+    }
+};
+
+pub const Child = struct {
+    pub const prototype = *Middle;
+
+    child_id: i32 = 0,
+    _padding_1: u8 = 0,
+    proto: Middle,
+
+    pub fn constructor() Child {
+        return .{
+            .child_id = 0,
+            .proto = .{ .middle_id = 0, .proto = .{ .parent_id = 0 } },
+        };
+    }
+
+    pub fn get_child(self: *const Child) i32 {
+        return self.child_id;
+    }
+    pub fn set_child(self: *Child, id: i32) void {
+        self.child_id = id;
+    }
+};
+
+pub const MiddlePtr = packed struct {
+    pub const prototype = *Parent;
+
+    middle_id: i32 = 0,
+    _padding_1: u8 = 0,
+    _padding_2: u8 = 1,
+    _padding_3: u8 = 2,
+    proto: *Parent,
+
+    pub fn constructor(state: State) !MiddlePtr {
+        const parent = try state.arena.create(Parent);
+        parent.* = .{ .parent_id = 0 };
+        return .{
+            .middle_id = 0,
+            .proto = parent,
+        };
+    }
+
+    pub fn get_middle(self: *const MiddlePtr) i32 {
+        return self.middle_id;
+    }
+    pub fn set_middle(self: *MiddlePtr, id: i32) void {
+        self.middle_id = id;
+    }
+};
+
+pub const ChildPtr = packed struct {
+    pub const prototype = *MiddlePtr;
+
+    child_id: i32 = 0,
+    _padding_1: u8 = 0,
+    _padding_2: u8 = 1,
+    proto: *MiddlePtr,
+
+    pub fn constructor(state: State) !ChildPtr {
+        const parent = try state.arena.create(Parent);
+        const middle = try state.arena.create(MiddlePtr);
+
+        parent.* = .{ .parent_id = 0 };
+        middle.* = .{ .middle_id = 0, .proto = parent };
+        return .{
+            .child_id = 0,
+            .proto = middle,
+        };
+    }
+
+    pub fn get_child(self: *const ChildPtr) i32 {
+        return self.child_id;
+    }
+    pub fn set_child(self: *ChildPtr, id: i32) void {
+        self.child_id = id;
+    }
+};
+
 const State = struct {
     arena: Allocator,
 };
@@ -90,6 +201,11 @@ test "JS: object types" {
         Other,
         MyObject,
         MyAPI,
+        Parent,
+        Middle,
+        Child,
+        MiddlePtr,
+        ChildPtr,
     }).init(.{ .arena = arena.allocator() }, {});
 
     defer runner.deinit();
@@ -120,4 +236,40 @@ test "JS: object types" {
         // check object property
         .{ "myObjIndirect.a.val()", "4" },
     }, .{});
+
+    try runner.testCases(&.{
+        .{ "let m1 = new Middle();", null },
+        .{ "m1.middle = 2", null },
+        .{ "m1.parent = 3", null },
+        .{ "m1.middle", "2" },
+        .{ "m1.parent", "3" },
+    }, .{});
+
+    try runner.testCases(&.{
+        .{ "let c1 = new Child();", null },
+        .{ "c1.child = 1", null },
+        .{ "c1.middle = 2", null },
+        .{ "c1.parent = 3", null },
+        .{ "c1.child", "1" },
+        .{ "c1.middle", "2" },
+        .{ "c1.parent", "3" },
+    }, .{});
+
+    try runner.testCases(&.{
+        .{ "let m2 = new MiddlePtr();", null },
+        .{ "m2.middle = 2", null },
+        .{ "m2.parent = 3", null },
+        .{ "m2.middle", "2" },
+        .{ "m2.parent", "3" },
+    }, .{});
+
+    try runner.testCases(&.{
+        .{ "let c2 = new ChildPtr();", null },
+        .{ "c2.child = 1", null },
+        .{ "c2.middle = 2", null },
+        .{ "c2.parent = 3", null },
+        .{ "c2.child", "1" },
+        .{ "c2.middle", "2" },
+        .{ "c2.parent", "3" },
+    }, .{});
 }

src/server.zig

@@ -39,7 +39,7 @@ const CDP = @import("cdp/cdp.zig").CDP;
 
 const TimeoutCheck = std.time.ns_per_ms * 100;
 
-const MAX_HTTP_REQUEST_SIZE = 2048;
+const MAX_HTTP_REQUEST_SIZE = 4096;
 
 // max message size
 // +14 for max websocket payload overhead
@@ -223,7 +223,7 @@ pub const Client = struct {
     }
 
     fn close(self: *Self) void {
-        log.info(.app, "client disconected", .{});
+        log.info(.app, "client disconnected", .{});
         self.connected = false;
         // recv only, because we might have pending writes we'd like to get
         // out (like the HTTP error response)
@@ -1142,7 +1142,7 @@ test "Client: http invalid request" {
     var c = try createTestClient();
     defer c.deinit();
 
-    const res = try c.httpRequest("GET /over/9000 HTTP/1.1\r\n" ++ "Header: " ++ ("a" ** 2050) ++ "\r\n\r\n");
+    const res = try c.httpRequest("GET /over/9000 HTTP/1.1\r\n" ++ "Header: " ++ ("a" ** 4100) ++ "\r\n\r\n");
     try testing.expectEqualStrings("HTTP/1.1 413 \r\n" ++
         "Connection: Close\r\n" ++
         "Content-Length: 17\r\n\r\n" ++