Merge pull request #2044 from lightpanda-io/proxy-auth-challenge

http: add connect code into auth challenge detection

.github/workflows/e2e-test.yml

@@ -100,14 +100,14 @@ jobs:
           ./proxy/proxy & echo $! > PROXY.id
           ./lightpanda serve --http-proxy 'http://127.0.0.1:3000' & echo $! > LPD.pid
           go run runner/main.go
+          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
-      - name: run request interception through proxy
+      - name: run request interception through proxy and playwright
         run: |
           export PROXY_USERNAME=username PROXY_PASSWORD=password
           ./proxy/proxy & echo $! > PROXY.id
           ./lightpanda serve & echo $! > LPD.pid
-          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
@@ -161,14 +161,18 @@ jobs:
             --http-proxy 'http://127.0.0.1:3000' \
             & echo $! > LPD.pid
           go run runner/main.go
+          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 
-      - name: run request interception through proxy
+      - name: run request interception through proxy and playwright
         run: |
           export PROXY_USERNAME=username PROXY_PASSWORD=password
           ./proxy/proxy & echo $! > PROXY.id
-          ./lightpanda serve & echo $! > LPD.pid
+          ./lightpanda serve \
-          URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
+            --web-bot-auth-key-file private_key.pem \
+            --web-bot-auth-keyid ${{ vars.WBA_KEY_ID }} \
+            --web-bot-auth-domain ${{ vars.WBA_DOMAIN }} \
+            & echo $! > LPD.pid
           BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
           kill `cat LPD.pid` `cat PROXY.id`
 

src/SemanticTree.zig

@@ -97,7 +97,6 @@ const NodeData = struct {
     options: ?[]OptionData = null,
     xpath: []const u8,
     is_interactive: bool,
-    is_disabled: bool,
     node_name: []const u8,
 };
 
@@ -149,7 +148,6 @@ fn walk(
     const role = try axn.getRole();
 
     var is_interactive = false;
-    var is_disabled = false;
     var value: ?[]const u8 = null;
     var options: ?[]OptionData = null;
     var node_name: []const u8 = "text";
@@ -174,8 +172,6 @@ fn walk(
                 is_interactive = true;
             }
         }
-
-        is_disabled = el.isDisabled();
     } else if (node._type == .document or node._type == .document_fragment) {
         node_name = "root";
     }
@@ -240,7 +236,6 @@ fn walk(
         .options = options,
         .xpath = xpath,
         .is_interactive = is_interactive,
-        .is_disabled = is_disabled,
         .node_name = node_name,
     };
 
@@ -352,11 +347,6 @@ const JsonVisitor = struct {
             try self.jw.objectField("isInteractive");
             try self.jw.write(data.is_interactive);
 
-            if (data.is_disabled) {
-                try self.jw.objectField("isDisabled");
-                try self.jw.write(true);
-            }
-
             try self.jw.objectField("role");
             try self.jw.write(data.role);
 
@@ -469,9 +459,6 @@ const TextVisitor = struct {
         const is_text_only = std.mem.eql(u8, data.role, "StaticText") or std.mem.eql(u8, data.role, "none") or std.mem.eql(u8, data.role, "generic");
 
         try self.writer.print("{d}", .{data.id});
-        if (data.is_interactive) {
-            try self.writer.writeAll(if (data.is_disabled) " [i:disabled]" else " [i]");
-        }
         if (!is_text_only) {
             try self.writer.print(" {s}", .{data.role});
         }
@@ -522,177 +509,6 @@ const TextVisitor = struct {
     }
 };
 
-pub const NodeDetails = struct {
-    backendNodeId: CDPNode.Id,
-    tag_name: []const u8,
-    role: []const u8,
-    name: ?[]const u8,
-    is_interactive: bool,
-    is_disabled: bool,
-    value: ?[]const u8 = null,
-    input_type: ?[]const u8 = null,
-    placeholder: ?[]const u8 = null,
-    href: ?[]const u8 = null,
-    id: ?[]const u8 = null,
-    class: ?[]const u8 = null,
-    checked: ?bool = null,
-    options: ?[]OptionData = null,
-
-    pub fn jsonStringify(self: *const NodeDetails, jw: anytype) !void {
-        try jw.beginObject();
-
-        try jw.objectField("backendNodeId");
-        try jw.write(self.backendNodeId);
-
-        try jw.objectField("tagName");
-        try jw.write(self.tag_name);
-
-        try jw.objectField("role");
-        try jw.write(self.role);
-
-        if (self.name) |n| {
-            try jw.objectField("name");
-            try jw.write(n);
-        }
-
-        try jw.objectField("isInteractive");
-        try jw.write(self.is_interactive);
-
-        if (self.is_disabled) {
-            try jw.objectField("isDisabled");
-            try jw.write(true);
-        }
-
-        if (self.value) |v| {
-            try jw.objectField("value");
-            try jw.write(v);
-        }
-
-        if (self.input_type) |v| {
-            try jw.objectField("inputType");
-            try jw.write(v);
-        }
-
-        if (self.placeholder) |v| {
-            try jw.objectField("placeholder");
-            try jw.write(v);
-        }
-
-        if (self.href) |v| {
-            try jw.objectField("href");
-            try jw.write(v);
-        }
-
-        if (self.id) |v| {
-            try jw.objectField("id");
-            try jw.write(v);
-        }
-
-        if (self.class) |v| {
-            try jw.objectField("class");
-            try jw.write(v);
-        }
-
-        if (self.checked) |c| {
-            try jw.objectField("checked");
-            try jw.write(c);
-        }
-
-        if (self.options) |opts| {
-            try jw.objectField("options");
-            try jw.beginArray();
-            for (opts) |opt| {
-                try jw.beginObject();
-                try jw.objectField("value");
-                try jw.write(opt.value);
-                try jw.objectField("text");
-                try jw.write(opt.text);
-                if (opt.selected) {
-                    try jw.objectField("selected");
-                    try jw.write(true);
-                }
-                try jw.endObject();
-            }
-            try jw.endArray();
-        }
-
-        try jw.endObject();
-    }
-};
-
-pub fn getNodeDetails(node: *Node, registry: *CDPNode.Registry, page: *Page, arena: std.mem.Allocator) !NodeDetails {
-    const cdp_node = try registry.register(node);
-    const axn = AXNode.fromNode(node);
-    const role = try axn.getRole();
-    const name = try axn.getName(page, arena);
-
-    var is_interactive_val = false;
-    var is_disabled = false;
-    var tag_name: []const u8 = "text";
-    var value: ?[]const u8 = null;
-    var input_type: ?[]const u8 = null;
-    var placeholder: ?[]const u8 = null;
-    var href: ?[]const u8 = null;
-    var id_attr: ?[]const u8 = null;
-    var class_attr: ?[]const u8 = null;
-    var checked: ?bool = null;
-    var options: ?[]OptionData = null;
-
-    if (node.is(Element)) |el| {
-        tag_name = el.getTagNameLower();
-        is_disabled = el.isDisabled();
-        id_attr = el.getAttributeSafe(comptime lp.String.wrap("id"));
-        class_attr = el.getAttributeSafe(comptime lp.String.wrap("class"));
-        placeholder = el.getAttributeSafe(comptime lp.String.wrap("placeholder"));
-
-        if (el.getAttributeSafe(comptime lp.String.wrap("href"))) |h| {
-            const URL = lp.URL;
-            href = URL.resolve(arena, page.base(), h, .{ .encode = true }) catch h;
-        }
-
-        if (el.is(Element.Html.Input)) |input| {
-            value = input.getValue();
-            input_type = input._input_type.toString();
-            if (input._input_type == .checkbox or input._input_type == .radio) {
-                checked = input.getChecked();
-            }
-            if (el.getAttributeSafe(comptime lp.String.wrap("list"))) |list_id| {
-                options = try extractDataListOptions(list_id, page, arena);
-            }
-        } else if (el.is(Element.Html.TextArea)) |textarea| {
-            value = textarea.getValue();
-        } else if (el.is(Element.Html.Select)) |select| {
-            value = select.getValue(page);
-            options = try extractSelectOptions(el.asNode(), page, arena);
-        }
-
-        if (el.is(Element.Html)) |html_el| {
-            const listener_targets = try interactive.buildListenerTargetMap(page, arena);
-            var pointer_events_cache: Element.PointerEventsCache = .empty;
-            if (interactive.classifyInteractivity(page, el, html_el, listener_targets, &pointer_events_cache) != null) {
-                is_interactive_val = true;
-            }
-        }
-    }
-
-    return .{
-        .backendNodeId = cdp_node.id,
-        .tag_name = tag_name,
-        .role = role,
-        .name = name,
-        .is_interactive = is_interactive_val,
-        .is_disabled = is_disabled,
-        .value = value,
-        .input_type = input_type,
-        .placeholder = placeholder,
-        .href = href,
-        .id = id_attr,
-        .class = class_attr,
-        .checked = checked,
-        .options = options,
-    };
-}
-
 const testing = @import("testing.zig");
 
 test "SemanticTree backendDOMNodeId" {

src/browser/HttpClient.zig

@@ -1261,15 +1261,21 @@ pub const Transfer = struct {
 
     fn detectAuthChallenge(transfer: *Transfer, conn: *const http.Connection) void {
         const status = conn.getResponseCode() catch return;
-        if (status != 401 and status != 407) {
+        const connect_status = conn.getConnectCode() catch return;
+
+        if (status != 401 and status != 407 and connect_status != 401 and connect_status != 407) {
             transfer._auth_challenge = null;
             return;
         }
 
         if (conn.getResponseHeader("WWW-Authenticate", 0)) |hdr| {
             transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null;
+        } else if (conn.getConnectHeader("WWW-Authenticate", 0)) |hdr| {
+            transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null;
         } else if (conn.getResponseHeader("Proxy-Authenticate", 0)) |hdr| {
             transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null;
+        } else if (conn.getConnectHeader("Proxy-Authenticate", 0)) |hdr| {
+            transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null;
         } else {
             transfer._auth_challenge = .{ .status = status, .source = null, .scheme = null, .realm = null };
         }

src/cdp/domains/lp.zig

@@ -30,7 +30,6 @@ pub fn processMessage(cmd: anytype) !void {
         getMarkdown,
         getSemanticTree,
         getInteractiveElements,
-        getNodeDetails,
         getStructuredData,
         detectForms,
         clickNode,
@@ -43,7 +42,6 @@ pub fn processMessage(cmd: anytype) !void {
         .getMarkdown => return getMarkdown(cmd),
         .getSemanticTree => return getSemanticTree(cmd),
         .getInteractiveElements => return getInteractiveElements(cmd),
-        .getNodeDetails => return getNodeDetails(cmd),
         .getStructuredData => return getStructuredData(cmd),
         .detectForms => return detectForms(cmd),
         .clickNode => return clickNode(cmd),
@@ -143,24 +141,6 @@ fn getInteractiveElements(cmd: anytype) !void {
     }, .{});
 }
 
-fn getNodeDetails(cmd: anytype) !void {
-    const Params = struct {
-        backendNodeId: Node.Id,
-    };
-    const params = (try cmd.params(Params)) orelse return error.InvalidParam;
-
-    const bc = cmd.browser_context orelse return error.NoBrowserContext;
-    const page = bc.session.currentPage() orelse return error.PageNotLoaded;
-
-    const node = (bc.node_registry.lookup_by_id.get(params.backendNodeId) orelse return error.InvalidNodeId).dom;
-
-    const details = SemanticTree.getNodeDetails(node, &bc.node_registry, page, cmd.arena) catch return error.InternalError;
-
-    return cmd.sendResult(.{
-        .nodeDetails = details,
-    }, .{});
-}
-
 fn getStructuredData(cmd: anytype) !void {
     const bc = cmd.browser_context orelse return error.NoBrowserContext;
     const page = bc.session.currentPage() orelse return error.PageNotLoaded;

src/mcp/tools.zig

@@ -75,19 +75,6 @@ pub const tool_list = [_]protocol.Tool{
             \\}
         ),
     },
-    .{
-        .name = "nodeDetails",
-        .description = "Get detailed information about a specific node by its backend node ID. Returns tag, role, name, interactivity, disabled state, value, input type, placeholder, href, checked state, and select options.",
-        .inputSchema = protocol.minify(
-            \\{
-            \\  "type": "object",
-            \\  "properties": {
-            \\    "backendNodeId": { "type": "integer", "description": "The backend node ID of the element to inspect." }
-            \\  },
-            \\  "required": ["backendNodeId"]
-            \\}
-        ),
-    },
     .{
         .name = "interactiveElements",
         .description = "Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.",
@@ -269,7 +256,6 @@ const ToolAction = enum {
     navigate,
     markdown,
     links,
-    nodeDetails,
     interactiveElements,
     structuredData,
     detectForms,
@@ -286,7 +272,6 @@ const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
     .{ "navigate", .navigate },
     .{ "markdown", .markdown },
     .{ "links", .links },
-    .{ "nodeDetails", .nodeDetails },
     .{ "interactiveElements", .interactiveElements },
     .{ "structuredData", .structuredData },
     .{ "detectForms", .detectForms },
@@ -320,7 +305,6 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
         .goto, .navigate => try handleGoto(server, arena, req.id.?, call_params.arguments),
         .markdown => try handleMarkdown(server, arena, req.id.?, call_params.arguments),
         .links => try handleLinks(server, arena, req.id.?, call_params.arguments),
-        .nodeDetails => try handleNodeDetails(server, arena, req.id.?, call_params.arguments),
         .interactiveElements => try handleInteractiveElements(server, arena, req.id.?, call_params.arguments),
         .structuredData => try handleStructuredData(server, arena, req.id.?, call_params.arguments),
         .detectForms => try handleDetectForms(server, arena, req.id.?, call_params.arguments),
@@ -389,32 +373,6 @@ fn handleSemanticTree(server: *Server, arena: std.mem.Allocator, id: std.json.Va
     };
 }
 
-fn handleNodeDetails(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
-    const Params = struct {
-        backendNodeId: CDPNode.Id,
-    };
-    const args = try parseArgs(Params, arena, arguments, server, id, "nodeDetails");
-
-    _ = server.session.currentPage() orelse {
-        return server.sendError(id, .PageNotLoaded, "Page not loaded");
-    };
-
-    const node = server.node_registry.lookup_by_id.get(args.backendNodeId) orelse {
-        return server.sendError(id, .InvalidParams, "Node not found");
-    };
-
-    const page = server.session.currentPage().?;
-    const details = lp.SemanticTree.getNodeDetails(node.dom, &server.node_registry, page, arena) catch {
-        return server.sendError(id, .InternalError, "Failed to get node details");
-    };
-
-    var aw: std.Io.Writer.Allocating = .init(arena);
-    try std.json.Stringify.value(&details, .{}, &aw.writer);
-
-    const content = [_]protocol.TextContent([]const u8){.{ .text = aw.written() }};
-    try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
-}
-
 fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
     const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
     const page = try ensurePage(server, id, args.url);

src/network/http.zig

@@ -389,6 +389,15 @@ pub const Connection = struct {
         return url;
     }
 
+    pub fn getConnectCode(self: *const Connection) !u16 {
+        var status: c_long = undefined;
+        try libcurl.curl_easy_getinfo(self._easy, .connect_code, &status);
+        if (status < 0 or status > std.math.maxInt(u16)) {
+            return 0;
+        }
+        return @intCast(status);
+    }
+
     pub fn getResponseCode(self: *const Connection) !u16 {
         var status: c_long = undefined;
         try libcurl.curl_easy_getinfo(self._easy, .response_code, &status);
@@ -404,6 +413,24 @@ pub const Connection = struct {
         return @intCast(count);
     }
 
+    pub fn getConnectHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
+        var hdr: ?*libcurl.CurlHeader = null;
+        libcurl.curl_easy_header(self._easy, name, index, .connect, -1, &hdr) catch |err| {
+            // ErrorHeader includes OutOfMemory — rare but real errors from curl internals.
+            // Logged and returned as null since callers don't expect errors.
+            log.err(.http, "get response header", .{
+                .name = name,
+                .err = err,
+            });
+            return null;
+        };
+        const h = hdr orelse return null;
+        return .{
+            .amount = h.amount,
+            .value = std.mem.span(h.value),
+        };
+    }
+
     pub fn getResponseHeader(self: *const Connection, name: [:0]const u8, index: usize) ?HeaderValue {
         var hdr: ?*libcurl.CurlHeader = null;
         libcurl.curl_easy_header(self._easy, name, index, .header, -1, &hdr) catch |err| {

src/sys/libcurl.zig

@@ -178,6 +178,7 @@ pub const CurlInfo = enum(c.CURLINFO) {
     private = c.CURLINFO_PRIVATE,
     redirect_count = c.CURLINFO_REDIRECT_COUNT,
     response_code = c.CURLINFO_RESPONSE_CODE,
+    connect_code = c.CURLINFO_HTTP_CONNECTCODE,
 };
 
 pub const Error = error{
@@ -662,6 +663,7 @@ pub fn curl_easy_getinfo(easy: *Curl, comptime info: CurlInfo, out: anytype) Err
             break :blk c.curl_easy_getinfo(easy, inf, p);
         },
         .response_code,
+        .connect_code,
         .redirect_count,
         => blk: {
             const p: *c_long = out;