From a417c73bf79a6ebd9b1a08c62ae3938715295e3a Mon Sep 17 00:00:00 2001 From: egrs Date: Mon, 9 Mar 2026 19:11:09 +0100 Subject: [PATCH 1/2] add LP.getInteractiveElements CDP command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returns a structured list of all interactive elements on a page: buttons, links, inputs, ARIA widgets, contenteditable regions, and elements with event listeners. Includes accessible names, roles, listener types, and key attributes. Event listener introspection (both addEventListener and inline handlers) is unique to LP — no other browser exposes this to automation code. --- src/browser/interactive.zig | 481 ++++++++++++++++++++++++++++++++++++ src/cdp/domains/lp.zig | 49 ++++ src/lightpanda.zig | 1 + 3 files changed, 531 insertions(+) create mode 100644 src/browser/interactive.zig diff --git a/src/browser/interactive.zig b/src/browser/interactive.zig new file mode 100644 index 00000000..cf623f6c --- /dev/null +++ b/src/browser/interactive.zig @@ -0,0 +1,481 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); + +const Page = @import("Page.zig"); +const TreeWalker = @import("webapi/TreeWalker.zig"); +const Element = @import("webapi/Element.zig"); +const Node = @import("webapi/Node.zig"); +const EventTarget = @import("webapi/EventTarget.zig"); + +const Allocator = std.mem.Allocator; + +pub const InteractivityType = enum { + native, + aria, + contenteditable, + listener, + focusable, +}; + +pub const InteractiveElement = struct { + node: *Node, + tag_name: []const u8, + role: ?[]const u8, + name: ?[]const u8, + interactivity_type: InteractivityType, + listener_types: []const []const u8, + disabled: bool, + tab_index: i32, + id: ?[]const u8, + class: ?[]const u8, + href: ?[]const u8, + input_type: ?[]const u8, + value: ?[]const u8, + element_name: ?[]const u8, + placeholder: ?[]const u8, + + pub fn jsonStringify(self: *const InteractiveElement, jw: anytype) !void { + try jw.beginObject(); + + try jw.objectField("tagName"); + try jw.write(self.tag_name); + + try jw.objectField("role"); + try jw.write(self.role); + + try jw.objectField("name"); + try jw.write(self.name); + + try jw.objectField("type"); + try jw.write(@tagName(self.interactivity_type)); + + if (self.listener_types.len > 0) { + try jw.objectField("listeners"); + try jw.beginArray(); + for (self.listener_types) |lt| { + try jw.write(lt); + } + try jw.endArray(); + } + + if (self.disabled) { + try jw.objectField("disabled"); + try jw.write(true); + } + + try jw.objectField("tabIndex"); + try jw.write(self.tab_index); + + if (self.id) |v| { + try jw.objectField("id"); + try jw.write(v); + } + + if (self.class) |v| { + try jw.objectField("class"); + try jw.write(v); + } + + if (self.href) |v| { + try jw.objectField("href"); + try jw.write(v); + } + + if (self.input_type) |v| { + try jw.objectField("inputType"); + try jw.write(v); + } + + if (self.value) |v| { + try jw.objectField("value"); + try jw.write(v); + } + + if (self.element_name) |v| { + try jw.objectField("elementName"); + try jw.write(v); + } + + if (self.placeholder) |v| { + try jw.objectField("placeholder"); + try jw.write(v); + } + + try jw.endObject(); + } +}; + +/// Collect all interactive elements under `root`. +pub fn collectInteractiveElements( + root: *Node, + page: *Page, + arena: Allocator, +) ![]InteractiveElement { + // Pre-build a map of event_target pointer → event type names, + // so classify and getListenerTypes are both O(1) per element. + const listener_targets = try buildListenerTargetMap(page, arena); + + var results: std.ArrayList(InteractiveElement) = .empty; + + var tw = TreeWalker.FullExcludeSelf.init(root, .{}); + while (tw.next()) |node| { + const el = node.is(Element) orelse continue; + const html_el = el.is(Element.Html) orelse continue; + + // Skip non-visual elements that are never user-interactive. + switch (el.getTag()) { + .script, .style, .link, .meta, .head, .noscript, .template => continue, + else => {}, + } + + const itype = classifyInteractivity(el, html_el, listener_targets) orelse continue; + + const listener_types = getListenerTypes( + el.asEventTarget(), + listener_targets, + ); + + try results.append(arena, .{ + .node = node, + .tag_name = el.getTagNameLower(), + .role = getRole(el), + .name = getAccessibleName(el), + .interactivity_type = itype, + .listener_types = listener_types, + .disabled = isDisabled(el), + .tab_index = html_el.getTabIndex(), + .id = el.getAttributeSafe(comptime .wrap("id")), + .class = el.getAttributeSafe(comptime .wrap("class")), + .href = el.getAttributeSafe(comptime .wrap("href")), + .input_type = getInputType(el), + .value = getInputValue(el), + .element_name = el.getAttributeSafe(comptime .wrap("name")), + .placeholder = el.getAttributeSafe(comptime .wrap("placeholder")), + }); + } + + return results.items; +} + +const ListenerTargetMap = std.AutoHashMapUnmanaged(usize, std.ArrayList([]const u8)); + +/// Pre-build a map from event_target pointer → list of event type names. +/// This lets both classifyInteractivity (O(1) "has any?") and +/// getListenerTypes (O(1) "which ones?") avoid re-iterating per element. +fn buildListenerTargetMap(page: *Page, arena: Allocator) !ListenerTargetMap { + var map = ListenerTargetMap{}; + + // addEventListener registrations + var it = page._event_manager.lookup.iterator(); + while (it.next()) |entry| { + const list = entry.value_ptr.*; + if (list.first != null) { + const gop = try map.getOrPut(arena, entry.key_ptr.event_target); + if (!gop.found_existing) gop.value_ptr.* = .empty; + try gop.value_ptr.append(arena, entry.key_ptr.type_string.str()); + } + } + + // Inline handlers (onclick, onmousedown, etc.) + var attr_it = page._event_target_attr_listeners.iterator(); + while (attr_it.next()) |entry| { + const gop = try map.getOrPut(arena, @intFromPtr(entry.key_ptr.target)); + if (!gop.found_existing) gop.value_ptr.* = .empty; + // Strip "on" prefix to get the event type name. + try gop.value_ptr.append(arena, @tagName(entry.key_ptr.handler)[2..]); + } + + return map; +} + +fn classifyInteractivity( + el: *Element, + html_el: *Element.Html, + listener_targets: ListenerTargetMap, +) ?InteractivityType { + // 1. Native interactive by tag + switch (el.getTag()) { + .button, .summary, .details, .select, .textarea => return .native, + .anchor, .area => { + if (el.getAttributeSafe(comptime .wrap("href")) != null) return .native; + }, + .input => { + if (el.is(Element.Html.Input)) |input| { + if (input._input_type != .hidden) return .native; + } + }, + else => {}, + } + + // 2. ARIA interactive role + if (el.getAttributeSafe(comptime .wrap("role"))) |role| { + if (isInteractiveRole(role)) return .aria; + } + + // 3. contenteditable (15 bytes, exceeds SSO limit for comptime) + if (el.getAttributeSafe(.wrap("contenteditable"))) |ce| { + if (ce.len == 0 or std.ascii.eqlIgnoreCase(ce, "true")) return .contenteditable; + } + + // 4. Event listeners (addEventListener or inline handlers) + const et_ptr = @intFromPtr(html_el.asEventTarget()); + if (listener_targets.get(et_ptr) != null) return .listener; + + // 5. Explicitly focusable via tabindex. + // Only count elements with an EXPLICIT tabindex attribute, + // since getTabIndex() returns 0 for all interactive tags by default + // (including anchors without href and hidden inputs). + if (el.getAttributeSafe(comptime .wrap("tabindex"))) |_| { + if (html_el.getTabIndex() >= 0) return .focusable; + } + + return null; +} + +fn isInteractiveRole(role: []const u8) bool { + const interactive_roles = [_][]const u8{ + "button", "link", "tab", "menuitem", + "menuitemcheckbox", "menuitemradio", "switch", "checkbox", + "radio", "slider", "spinbutton", "searchbox", + "combobox", "option", "treeitem", + }; + for (interactive_roles) |r| { + if (std.ascii.eqlIgnoreCase(role, r)) return true; + } + return false; +} + +fn getRole(el: *Element) ?[]const u8 { + // Explicit role attribute takes precedence + if (el.getAttributeSafe(comptime .wrap("role"))) |role| return role; + + // Implicit role from tag + return switch (el.getTag()) { + .button, .summary => "button", + .anchor, .area => if (el.getAttributeSafe(comptime .wrap("href")) != null) "link" else null, + .input => blk: { + if (el.is(Element.Html.Input)) |input| { + break :blk switch (input._input_type) { + .text, .tel, .url, .email => "textbox", + .checkbox => "checkbox", + .radio => "radio", + .button, .submit, .reset, .image => "button", + .range => "slider", + .number => "spinbutton", + .search => "searchbox", + else => null, + }; + } + break :blk null; + }, + .select => "combobox", + .textarea => "textbox", + .details => "group", + else => null, + }; +} + +fn getAccessibleName(el: *Element) ?[]const u8 { + // aria-label + if (el.getAttributeSafe(comptime .wrap("aria-label"))) |v| { + if (v.len > 0) return v; + } + + // alt (for img, input[type=image]) + if (el.getAttributeSafe(comptime .wrap("alt"))) |v| { + if (v.len > 0) return v; + } + + // title + if (el.getAttributeSafe(comptime .wrap("title"))) |v| { + if (v.len > 0) return v; + } + + // placeholder + if (el.getAttributeSafe(comptime .wrap("placeholder"))) |v| { + if (v.len > 0) return v; + } + + // value (for buttons) + if (el.getTag() == .input) { + if (el.getAttributeSafe(comptime .wrap("value"))) |v| { + if (v.len > 0) return v; + } + } + + // Text content (first non-empty text node, trimmed) + return getTextContent(el.asNode()); +} + +fn getTextContent(node: *Node) ?[]const u8 { + var tw = TreeWalker.FullExcludeSelf.init(node, .{}); + while (tw.next()) |child| { + // Skip text inside script/style elements. + if (child.is(Element)) |el| { + switch (el.getTag()) { + .script, .style => { + tw.skipChildren(); + continue; + }, + else => {}, + } + } + if (child.is(Node.CData)) |cdata| { + if (cdata.is(Node.CData.Text)) |text| { + const content = std.mem.trim(u8, text.getWholeText(), &std.ascii.whitespace); + if (content.len > 0) return content; + } + } + } + return null; +} + +fn isDisabled(el: *Element) bool { + return el.getAttributeSafe(comptime .wrap("disabled")) != null; +} + +fn getInputType(el: *Element) ?[]const u8 { + if (el.is(Element.Html.Input)) |input| { + return input._input_type.toString(); + } + return null; +} + +fn getInputValue(el: *Element) ?[]const u8 { + if (el.is(Element.Html.Input)) |input| { + return input.getValue(); + } + return null; +} + +/// Get all event listener types registered on this target. +fn getListenerTypes(target: *EventTarget, listener_targets: ListenerTargetMap) []const []const u8 { + if (listener_targets.get(@intFromPtr(target))) |types| return types.items; + return &.{}; +} + +const testing = @import("../testing.zig"); + +fn testInteractive(html: []const u8) ![]InteractiveElement { + const page = try testing.test_session.createPage(); + defer testing.test_session.removePage(); + + const doc = page.window._document; + const div = try doc.createElement("div", null, page); + try page.parseHtmlAsChildren(div.asNode(), html); + + return collectInteractiveElements(div.asNode(), page, page.call_arena); +} + +test "browser.interactive: button" { + const elements = try testInteractive(""); + try testing.expectEqual(1, elements.len); + try testing.expectEqual("button", elements[0].tag_name); + try testing.expectEqual("button", elements[0].role.?); + try testing.expectEqual("Click me", elements[0].name.?); + try testing.expectEqual(InteractivityType.native, elements[0].interactivity_type); +} + +test "browser.interactive: anchor with href" { + const elements = try testInteractive("Link"); + try testing.expectEqual(1, elements.len); + try testing.expectEqual("a", elements[0].tag_name); + try testing.expectEqual("link", elements[0].role.?); + try testing.expectEqual("Link", elements[0].name.?); +} + +test "browser.interactive: anchor without href" { + const elements = try testInteractive("Not a link"); + try testing.expectEqual(0, elements.len); +} + +test "browser.interactive: input types" { + const elements = try testInteractive( + \\ + \\ + ); + try testing.expectEqual(1, elements.len); + try testing.expectEqual("input", elements[0].tag_name); + try testing.expectEqual("text", elements[0].input_type.?); + try testing.expectEqual("Search", elements[0].placeholder.?); +} + +test "browser.interactive: select and textarea" { + const elements = try testInteractive( + \\ + \\ + ); + try testing.expectEqual(2, elements.len); + try testing.expectEqual("select", elements[0].tag_name); + try testing.expectEqual("textarea", elements[1].tag_name); +} + +test "browser.interactive: aria role" { + const elements = try testInteractive("
Custom
"); + try testing.expectEqual(1, elements.len); + try testing.expectEqual("div", elements[0].tag_name); + try testing.expectEqual("button", elements[0].role.?); + try testing.expectEqual(InteractivityType.aria, elements[0].interactivity_type); +} + +test "browser.interactive: contenteditable" { + const elements = try testInteractive("
Edit me
"); + try testing.expectEqual(1, elements.len); + try testing.expectEqual(InteractivityType.contenteditable, elements[0].interactivity_type); +} + +test "browser.interactive: tabindex" { + const elements = try testInteractive("
Focusable
"); + try testing.expectEqual(1, elements.len); + try testing.expectEqual(InteractivityType.focusable, elements[0].interactivity_type); + try testing.expectEqual(@as(i32, 0), elements[0].tab_index); +} + +test "browser.interactive: disabled" { + const elements = try testInteractive(""); + try testing.expectEqual(1, elements.len); + try testing.expect(elements[0].disabled); +} + +test "browser.interactive: non-interactive div" { + const elements = try testInteractive("
Just text
"); + try testing.expectEqual(0, elements.len); +} + +test "browser.interactive: details and summary" { + const elements = try testInteractive("
More

Content

"); + try testing.expectEqual(2, elements.len); + try testing.expectEqual("details", elements[0].tag_name); + try testing.expectEqual("summary", elements[1].tag_name); +} + +test "browser.interactive: mixed elements" { + const elements = try testInteractive( + \\
+ \\ Home + \\

Some text

+ \\ + \\ + \\
Not interactive
+ \\
Tab
+ \\
+ ); + try testing.expectEqual(4, elements.len); +} diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig index 5503c356..d5f5cff3 100644 --- a/src/cdp/domains/lp.zig +++ b/src/cdp/domains/lp.zig @@ -19,15 +19,18 @@ const std = @import("std"); const lp = @import("lightpanda"); const markdown = lp.markdown; +const interactive = lp.interactive; const Node = @import("../Node.zig"); pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { getMarkdown, + getInteractiveElements, }, cmd.input.action) orelse return error.UnknownMethod; switch (action) { .getMarkdown => return getMarkdown(cmd), + .getInteractiveElements => return getInteractiveElements(cmd), } } @@ -54,6 +57,35 @@ fn getMarkdown(cmd: anytype) !void { }, .{}); } +fn getInteractiveElements(cmd: anytype) !void { + const Params = struct { + nodeId: ?Node.Id = null, + }; + const params = (try cmd.params(Params)) orelse Params{}; + + const bc = cmd.browser_context orelse return error.NoBrowserContext; + const page = bc.session.currentPage() orelse return error.PageNotLoaded; + + const root = if (params.nodeId) |nodeId| + (bc.node_registry.lookup_by_id.get(nodeId) orelse return error.InvalidNodeId).dom + else + page.document.asNode(); + + const elements = try interactive.collectInteractiveElements(root, page, cmd.arena); + + // Register nodes so nodeIds are valid for subsequent CDP calls. + var node_ids: std.ArrayList(Node.Id) = try .initCapacity(cmd.arena, elements.len); + for (elements) |el| { + const registered = try bc.node_registry.register(el.node); + node_ids.appendAssumeCapacity(registered.id); + } + + return cmd.sendResult(.{ + .elements = elements, + .nodeIds = node_ids.items, + }, .{}); +} + const testing = @import("../testing.zig"); test "cdp.lp: getMarkdown" { var ctx = testing.context(); @@ -70,3 +102,20 @@ test "cdp.lp: getMarkdown" { const result = ctx.client.?.sent.items[0].object.get("result").?.object; try testing.expect(result.get("markdown") != null); } + +test "cdp.lp: getInteractiveElements" { + var ctx = testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{}); + _ = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "LP.getInteractiveElements", + }); + + const result = ctx.client.?.sent.items[0].object.get("result").?.object; + try testing.expect(result.get("elements") != null); + try testing.expect(result.get("nodeIds") != null); +} diff --git a/src/lightpanda.zig b/src/lightpanda.zig index 26bc23f0..34deb146 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -30,6 +30,7 @@ pub const log = @import("log.zig"); pub const js = @import("browser/js/js.zig"); pub const dump = @import("browser/dump.zig"); pub const markdown = @import("browser/markdown.zig"); +pub const interactive = @import("browser/interactive.zig"); pub const mcp = @import("mcp.zig"); pub const build_config = @import("build_config"); pub const crash_handler = @import("crash_handler.zig"); From dc3958356d9f3da6d887e3216d5db256bc28bd9a Mon Sep 17 00:00:00 2001 From: egrs Date: Tue, 10 Mar 2026 08:13:01 +0100 Subject: [PATCH 2/2] address review feedback - TreeWalker.Full instead of FullExcludeSelf so querying a specific nodeId evaluates the root element itself - resolve href to absolute URL via URL.resolve - isDisabled checks ancestor
with legend exemption - parameter order: allocator before *Page per convention --- src/browser/interactive.zig | 55 +++++++++++++++++++++++++++++++++---- src/cdp/domains/lp.zig | 2 +- 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/browser/interactive.zig b/src/browser/interactive.zig index cf623f6c..b0428a6c 100644 --- a/src/browser/interactive.zig +++ b/src/browser/interactive.zig @@ -19,6 +19,7 @@ const std = @import("std"); const Page = @import("Page.zig"); +const URL = @import("URL.zig"); const TreeWalker = @import("webapi/TreeWalker.zig"); const Element = @import("webapi/Element.zig"); const Node = @import("webapi/Node.zig"); @@ -125,8 +126,8 @@ pub const InteractiveElement = struct { /// Collect all interactive elements under `root`. pub fn collectInteractiveElements( root: *Node, - page: *Page, arena: Allocator, + page: *Page, ) ![]InteractiveElement { // Pre-build a map of event_target pointer → event type names, // so classify and getListenerTypes are both O(1) per element. @@ -134,7 +135,7 @@ pub fn collectInteractiveElements( var results: std.ArrayList(InteractiveElement) = .empty; - var tw = TreeWalker.FullExcludeSelf.init(root, .{}); + var tw = TreeWalker.Full.init(root, .{}); while (tw.next()) |node| { const el = node.is(Element) orelse continue; const html_el = el.is(Element.Html) orelse continue; @@ -163,7 +164,10 @@ pub fn collectInteractiveElements( .tab_index = html_el.getTabIndex(), .id = el.getAttributeSafe(comptime .wrap("id")), .class = el.getAttributeSafe(comptime .wrap("class")), - .href = el.getAttributeSafe(comptime .wrap("href")), + .href = if (el.getAttributeSafe(comptime .wrap("href"))) |href| + URL.resolve(arena, page.base(), href, .{ .encode = true }) catch href + else + null, .input_type = getInputType(el), .value = getInputValue(el), .element_name = el.getAttributeSafe(comptime .wrap("name")), @@ -348,7 +352,34 @@ fn getTextContent(node: *Node) ?[]const u8 { } fn isDisabled(el: *Element) bool { - return el.getAttributeSafe(comptime .wrap("disabled")) != null; + if (el.getAttributeSafe(comptime .wrap("disabled")) != null) return true; + return isDisabledByFieldset(el); +} + +/// Check if an element is disabled by an ancestor
. +/// Per spec, elements inside the first child of a disabled fieldset +/// are NOT disabled by that fieldset. +fn isDisabledByFieldset(el: *Element) bool { + const element_node = el.asNode(); + var current: ?*Node = element_node._parent; + while (current) |node| { + current = node._parent; + const ancestor = node.is(Element) orelse continue; + + if (ancestor.getTag() == .fieldset and ancestor.getAttributeSafe(comptime .wrap("disabled")) != null) { + // Check if element is inside the first child of this fieldset + var child = ancestor.firstElementChild(); + while (child) |c| { + if (c.getTag() == .legend) { + if (c.asNode().contains(element_node)) return false; + break; + } + child = c.nextElementSibling(); + } + return true; + } + } + return false; } fn getInputType(el: *Element) ?[]const u8 { @@ -381,7 +412,7 @@ fn testInteractive(html: []const u8) ![]InteractiveElement { const div = try doc.createElement("div", null, page); try page.parseHtmlAsChildren(div.asNode(), html); - return collectInteractiveElements(div.asNode(), page, page.call_arena); + return collectInteractiveElements(div.asNode(), page.call_arena, page); } test "browser.interactive: button" { @@ -454,6 +485,20 @@ test "browser.interactive: disabled" { try testing.expect(elements[0].disabled); } +test "browser.interactive: disabled by fieldset" { + const elements = try testInteractive( + \\
+ \\ + \\ + \\
+ ); + try testing.expectEqual(2, elements.len); + // Button outside legend is disabled by fieldset + try testing.expect(elements[0].disabled); + // Button inside first legend is NOT disabled + try testing.expect(!elements[1].disabled); +} + test "browser.interactive: non-interactive div" { const elements = try testInteractive("
Just text
"); try testing.expectEqual(0, elements.len); diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig index d5f5cff3..8263322b 100644 --- a/src/cdp/domains/lp.zig +++ b/src/cdp/domains/lp.zig @@ -71,7 +71,7 @@ fn getInteractiveElements(cmd: anytype) !void { else page.document.asNode(); - const elements = try interactive.collectInteractiveElements(root, page, cmd.arena); + const elements = try interactive.collectInteractiveElements(root, cmd.arena, page); // Register nodes so nodeIds are valid for subsequent CDP calls. var node_ids: std.ArrayList(Node.Id) = try .initCapacity(cmd.arena, elements.len);