mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-04-01 18:06:46 +00:00
Compare commits
3 Commits
percent-en
...
fix/agent-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1854627b69 | ||
|
|
fffa8b6d4b | ||
|
|
58fc60d669 |
15
.github/workflows/e2e-integration-test.yml
vendored
15
.github/workflows/e2e-integration-test.yml
vendored
@@ -60,20 +60,7 @@ jobs:
|
||||
- run: chmod a+x ./lightpanda
|
||||
|
||||
- name: run end to end integration tests
|
||||
continue-on-error: true
|
||||
run: |
|
||||
./lightpanda serve --log-level error & echo $! > LPD.pid
|
||||
go run integration/main.go |tee result.log
|
||||
go run integration/main.go
|
||||
kill `cat LPD.pid`
|
||||
|
||||
- name: Send result to slack
|
||||
uses: slackapi/slack-github-action@v3.0.1
|
||||
with:
|
||||
errors: true
|
||||
method: files.uploadV2
|
||||
token: ${{ secrets.CI_SLACK_BOT_TOKEN }}
|
||||
payload: |
|
||||
channel_id: ${{ vars.E2E_SLACK_CHANNEL_ID }}
|
||||
initial_comment: "Last e2e integration tests"
|
||||
file: "./result.log"
|
||||
filename: "e2e-integration-${{ github.sha }}.txt"
|
||||
|
||||
31
.github/workflows/wpt.yml
vendored
31
.github/workflows/wpt.yml
vendored
@@ -153,34 +153,3 @@ jobs:
|
||||
|
||||
- name: format and send json result
|
||||
run: /perf-fmt wpt ${{ github.sha }} wpt.json
|
||||
|
||||
wptdiff:
|
||||
name: perf-fmt
|
||||
needs: perf-fmt
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
repository: 'lightpanda-io/demo'
|
||||
fetch-depth: 0
|
||||
|
||||
- run: |
|
||||
cd ./wptdiff
|
||||
CGO_ENABLED=0 go build
|
||||
|
||||
- run: |
|
||||
./wptdiff/wptdiff |tee diff.log
|
||||
|
||||
- name: Send regression to slack
|
||||
uses: slackapi/slack-github-action@v3.0.1
|
||||
with:
|
||||
errors: true
|
||||
method: files.uploadV2
|
||||
token: ${{ secrets.CI_SLACK_BOT_TOKEN }}
|
||||
payload: |
|
||||
channel_id: ${{ vars.WPT_SLACK_CHANNEL_ID }}
|
||||
initial_comment: "Last WPT regressions"
|
||||
file: "./diff.log"
|
||||
filename: "wpt-regression-${{ github.sha }}.txt"
|
||||
|
||||
@@ -46,12 +46,8 @@ pub fn build(b: *Build) !void {
|
||||
var stdout = std.fs.File.stdout().writer(&.{});
|
||||
try stdout.interface.print("Lightpanda {f}\n", .{version});
|
||||
|
||||
const version_string = b.fmt("{f}", .{version});
|
||||
const version_encoded = std.mem.replaceOwned(u8, b.allocator, version_string, "+", "%2B") catch @panic("OOM");
|
||||
|
||||
var opts = b.addOptions();
|
||||
opts.addOption([]const u8, "version", version_string);
|
||||
opts.addOption([]const u8, "version_encoded", version_encoded);
|
||||
opts.addOption([]const u8, "version", b.fmt("{f}", .{version}));
|
||||
opts.addOption(?[]const u8, "snapshot_path", snapshot_path);
|
||||
|
||||
const enable_tsan = b.option(bool, "tsan", "Enable Thread Sanitizer") orelse false;
|
||||
|
||||
@@ -26,7 +26,7 @@ const Snapshot = @import("browser/js/Snapshot.zig");
|
||||
const Platform = @import("browser/js/Platform.zig");
|
||||
const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
|
||||
|
||||
const Network = @import("network/Network.zig");
|
||||
const Network = @import("network/Runtime.zig");
|
||||
pub const ArenaPool = @import("ArenaPool.zig");
|
||||
|
||||
const App = @This();
|
||||
|
||||
@@ -34,6 +34,7 @@ pub const RunMode = enum {
|
||||
mcp,
|
||||
};
|
||||
|
||||
pub const MAX_LISTENERS = 16;
|
||||
pub const CDP_MAX_HTTP_REQUEST_SIZE = 4096;
|
||||
|
||||
// max message size
|
||||
@@ -159,6 +160,7 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
|
||||
pub fn cdpTimeout(self: *const Config) usize {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| if (opts.timeout > 604_800) 604_800_000 else @as(usize, opts.timeout) * 1000,
|
||||
.mcp => 10000, // Default timeout for MCP-CDP
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -166,6 +168,7 @@ pub fn cdpTimeout(self: *const Config) usize {
|
||||
pub fn port(self: *const Config) u16 {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| opts.port,
|
||||
.mcp => |opts| opts.cdp_port orelse 0,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -173,6 +176,7 @@ pub fn port(self: *const Config) u16 {
|
||||
pub fn advertiseHost(self: *const Config) []const u8 {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| opts.advertise_host orelse opts.host,
|
||||
.mcp => "127.0.0.1",
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -191,6 +195,7 @@ pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
|
||||
pub fn maxConnections(self: *const Config) u16 {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| opts.cdp_max_connections,
|
||||
.mcp => 16,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -198,6 +203,7 @@ pub fn maxConnections(self: *const Config) u16 {
|
||||
pub fn maxPendingConnections(self: *const Config) u31 {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| opts.cdp_max_pending_connections,
|
||||
.mcp => 128,
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
@@ -223,6 +229,7 @@ pub const Serve = struct {
|
||||
pub const Mcp = struct {
|
||||
common: Common = .{},
|
||||
version: mcp.Version = .default,
|
||||
cdp_port: ?u16 = null,
|
||||
};
|
||||
|
||||
pub const DumpFormat = enum {
|
||||
@@ -676,6 +683,19 @@ fn parseMcpArgs(
|
||||
continue;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--cdp-port", opt) or std.mem.eql(u8, "--cdp_port", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.mcp, "missing argument value", .{ .arg = opt });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
|
||||
result.cdp_port = std.fmt.parseInt(u16, str, 10) catch |err| {
|
||||
log.fatal(.mcp, "invalid argument value", .{ .arg = opt, .err = err });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
continue;
|
||||
}
|
||||
|
||||
if (try parseCommonArg(allocator, opt, args, &result.common)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -330,6 +330,11 @@ pub const Client = struct {
|
||||
ms_remaining = self.ws.timeout_ms;
|
||||
},
|
||||
.done => {
|
||||
if (self.isBusy()) {
|
||||
last_message = milliTimestamp(.monotonic);
|
||||
ms_remaining = self.ws.timeout_ms;
|
||||
continue;
|
||||
}
|
||||
const now = milliTimestamp(.monotonic);
|
||||
const elapsed = now - last_message;
|
||||
if (elapsed >= ms_remaining) {
|
||||
@@ -343,6 +348,24 @@ pub const Client = struct {
|
||||
}
|
||||
}
|
||||
|
||||
fn isBusy(self: *const Client) bool {
|
||||
if (self.http.active > 0 or self.http.intercepted > 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const cdp = switch (self.mode) {
|
||||
.cdp => |*c| c,
|
||||
.http => return false,
|
||||
};
|
||||
|
||||
const session = cdp.browser.session orelse return false;
|
||||
if (session.browser.hasBackgroundTasks() or session.browser.msToNextMacrotask() != null) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
fn blockingReadStart(ctx: *anyopaque) bool {
|
||||
const self: *Client = @ptrCast(@alignCast(ctx));
|
||||
self.ws.setBlocking(true) catch |err| {
|
||||
|
||||
@@ -30,7 +30,7 @@ const Notification = @import("../Notification.zig");
|
||||
const CookieJar = @import("webapi/storage/Cookie.zig").Jar;
|
||||
|
||||
const http = @import("../network/http.zig");
|
||||
const Network = @import("../network/Network.zig");
|
||||
const Runtime = @import("../network/Runtime.zig");
|
||||
const Robots = @import("../network/Robots.zig");
|
||||
|
||||
const IS_DEBUG = builtin.mode == .Debug;
|
||||
@@ -86,7 +86,7 @@ queue: std.DoublyLinkedList = .{},
|
||||
// The main app allocator
|
||||
allocator: Allocator,
|
||||
|
||||
network: *Network,
|
||||
network: *Runtime,
|
||||
|
||||
// Queue of requests that depend on a robots.txt.
|
||||
// Allows us to fetch the robots.txt just once.
|
||||
@@ -131,7 +131,7 @@ pub const CDPClient = struct {
|
||||
blocking_read_end: *const fn (*anyopaque) bool,
|
||||
};
|
||||
|
||||
pub fn init(allocator: Allocator, network: *Network) !*Client {
|
||||
pub fn init(allocator: Allocator, network: *Runtime) !*Client {
|
||||
var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
|
||||
errdefer transfer_pool.deinit();
|
||||
|
||||
@@ -695,7 +695,7 @@ fn perform(self: *Client, timeout_ms: c_int) anyerror!PerformStatus {
|
||||
break :blk try self.handles.perform();
|
||||
};
|
||||
|
||||
// Process dirty connections — return them to Network pool.
|
||||
// Process dirty connections — return them to Runtime pool.
|
||||
while (self.dirty.popFirst()) |node| {
|
||||
const conn: *http.Connection = @fieldParentPtr("node", node);
|
||||
self.handles.remove(conn) catch |err| {
|
||||
|
||||
@@ -22,7 +22,7 @@ const builtin = @import("builtin");
|
||||
|
||||
const log = @import("../log.zig");
|
||||
const HttpClient = @import("HttpClient.zig");
|
||||
const http = @import("../network/http.zig");
|
||||
const net_http = @import("../network/http.zig");
|
||||
const String = @import("../string.zig").String;
|
||||
|
||||
const js = @import("js/js.zig");
|
||||
@@ -136,7 +136,7 @@ fn clearList(list: *std.DoublyLinkedList) void {
|
||||
}
|
||||
}
|
||||
|
||||
fn getHeaders(self: *ScriptManager) !http.Headers {
|
||||
fn getHeaders(self: *ScriptManager) !net_http.Headers {
|
||||
var headers = try self.client.newHeaders();
|
||||
try self.page.headersForRequest(&headers);
|
||||
return headers;
|
||||
|
||||
@@ -25,72 +25,28 @@ const ResolveOpts = struct {
|
||||
};
|
||||
|
||||
// path is anytype, so that it can be used with both []const u8 and [:0]const u8
|
||||
pub fn resolve(allocator: Allocator, base: [:0]const u8, source_path: anytype, comptime opts: ResolveOpts) ![:0]const u8 {
|
||||
const PT = @TypeOf(source_path);
|
||||
|
||||
var path: [:0]const u8 = if (comptime !isNullTerminated(PT) or opts.always_dupe) try allocator.dupeZ(u8, source_path) else source_path;
|
||||
|
||||
if (base.len == 0) {
|
||||
return processResolved(allocator, path, opts);
|
||||
}
|
||||
|
||||
// Minimum is "x:" and skip relative path (very common case)
|
||||
if (path.len >= 2 and path[0] != '/') {
|
||||
if (std.mem.indexOfScalar(u8, path[0..], ':')) |scheme_path_end| {
|
||||
scheme_check: {
|
||||
const scheme_path = path[0..scheme_path_end];
|
||||
//from "ws" to "https"
|
||||
if (scheme_path_end >= 2 and scheme_path_end <= 5) {
|
||||
const has_double_slashes: bool = scheme_path_end + 3 <= path.len and path[scheme_path_end + 1] == '/' and path[scheme_path_end + 2] == '/';
|
||||
const special_schemes = [_][]const u8{ "https", "http", "ws", "wss", "file", "ftp" };
|
||||
|
||||
for (special_schemes) |special_scheme| {
|
||||
if (std.ascii.eqlIgnoreCase(scheme_path, special_scheme)) {
|
||||
const base_scheme_end = std.mem.indexOf(u8, base, "://") orelse 0;
|
||||
|
||||
if (base_scheme_end > 0 and std.mem.eql(u8, base[0..base_scheme_end], scheme_path) and !has_double_slashes) {
|
||||
//Skip ":" and exit as relative state
|
||||
path = path[scheme_path_end + 1 ..];
|
||||
break :scheme_check;
|
||||
} else {
|
||||
var rest_start: usize = scheme_path_end + 1;
|
||||
//Skip any slashas after "scheme:"
|
||||
while (rest_start < path.len and (path[rest_start] == '/' or path[rest_start] == '\\')) {
|
||||
rest_start += 1;
|
||||
}
|
||||
// A special scheme (exclude "file") must contain at least any chars after "://"
|
||||
if (rest_start == path.len and !std.ascii.eqlIgnoreCase(scheme_path, "file")) {
|
||||
return error.TypeError;
|
||||
}
|
||||
//File scheme allow empty host
|
||||
const separator: []const u8 = if (!has_double_slashes and std.ascii.eqlIgnoreCase(scheme_path, "file")) ":///" else "://";
|
||||
|
||||
path = try std.mem.joinZ(allocator, "", &.{ scheme_path, separator, path[rest_start..] });
|
||||
return processResolved(allocator, path, opts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (scheme_path.len > 0) {
|
||||
for (scheme_path[1..]) |c| {
|
||||
if (!std.ascii.isAlphanumeric(c) and c != '+' and c != '-' and c != '.') {
|
||||
//Exit as relative state
|
||||
break :scheme_check;
|
||||
}
|
||||
}
|
||||
}
|
||||
//path is complete http url
|
||||
return processResolved(allocator, path, opts);
|
||||
}
|
||||
pub fn resolve(allocator: Allocator, base: [:0]const u8, path: anytype, comptime opts: ResolveOpts) ![:0]const u8 {
|
||||
const PT = @TypeOf(path);
|
||||
if (base.len == 0 or isCompleteHTTPUrl(path)) {
|
||||
if (comptime opts.always_dupe or !isNullTerminated(PT)) {
|
||||
const duped = try allocator.dupeZ(u8, path);
|
||||
return processResolved(allocator, duped, opts);
|
||||
}
|
||||
if (comptime opts.encode) {
|
||||
return processResolved(allocator, path, opts);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
if (path.len == 0) {
|
||||
if (opts.always_dupe) {
|
||||
const dupe = try allocator.dupeZ(u8, base);
|
||||
return processResolved(allocator, dupe, opts);
|
||||
if (comptime opts.always_dupe) {
|
||||
const duped = try allocator.dupeZ(u8, base);
|
||||
return processResolved(allocator, duped, opts);
|
||||
}
|
||||
return processResolved(allocator, base, opts);
|
||||
if (comptime opts.encode) {
|
||||
return processResolved(allocator, base, opts);
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
if (path[0] == '?') {
|
||||
@@ -107,7 +63,14 @@ pub fn resolve(allocator: Allocator, base: [:0]const u8, source_path: anytype, c
|
||||
if (std.mem.startsWith(u8, path, "//")) {
|
||||
// network-path reference
|
||||
const index = std.mem.indexOfScalar(u8, base, ':') orelse {
|
||||
return processResolved(allocator, path, opts);
|
||||
if (comptime isNullTerminated(PT)) {
|
||||
if (comptime opts.encode) {
|
||||
return processResolved(allocator, path, opts);
|
||||
}
|
||||
return path;
|
||||
}
|
||||
const duped = try allocator.dupeZ(u8, path);
|
||||
return processResolved(allocator, duped, opts);
|
||||
};
|
||||
const protocol = base[0 .. index + 1];
|
||||
const result = try std.mem.joinZ(allocator, "", &.{ protocol, path });
|
||||
@@ -133,7 +96,6 @@ pub fn resolve(allocator: Allocator, base: [:0]const u8, source_path: anytype, c
|
||||
// trailing space so that we always have space to append the null terminator
|
||||
// and so that we can compare the next two characters without needing to length check
|
||||
var out = try std.mem.join(allocator, "", &.{ normalized_base, "/", path, " " });
|
||||
|
||||
const end = out.len - 2;
|
||||
|
||||
const path_marker = path_start + 1;
|
||||
@@ -1608,182 +1570,3 @@ test "URL: getOrigin" {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test "URL: resolve path scheme" {
|
||||
const Case = struct {
|
||||
base: [:0]const u8,
|
||||
path: [:0]const u8,
|
||||
expected: [:0]const u8,
|
||||
expected_error: bool = false,
|
||||
};
|
||||
|
||||
const cases = [_]Case{
|
||||
//same schemes and path as relative path (one slash)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "https:/about",
|
||||
.expected = "https://www.example.com/about",
|
||||
},
|
||||
//same schemes and path as relative path (without slash)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "https:about",
|
||||
.expected = "https://www.example.com/about",
|
||||
},
|
||||
//same schemes and path as absolute path (two slashes)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "https://about",
|
||||
.expected = "https://about",
|
||||
},
|
||||
//different schemes and path as absolute (without slash)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "http:about",
|
||||
.expected = "http://about",
|
||||
},
|
||||
//different schemes and path as absolute (with one slash)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "http:/about",
|
||||
.expected = "http://about",
|
||||
},
|
||||
//different schemes and path as absolute (with two slashes)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "http://about",
|
||||
.expected = "http://about",
|
||||
},
|
||||
//same schemes and path as absolute (with more slashes)
|
||||
.{
|
||||
.base = "https://site/",
|
||||
.path = "https://path",
|
||||
.expected = "https://path",
|
||||
},
|
||||
//path scheme is not special and path as absolute (without additional slashes)
|
||||
.{
|
||||
.base = "http://localhost/",
|
||||
.path = "data:test",
|
||||
.expected = "data:test",
|
||||
},
|
||||
//different schemes and path as absolute (pathscheme=ws)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "ws://about",
|
||||
.expected = "ws://about",
|
||||
},
|
||||
//different schemes and path as absolute (path scheme=wss)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "wss://about",
|
||||
.expected = "wss://about",
|
||||
},
|
||||
//different schemes and path as absolute (path scheme=ftp)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "ftp://about",
|
||||
.expected = "ftp://about",
|
||||
},
|
||||
//different schemes and path as absolute (path scheme=file)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "file://path/to/file",
|
||||
.expected = "file://path/to/file",
|
||||
},
|
||||
//different schemes and path as absolute (path scheme=file, host is empty)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "file:/path/to/file",
|
||||
.expected = "file:///path/to/file",
|
||||
},
|
||||
//different schemes and path as absolute (path scheme=file, host is empty)
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "file:/",
|
||||
.expected = "file:///",
|
||||
},
|
||||
//different schemes without :// and normalize "file" scheme, absolute path
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "file:path/to/file",
|
||||
.expected = "file:///path/to/file",
|
||||
},
|
||||
//same schemes without :// in path and rest starts with scheme:/, relative path
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "https:/file:/relative/path/",
|
||||
.expected = "https://www.example.com/file:/relative/path/",
|
||||
},
|
||||
//same schemes without :// in path and rest starts with scheme://, relative path
|
||||
.{
|
||||
.base = "https://www.example.com/example",
|
||||
.path = "https:/http://relative/path/",
|
||||
.expected = "https://www.example.com/http://relative/path/",
|
||||
},
|
||||
//same schemes without :// in path , relative state
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "http:relative:path",
|
||||
.expected = "http://www.example.com/relative:path",
|
||||
},
|
||||
//repeat different schemes in path
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "http:http:/relative/path/",
|
||||
.expected = "http://www.example.com/http:/relative/path/",
|
||||
},
|
||||
//repeat different schemes in path
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "http:https://relative:path",
|
||||
.expected = "http://www.example.com/https://relative:path",
|
||||
},
|
||||
//NOT required :// for blob scheme
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "blob:other",
|
||||
.expected = "blob:other",
|
||||
},
|
||||
//NOT required :// for NON-special schemes and can contains "+" or "-" or "." in scheme
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "custom+foo:other",
|
||||
.expected = "custom+foo:other",
|
||||
},
|
||||
//NOT required :// for NON-special schemes
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "blob:",
|
||||
.expected = "blob:",
|
||||
},
|
||||
//NOT required :// for special scheme equal base scheme
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "http:",
|
||||
.expected = "http://www.example.com/example",
|
||||
},
|
||||
//required :// for special scheme, so throw error.InvalidURL
|
||||
.{
|
||||
.base = "http://www.example.com/example",
|
||||
.path = "https:",
|
||||
.expected = "",
|
||||
.expected_error = true,
|
||||
},
|
||||
//incorrect symbols in path scheme
|
||||
.{
|
||||
.base = "https://site",
|
||||
.path = "http?://host/some",
|
||||
.expected = "https://site/http?://host/some",
|
||||
},
|
||||
};
|
||||
|
||||
for (cases) |case| {
|
||||
if (case.expected_error) {
|
||||
const result = resolve(testing.arena_allocator, case.base, case.path, .{});
|
||||
try testing.expectError(error.TypeError, result);
|
||||
} else {
|
||||
const result = try resolve(testing.arena_allocator, case.base, case.path, .{});
|
||||
try testing.expectString(case.expected, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -523,31 +523,6 @@ pub fn setDir(self: *Element, value: []const u8, page: *Page) !void {
|
||||
return self.setAttributeSafe(comptime .wrap("dir"), .wrap(value), page);
|
||||
}
|
||||
|
||||
// ARIAMixin - ARIA attribute reflection
|
||||
pub fn getAriaAtomic(self: *const Element) ?[]const u8 {
|
||||
return self.getAttributeSafe(comptime .wrap("aria-atomic"));
|
||||
}
|
||||
|
||||
pub fn setAriaAtomic(self: *Element, value: ?[]const u8, page: *Page) !void {
|
||||
if (value) |v| {
|
||||
try self.setAttributeSafe(comptime .wrap("aria-atomic"), .wrap(v), page);
|
||||
} else {
|
||||
try self.removeAttribute(comptime .wrap("aria-atomic"), page);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getAriaLive(self: *const Element) ?[]const u8 {
|
||||
return self.getAttributeSafe(comptime .wrap("aria-live"));
|
||||
}
|
||||
|
||||
pub fn setAriaLive(self: *Element, value: ?[]const u8, page: *Page) !void {
|
||||
if (value) |v| {
|
||||
try self.setAttributeSafe(comptime .wrap("aria-live"), .wrap(v), page);
|
||||
} else {
|
||||
try self.removeAttribute(comptime .wrap("aria-live"), page);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getClassName(self: *const Element) []const u8 {
|
||||
return self.getAttributeSafe(comptime .wrap("class")) orelse "";
|
||||
}
|
||||
@@ -1711,8 +1686,6 @@ pub const JsApi = struct {
|
||||
pub const localName = bridge.accessor(Element.getLocalName, null, .{});
|
||||
pub const id = bridge.accessor(Element.getId, Element.setId, .{});
|
||||
pub const slot = bridge.accessor(Element.getSlot, Element.setSlot, .{});
|
||||
pub const ariaAtomic = bridge.accessor(Element.getAriaAtomic, Element.setAriaAtomic, .{});
|
||||
pub const ariaLive = bridge.accessor(Element.getAriaLive, Element.setAriaLive, .{});
|
||||
pub const dir = bridge.accessor(Element.getDir, Element.setDir, .{});
|
||||
pub const className = bridge.accessor(Element.getClassName, Element.setClassName, .{});
|
||||
pub const classList = bridge.accessor(Element.getClassList, Element.setClassList, .{});
|
||||
|
||||
@@ -391,14 +391,6 @@ pub fn setLang(self: *HtmlElement, value: []const u8, page: *Page) !void {
|
||||
try self.asElement().setAttributeSafe(comptime .wrap("lang"), .wrap(value), page);
|
||||
}
|
||||
|
||||
pub fn getTitle(self: *HtmlElement) []const u8 {
|
||||
return self.asElement().getAttributeSafe(comptime .wrap("title")) orelse "";
|
||||
}
|
||||
|
||||
pub fn setTitle(self: *HtmlElement, value: []const u8, page: *Page) !void {
|
||||
try self.asElement().setAttributeSafe(comptime .wrap("title"), .wrap(value), page);
|
||||
}
|
||||
|
||||
pub fn getAttributeFunction(
|
||||
self: *HtmlElement,
|
||||
listener_type: GlobalEventHandler,
|
||||
@@ -1239,7 +1231,6 @@ pub const JsApi = struct {
|
||||
pub const hidden = bridge.accessor(HtmlElement.getHidden, HtmlElement.setHidden, .{});
|
||||
pub const lang = bridge.accessor(HtmlElement.getLang, HtmlElement.setLang, .{});
|
||||
pub const tabIndex = bridge.accessor(HtmlElement.getTabIndex, HtmlElement.setTabIndex, .{});
|
||||
pub const title = bridge.accessor(HtmlElement.getTitle, HtmlElement.setTitle, .{});
|
||||
|
||||
pub const onabort = bridge.accessor(HtmlElement.getOnAbort, HtmlElement.setOnAbort, .{});
|
||||
pub const onanimationcancel = bridge.accessor(HtmlElement.getOnAnimationCancel, HtmlElement.setOnAnimationCancel, .{});
|
||||
|
||||
@@ -174,14 +174,6 @@ pub fn setType(self: *Anchor, value: []const u8, page: *Page) !void {
|
||||
try self.asElement().setAttributeSafe(comptime .wrap("type"), .wrap(value), page);
|
||||
}
|
||||
|
||||
pub fn getRel(self: *Anchor) []const u8 {
|
||||
return self.asConstElement().getAttributeSafe(comptime .wrap("rel")) orelse "";
|
||||
}
|
||||
|
||||
pub fn setRel(self: *Anchor, value: []const u8, page: *Page) !void {
|
||||
try self.asElement().setAttributeSafe(comptime .wrap("rel"), .wrap(value), page);
|
||||
}
|
||||
|
||||
pub fn getName(self: *const Anchor) []const u8 {
|
||||
return self.asConstElement().getAttributeSafe(comptime .wrap("name")) orelse "";
|
||||
}
|
||||
@@ -226,7 +218,6 @@ pub const JsApi = struct {
|
||||
pub const pathname = bridge.accessor(Anchor.getPathname, Anchor.setPathname, .{});
|
||||
pub const search = bridge.accessor(Anchor.getSearch, Anchor.setSearch, .{});
|
||||
pub const hash = bridge.accessor(Anchor.getHash, Anchor.setHash, .{});
|
||||
pub const rel = bridge.accessor(Anchor.getRel, Anchor.setRel, .{});
|
||||
pub const @"type" = bridge.accessor(Anchor.getType, Anchor.setType, .{});
|
||||
pub const text = bridge.accessor(Anchor.getText, Anchor.setText, .{});
|
||||
pub const relList = bridge.accessor(_getRelList, null, .{ .null_as_undefined = true });
|
||||
|
||||
@@ -86,8 +86,8 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {
|
||||
}
|
||||
|
||||
// TODO: do we really need 2 different header structs??
|
||||
const http = @import("../../../network/http.zig");
|
||||
pub fn populateHttpHeader(self: *Headers, allocator: Allocator, http_headers: *http.Headers) !void {
|
||||
const net_http = @import("../../../network/http.zig");
|
||||
pub fn populateHttpHeader(self: *Headers, allocator: Allocator, http_headers: *net_http.Headers) !void {
|
||||
for (self._list._entries.items) |entry| {
|
||||
const merged = try std.mem.concatWithSentinel(allocator, u8, &.{ entry.name.str(), ": ", entry.value.str() }, 0);
|
||||
try http_headers.add(merged);
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
const std = @import("std");
|
||||
|
||||
const js = @import("../../js/js.zig");
|
||||
const http = @import("../../../network/http.zig");
|
||||
const net_http = @import("../../../network/http.zig");
|
||||
|
||||
const URL = @import("../URL.zig");
|
||||
const Page = @import("../../Page.zig");
|
||||
@@ -31,7 +31,7 @@ const Allocator = std.mem.Allocator;
|
||||
const Request = @This();
|
||||
|
||||
_url: [:0]const u8,
|
||||
_method: http.Method,
|
||||
_method: net_http.Method,
|
||||
_headers: ?*Headers,
|
||||
_body: ?[]const u8,
|
||||
_arena: Allocator,
|
||||
@@ -119,14 +119,14 @@ pub fn init(input: Input, opts_: ?InitOpts, page: *Page) !*Request {
|
||||
});
|
||||
}
|
||||
|
||||
fn parseMethod(method: []const u8, page: *Page) !http.Method {
|
||||
fn parseMethod(method: []const u8, page: *Page) !net_http.Method {
|
||||
if (method.len > "propfind".len) {
|
||||
return error.InvalidMethod;
|
||||
}
|
||||
|
||||
const lower = std.ascii.lowerString(&page.buf, method);
|
||||
|
||||
const method_lookup = std.StaticStringMap(http.Method).initComptime(.{
|
||||
const method_lookup = std.StaticStringMap(net_http.Method).initComptime(.{
|
||||
.{ "get", .GET },
|
||||
.{ "post", .POST },
|
||||
.{ "delete", .DELETE },
|
||||
|
||||
@@ -22,7 +22,7 @@ const js = @import("../../js/js.zig");
|
||||
|
||||
const log = @import("../../../log.zig");
|
||||
const HttpClient = @import("../../HttpClient.zig");
|
||||
const http = @import("../../../network/http.zig");
|
||||
const net_http = @import("../../../network/http.zig");
|
||||
|
||||
const URL = @import("../../URL.zig");
|
||||
const Mime = @import("../../Mime.zig");
|
||||
@@ -47,7 +47,7 @@ _transfer: ?*HttpClient.Transfer = null,
|
||||
_active_request: bool = false,
|
||||
|
||||
_url: [:0]const u8 = "",
|
||||
_method: http.Method = .GET,
|
||||
_method: net_http.Method = .GET,
|
||||
_request_headers: *Headers,
|
||||
_request_body: ?[]const u8 = null,
|
||||
|
||||
@@ -406,7 +406,7 @@ fn httpStartCallback(transfer: *HttpClient.Transfer) !void {
|
||||
self._transfer = transfer;
|
||||
}
|
||||
|
||||
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: http.Header) !void {
|
||||
fn httpHeaderCallback(transfer: *HttpClient.Transfer, header: net_http.Header) !void {
|
||||
const self: *XMLHttpRequest = @ptrCast(@alignCast(transfer.ctx));
|
||||
const joined = try std.fmt.allocPrint(self._arena, "{s}: {s}", .{ header.name, header.value });
|
||||
try self._response_headers.append(self._arena, joined);
|
||||
@@ -574,7 +574,7 @@ fn stateChanged(self: *XMLHttpRequest, state: ReadyState, page: *Page) !void {
|
||||
}
|
||||
}
|
||||
|
||||
fn parseMethod(method: []const u8) !http.Method {
|
||||
fn parseMethod(method: []const u8) !net_http.Method {
|
||||
if (std.ascii.eqlIgnoreCase(method, "get")) {
|
||||
return .GET;
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ const CDP = @import("../CDP.zig");
|
||||
const log = @import("../../log.zig");
|
||||
|
||||
const HttpClient = @import("../../browser/HttpClient.zig");
|
||||
const http = @import("../../network/http.zig");
|
||||
const net_http = @import("../../network/http.zig");
|
||||
const Notification = @import("../../Notification.zig");
|
||||
|
||||
const network = @import("network.zig");
|
||||
@@ -224,7 +224,7 @@ fn continueRequest(cmd: *CDP.Command) !void {
|
||||
url: ?[]const u8 = null,
|
||||
method: ?[]const u8 = null,
|
||||
postData: ?[]const u8 = null,
|
||||
headers: ?[]const http.Header = null,
|
||||
headers: ?[]const net_http.Header = null,
|
||||
interceptResponse: bool = false,
|
||||
})) orelse return error.InvalidParams;
|
||||
|
||||
@@ -249,7 +249,7 @@ fn continueRequest(cmd: *CDP.Command) !void {
|
||||
try transfer.updateURL(try arena.dupeZ(u8, url));
|
||||
}
|
||||
if (params.method) |method| {
|
||||
transfer.req.method = std.meta.stringToEnum(http.Method, method) orelse return error.InvalidParams;
|
||||
transfer.req.method = std.meta.stringToEnum(net_http.Method, method) orelse return error.InvalidParams;
|
||||
}
|
||||
|
||||
if (params.headers) |headers| {
|
||||
@@ -326,7 +326,7 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
|
||||
const params = (try cmd.params(struct {
|
||||
requestId: []const u8, // "INT-{d}"
|
||||
responseCode: u16,
|
||||
responseHeaders: ?[]const http.Header = null,
|
||||
responseHeaders: ?[]const net_http.Header = null,
|
||||
binaryResponseHeaders: ?[]const u8 = null,
|
||||
body: ?[]const u8 = null,
|
||||
responsePhrase: ?[]const u8 = null,
|
||||
|
||||
@@ -47,7 +47,7 @@ pub noinline fn crash(
|
||||
writer.print("\nreason: {s}\n", .{reason}) catch abort();
|
||||
writer.print("OS: {s}\n", .{@tagName(builtin.os.tag)}) catch abort();
|
||||
writer.print("mode: {s}\n", .{@tagName(builtin.mode)}) catch abort();
|
||||
writer.print("version: {s}\n", .{lp.build_config.version_encoded}) catch abort();
|
||||
writer.print("version: {s}\n", .{lp.build_config.version}) catch abort();
|
||||
inline for (@typeInfo(@TypeOf(args)).@"struct".fields) |f| {
|
||||
writer.writeAll(f.name ++ ": ") catch break;
|
||||
@import("log.zig").writeValue(.pretty, @field(args, f.name), writer) catch abort();
|
||||
@@ -86,15 +86,7 @@ fn report(reason: []const u8, begin_addr: usize, args: anytype) !void {
|
||||
var url_buffer: [4096]u8 = undefined;
|
||||
const url = blk: {
|
||||
var writer: std.Io.Writer = .fixed(&url_buffer);
|
||||
try writer.writeAll("https://crash.lightpanda.io/c?v=");
|
||||
for (lp.build_config.version) |b| {
|
||||
if (b == '+') {
|
||||
try writer.writeAll("%2B");
|
||||
} else {
|
||||
try writer.writeByte(b);
|
||||
}
|
||||
}
|
||||
try writer.writeAll("&r=");
|
||||
try writer.print("https://crash.lightpanda.io/c?v={s}&r=", .{lp.build_config.version});
|
||||
for (reason) |b| {
|
||||
switch (b) {
|
||||
'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_' => try writer.writeByte(b),
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
const std = @import("std");
|
||||
pub const App = @import("App.zig");
|
||||
pub const Network = @import("network/Network.zig");
|
||||
pub const Network = @import("network/Runtime.zig");
|
||||
pub const Server = @import("Server.zig");
|
||||
pub const Config = @import("Config.zig");
|
||||
pub const URL = @import("browser/URL.zig");
|
||||
|
||||
13
src/main.zig
13
src/main.zig
@@ -144,11 +144,22 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
|
||||
|
||||
app.network.run();
|
||||
},
|
||||
.mcp => {
|
||||
.mcp => |opts| {
|
||||
log.info(.mcp, "starting server", .{});
|
||||
|
||||
log.opts.format = .logfmt;
|
||||
|
||||
var cdp_server: ?*lp.Server = null;
|
||||
if (opts.cdp_port) |port| {
|
||||
const address = std.net.Address.parseIp("127.0.0.1", port) catch |err| {
|
||||
log.fatal(.mcp, "invalid cdp address", .{ .err = err, .port = port });
|
||||
return;
|
||||
};
|
||||
cdp_server = try lp.Server.init(app, address);
|
||||
try sighandler.on(lp.Server.shutdown, .{cdp_server.?});
|
||||
}
|
||||
defer if (cdp_server) |s| s.deinit();
|
||||
|
||||
var worker_thread = try std.Thread.spawn(.{}, mcpThread, .{ allocator, app });
|
||||
defer worker_thread.join();
|
||||
|
||||
|
||||
@@ -9,57 +9,72 @@ const protocol = @import("protocol.zig");
|
||||
const Server = @import("Server.zig");
|
||||
const CDPNode = @import("../cdp/Node.zig");
|
||||
|
||||
const goto_schema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "The URL to navigate to, must be a valid URL." },
|
||||
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." },
|
||||
\\ "waitUntil": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "done"], "description": "Optional wait strategy. Defaults to 'done'." }
|
||||
\\ },
|
||||
\\ "required": ["url"]
|
||||
\\}
|
||||
);
|
||||
|
||||
const url_params_schema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before processing." },
|
||||
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." },
|
||||
\\ "waitUntil": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "done"], "description": "Optional wait strategy. Defaults to 'done'." }
|
||||
\\ }
|
||||
\\}
|
||||
);
|
||||
|
||||
const evaluate_schema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "script": { "type": "string" },
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before evaluating." },
|
||||
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." },
|
||||
\\ "waitUntil": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "done"], "description": "Optional wait strategy. Defaults to 'done'." }
|
||||
\\ },
|
||||
\\ "required": ["script"]
|
||||
\\}
|
||||
);
|
||||
|
||||
pub const tool_list = [_]protocol.Tool{
|
||||
.{
|
||||
.name = "goto",
|
||||
.description = "Navigate to a specified URL and load the page in memory so it can be reused later for info extraction.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "The URL to navigate to, must be a valid URL." }
|
||||
\\ },
|
||||
\\ "required": ["url"]
|
||||
\\}
|
||||
),
|
||||
.inputSchema = goto_schema,
|
||||
},
|
||||
.{
|
||||
.name = "navigate",
|
||||
.description = "Alias for goto. Navigate to a specified URL and load the page in memory.",
|
||||
.inputSchema = goto_schema,
|
||||
},
|
||||
.{
|
||||
.name = "markdown",
|
||||
.description = "Get the page content in markdown format. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before fetching markdown." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
.inputSchema = url_params_schema,
|
||||
},
|
||||
.{
|
||||
.name = "links",
|
||||
.description = "Extract all links in the opened page. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before extracting links." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
.inputSchema = url_params_schema,
|
||||
},
|
||||
.{
|
||||
.name = "evaluate",
|
||||
.description = "Evaluate JavaScript in the current page context. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "script": { "type": "string" },
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before evaluating." }
|
||||
\\ },
|
||||
\\ "required": ["script"]
|
||||
\\}
|
||||
),
|
||||
.inputSchema = evaluate_schema,
|
||||
},
|
||||
.{
|
||||
.name = "eval",
|
||||
.description = "Alias for evaluate. Evaluate JavaScript in the current page context.",
|
||||
.inputSchema = evaluate_schema,
|
||||
},
|
||||
.{
|
||||
.name = "semantic_tree",
|
||||
@@ -69,6 +84,8 @@ pub const tool_list = [_]protocol.Tool{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before fetching the semantic tree." },
|
||||
\\ "timeout": { "type": "integer", "description": "Optional timeout in milliseconds. Defaults to 10000." },
|
||||
\\ "waitUntil": { "type": "string", "enum": ["load", "domcontentloaded", "networkidle", "done"], "description": "Optional wait strategy. Defaults to 'done'." },
|
||||
\\ "backendNodeId": { "type": "integer", "description": "Optional backend node ID to get the tree for a specific element instead of the document root." },
|
||||
\\ "maxDepth": { "type": "integer", "description": "Optional maximum depth of the tree to return. Useful for exploring high-level structure first." }
|
||||
\\ }
|
||||
@@ -91,38 +108,17 @@ pub const tool_list = [_]protocol.Tool{
|
||||
.{
|
||||
.name = "interactiveElements",
|
||||
.description = "Extract interactive elements from the opened page. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before extracting interactive elements." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
.inputSchema = url_params_schema,
|
||||
},
|
||||
.{
|
||||
.name = "structuredData",
|
||||
.description = "Extract structured data (like JSON-LD, OpenGraph, etc) from the opened page. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before extracting structured data." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
.inputSchema = url_params_schema,
|
||||
},
|
||||
.{
|
||||
.name = "detectForms",
|
||||
.description = "Detect all forms on the page and return their structure including fields, types, and required status. If a url is provided, it navigates to that url first.",
|
||||
.inputSchema = protocol.minify(
|
||||
\\{
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "url": { "type": "string", "description": "Optional URL to navigate to before detecting forms." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
.inputSchema = url_params_schema,
|
||||
},
|
||||
.{
|
||||
.name = "click",
|
||||
@@ -189,15 +185,21 @@ pub fn handleList(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
||||
|
||||
const GotoParams = struct {
|
||||
url: [:0]const u8,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
|
||||
const UrlParams = struct {
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
|
||||
const EvaluateParams = struct {
|
||||
script: [:0]const u8,
|
||||
url: ?[:0]const u8 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
|
||||
const ToolStreamingText = struct {
|
||||
@@ -274,6 +276,7 @@ const ToolAction = enum {
|
||||
structuredData,
|
||||
detectForms,
|
||||
evaluate,
|
||||
eval,
|
||||
semantic_tree,
|
||||
click,
|
||||
fill,
|
||||
@@ -291,6 +294,7 @@ const tool_map = std.StaticStringMap(ToolAction).initComptime(.{
|
||||
.{ "structuredData", .structuredData },
|
||||
.{ "detectForms", .detectForms },
|
||||
.{ "evaluate", .evaluate },
|
||||
.{ "eval", .eval },
|
||||
.{ "semantic_tree", .semantic_tree },
|
||||
.{ "click", .click },
|
||||
.{ "fill", .fill },
|
||||
@@ -324,7 +328,7 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
||||
.interactiveElements => try handleInteractiveElements(server, arena, req.id.?, call_params.arguments),
|
||||
.structuredData => try handleStructuredData(server, arena, req.id.?, call_params.arguments),
|
||||
.detectForms => try handleDetectForms(server, arena, req.id.?, call_params.arguments),
|
||||
.evaluate => try handleEvaluate(server, arena, req.id.?, call_params.arguments),
|
||||
.eval, .evaluate => try handleEvaluate(server, arena, req.id.?, call_params.arguments),
|
||||
.semantic_tree => try handleSemanticTree(server, arena, req.id.?, call_params.arguments),
|
||||
.click => try handleClick(server, arena, req.id.?, call_params.arguments),
|
||||
.fill => try handleFill(server, arena, req.id.?, call_params.arguments),
|
||||
@@ -335,7 +339,7 @@ pub fn handleCall(server: *Server, arena: std.mem.Allocator, req: protocol.Reque
|
||||
|
||||
fn handleGoto(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgs(GotoParams, arena, arguments, server, id, "goto");
|
||||
try performGoto(server, args.url, id);
|
||||
try performGoto(server, args.url, id, args.timeout, args.waitUntil);
|
||||
|
||||
const content = [_]protocol.TextContent([]const u8){.{ .text = "Navigated successfully." }};
|
||||
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
@@ -343,7 +347,7 @@ fn handleGoto(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arg
|
||||
|
||||
fn handleMarkdown(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const content = [_]protocol.TextContent(ToolStreamingText){.{
|
||||
.text = .{ .page = page, .action = .markdown },
|
||||
@@ -355,7 +359,7 @@ fn handleMarkdown(server: *Server, arena: std.mem.Allocator, id: std.json.Value,
|
||||
|
||||
fn handleLinks(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const content = [_]protocol.TextContent(ToolStreamingText){.{
|
||||
.text = .{ .page = page, .action = .links },
|
||||
@@ -370,9 +374,11 @@ fn handleSemanticTree(server: *Server, arena: std.mem.Allocator, id: std.json.Va
|
||||
url: ?[:0]const u8 = null,
|
||||
backendNodeId: ?u32 = null,
|
||||
maxDepth: ?u32 = null,
|
||||
timeout: ?u32 = null,
|
||||
waitUntil: ?lp.Config.WaitUntil = null,
|
||||
};
|
||||
const args = try parseArgsOrDefault(TreeParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const content = [_]protocol.TextContent(ToolStreamingText){.{
|
||||
.text = .{
|
||||
@@ -417,7 +423,7 @@ fn handleNodeDetails(server: *Server, arena: std.mem.Allocator, id: std.json.Val
|
||||
|
||||
fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const elements = lp.interactive.collectInteractiveElements(page.document.asNode(), arena, page) catch |err| {
|
||||
log.err(.mcp, "elements collection failed", .{ .err = err });
|
||||
@@ -438,7 +444,7 @@ fn handleInteractiveElements(server: *Server, arena: std.mem.Allocator, id: std.
|
||||
|
||||
fn handleStructuredData(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const data = lp.structured_data.collectStructuredData(page.document.asNode(), arena, page) catch |err| {
|
||||
log.err(.mcp, "struct data collection failed", .{ .err = err });
|
||||
@@ -453,7 +459,7 @@ fn handleStructuredData(server: *Server, arena: std.mem.Allocator, id: std.json.
|
||||
|
||||
fn handleDetectForms(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgsOrDefault(UrlParams, arena, arguments, server, id);
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
const forms_data = lp.forms.collectForms(arena, page.document.asNode(), page) catch |err| {
|
||||
log.err(.mcp, "form collection failed", .{ .err = err });
|
||||
@@ -474,7 +480,7 @@ fn handleDetectForms(server: *Server, arena: std.mem.Allocator, id: std.json.Val
|
||||
|
||||
fn handleEvaluate(server: *Server, arena: std.mem.Allocator, id: std.json.Value, arguments: ?std.json.Value) !void {
|
||||
const args = try parseArgs(EvaluateParams, arena, arguments, server, id, "evaluate");
|
||||
const page = try ensurePage(server, id, args.url);
|
||||
const page = try ensurePage(server, id, args.url, args.timeout, args.waitUntil);
|
||||
|
||||
var ls: js.Local.Scope = undefined;
|
||||
page.js.localScope(&ls);
|
||||
@@ -630,9 +636,9 @@ fn handleWaitForSelector(server: *Server, arena: std.mem.Allocator, id: std.json
|
||||
return server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content });
|
||||
}
|
||||
|
||||
fn ensurePage(server: *Server, id: std.json.Value, url: ?[:0]const u8) !*lp.Page {
|
||||
fn ensurePage(server: *Server, id: std.json.Value, url: ?[:0]const u8, timeout: ?u32, waitUntil: ?lp.Config.WaitUntil) !*lp.Page {
|
||||
if (url) |u| {
|
||||
try performGoto(server, u, id);
|
||||
try performGoto(server, u, id, timeout, waitUntil);
|
||||
}
|
||||
return server.session.currentPage() orelse {
|
||||
try server.sendError(id, .PageNotLoaded, "Page not loaded");
|
||||
@@ -668,7 +674,7 @@ fn parseArgs(comptime T: type, arena: std.mem.Allocator, arguments: ?std.json.Va
|
||||
};
|
||||
}
|
||||
|
||||
fn performGoto(server: *Server, url: [:0]const u8, id: std.json.Value) !void {
|
||||
fn performGoto(server: *Server, url: [:0]const u8, id: std.json.Value, timeout: ?u32, waitUntil: ?lp.Config.WaitUntil) !void {
|
||||
const session = server.session;
|
||||
if (session.page != null) {
|
||||
session.removePage();
|
||||
@@ -689,7 +695,10 @@ fn performGoto(server: *Server, url: [:0]const u8, id: std.json.Value) !void {
|
||||
try server.sendError(id, .InternalError, "Failed to start page runner");
|
||||
return error.NavigationFailed;
|
||||
};
|
||||
runner.wait(.{ .ms = 2000 }) catch {
|
||||
runner.wait(.{
|
||||
.ms = timeout orelse 10000,
|
||||
.until = waitUntil orelse .done,
|
||||
}) catch {
|
||||
try server.sendError(id, .InternalError, "Timeout waiting for page load");
|
||||
return error.NavigationFailed;
|
||||
};
|
||||
|
||||
@@ -26,11 +26,11 @@ const lp = @import("lightpanda");
|
||||
const Config = @import("../Config.zig");
|
||||
const libcurl = @import("../sys/libcurl.zig");
|
||||
|
||||
const http = @import("http.zig");
|
||||
const net_http = @import("http.zig");
|
||||
const RobotStore = @import("Robots.zig").RobotStore;
|
||||
const WebBotAuth = @import("WebBotAuth.zig");
|
||||
|
||||
const Network = @This();
|
||||
const Runtime = @This();
|
||||
|
||||
const Listener = struct {
|
||||
socket: posix.socket_t,
|
||||
@@ -46,11 +46,11 @@ const MAX_TICK_CALLBACKS = 16;
|
||||
allocator: Allocator,
|
||||
|
||||
config: *const Config,
|
||||
ca_blob: ?http.Blob,
|
||||
ca_blob: ?net_http.Blob,
|
||||
robot_store: RobotStore,
|
||||
web_bot_auth: ?WebBotAuth,
|
||||
|
||||
connections: []http.Connection,
|
||||
connections: []net_http.Connection,
|
||||
available: std.DoublyLinkedList = .{},
|
||||
conn_mutex: std.Thread.Mutex = .{},
|
||||
|
||||
@@ -63,8 +63,8 @@ wakeup_pipe: [2]posix.fd_t = .{ -1, -1 },
|
||||
shutdown: std.atomic.Value(bool) = .init(false),
|
||||
|
||||
// Multi is a heavy structure that can consume up to 2MB of RAM.
|
||||
// Currently, Network is used sparingly, and we only create it on demand.
|
||||
// When Network becomes truly shared, it should become a regular field.
|
||||
// Currently, Runtime is used sparingly, and we only create it on demand.
|
||||
// When Runtime becomes truly shared, it should become a regular field.
|
||||
multi: ?*libcurl.CurlM = null,
|
||||
submission_mutex: std.Thread.Mutex = .{},
|
||||
submission_queue: std.DoublyLinkedList = .{},
|
||||
@@ -200,7 +200,7 @@ fn globalDeinit() void {
|
||||
libcurl.curl_global_cleanup();
|
||||
}
|
||||
|
||||
pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
pub fn init(allocator: Allocator, config: *const Config) !Runtime {
|
||||
globalInit(allocator);
|
||||
errdefer globalDeinit();
|
||||
|
||||
@@ -213,18 +213,18 @@ pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });
|
||||
pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };
|
||||
|
||||
var ca_blob: ?http.Blob = null;
|
||||
var ca_blob: ?net_http.Blob = null;
|
||||
if (config.tlsVerifyHost()) {
|
||||
ca_blob = try loadCerts(allocator);
|
||||
}
|
||||
|
||||
const count: usize = config.httpMaxConcurrent();
|
||||
const connections = try allocator.alloc(http.Connection, count);
|
||||
const connections = try allocator.alloc(net_http.Connection, count);
|
||||
errdefer allocator.free(connections);
|
||||
|
||||
var available: std.DoublyLinkedList = .{};
|
||||
for (0..count) |i| {
|
||||
connections[i] = try http.Connection.init(ca_blob, config);
|
||||
connections[i] = try net_http.Connection.init(ca_blob, config);
|
||||
available.append(&connections[i].node);
|
||||
}
|
||||
|
||||
@@ -249,7 +249,7 @@ pub fn init(allocator: Allocator, config: *const Config) !Network {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Network) void {
|
||||
pub fn deinit(self: *Runtime) void {
|
||||
if (self.multi) |multi| {
|
||||
libcurl.curl_multi_cleanup(multi) catch {};
|
||||
}
|
||||
@@ -282,7 +282,7 @@ pub fn deinit(self: *Network) void {
|
||||
}
|
||||
|
||||
pub fn bind(
|
||||
self: *Network,
|
||||
self: *Runtime,
|
||||
address: net.Address,
|
||||
ctx: *anyopaque,
|
||||
on_accept: *const fn (ctx: *anyopaque, socket: posix.socket_t) void,
|
||||
@@ -313,7 +313,7 @@ pub fn bind(
|
||||
};
|
||||
}
|
||||
|
||||
pub fn onTick(self: *Network, ctx: *anyopaque, callback: *const fn (*anyopaque) void) void {
|
||||
pub fn onTick(self: *Runtime, ctx: *anyopaque, callback: *const fn (*anyopaque) void) void {
|
||||
self.callbacks_mutex.lock();
|
||||
defer self.callbacks_mutex.unlock();
|
||||
|
||||
@@ -328,7 +328,7 @@ pub fn onTick(self: *Network, ctx: *anyopaque, callback: *const fn (*anyopaque)
|
||||
self.wakeupPoll();
|
||||
}
|
||||
|
||||
pub fn fireTicks(self: *Network) void {
|
||||
pub fn fireTicks(self: *Runtime) void {
|
||||
self.callbacks_mutex.lock();
|
||||
defer self.callbacks_mutex.unlock();
|
||||
|
||||
@@ -337,7 +337,7 @@ pub fn fireTicks(self: *Network) void {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(self: *Network) void {
|
||||
pub fn run(self: *Runtime) void {
|
||||
var drain_buf: [64]u8 = undefined;
|
||||
var running_handles: c_int = 0;
|
||||
|
||||
@@ -428,18 +428,18 @@ pub fn run(self: *Network) void {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn submitRequest(self: *Network, conn: *http.Connection) void {
|
||||
pub fn submitRequest(self: *Runtime, conn: *net_http.Connection) void {
|
||||
self.submission_mutex.lock();
|
||||
self.submission_queue.append(&conn.node);
|
||||
self.submission_mutex.unlock();
|
||||
self.wakeupPoll();
|
||||
}
|
||||
|
||||
fn wakeupPoll(self: *Network) void {
|
||||
fn wakeupPoll(self: *Runtime) void {
|
||||
_ = posix.write(self.wakeup_pipe[1], &.{1}) catch {};
|
||||
}
|
||||
|
||||
fn drainQueue(self: *Network) void {
|
||||
fn drainQueue(self: *Runtime) void {
|
||||
self.submission_mutex.lock();
|
||||
defer self.submission_mutex.unlock();
|
||||
|
||||
@@ -455,7 +455,7 @@ fn drainQueue(self: *Network) void {
|
||||
};
|
||||
|
||||
while (self.submission_queue.popFirst()) |node| {
|
||||
const conn: *http.Connection = @fieldParentPtr("node", node);
|
||||
const conn: *net_http.Connection = @fieldParentPtr("node", node);
|
||||
conn.setPrivate(conn) catch |err| {
|
||||
lp.log.err(.app, "curl set private", .{ .err = err });
|
||||
self.releaseConnection(conn);
|
||||
@@ -468,12 +468,12 @@ fn drainQueue(self: *Network) void {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stop(self: *Network) void {
|
||||
pub fn stop(self: *Runtime) void {
|
||||
self.shutdown.store(true, .release);
|
||||
self.wakeupPoll();
|
||||
}
|
||||
|
||||
fn acceptConnections(self: *Network) void {
|
||||
fn acceptConnections(self: *Runtime) void {
|
||||
if (self.shutdown.load(.acquire)) {
|
||||
return;
|
||||
}
|
||||
@@ -503,7 +503,7 @@ fn acceptConnections(self: *Network) void {
|
||||
}
|
||||
}
|
||||
|
||||
fn preparePollFds(self: *Network, multi: *libcurl.CurlM) void {
|
||||
fn preparePollFds(self: *Runtime, multi: *libcurl.CurlM) void {
|
||||
const curl_fds = self.pollfds[PSEUDO_POLLFDS..];
|
||||
@memset(curl_fds, .{ .fd = -1, .events = 0, .revents = 0 });
|
||||
|
||||
@@ -514,14 +514,14 @@ fn preparePollFds(self: *Network, multi: *libcurl.CurlM) void {
|
||||
};
|
||||
}
|
||||
|
||||
fn getCurlTimeout(self: *Network) i32 {
|
||||
fn getCurlTimeout(self: *Runtime) i32 {
|
||||
const multi = self.multi orelse return -1;
|
||||
var timeout_ms: c_long = -1;
|
||||
libcurl.curl_multi_timeout(multi, &timeout_ms) catch return -1;
|
||||
return @intCast(@min(timeout_ms, std.math.maxInt(i32)));
|
||||
}
|
||||
|
||||
fn processCompletions(self: *Network, multi: *libcurl.CurlM) void {
|
||||
fn processCompletions(self: *Runtime, multi: *libcurl.CurlM) void {
|
||||
var msgs_in_queue: c_int = 0;
|
||||
while (libcurl.curl_multi_info_read(multi, &msgs_in_queue)) |msg| {
|
||||
switch (msg.data) {
|
||||
@@ -537,7 +537,7 @@ fn processCompletions(self: *Network, multi: *libcurl.CurlM) void {
|
||||
var ptr: *anyopaque = undefined;
|
||||
libcurl.curl_easy_getinfo(easy, .private, &ptr) catch
|
||||
lp.assert(false, "curl getinfo private", .{});
|
||||
const conn: *http.Connection = @ptrCast(@alignCast(ptr));
|
||||
const conn: *net_http.Connection = @ptrCast(@alignCast(ptr));
|
||||
|
||||
libcurl.curl_multi_remove_handle(multi, easy) catch {};
|
||||
self.releaseConnection(conn);
|
||||
@@ -556,7 +556,7 @@ comptime {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn getConnection(self: *Network) ?*http.Connection {
|
||||
pub fn getConnection(self: *Runtime) ?*net_http.Connection {
|
||||
self.conn_mutex.lock();
|
||||
defer self.conn_mutex.unlock();
|
||||
|
||||
@@ -564,7 +564,7 @@ pub fn getConnection(self: *Network) ?*http.Connection {
|
||||
return @fieldParentPtr("node", node);
|
||||
}
|
||||
|
||||
pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
|
||||
pub fn releaseConnection(self: *Runtime, conn: *net_http.Connection) void {
|
||||
conn.reset(self.config, self.ca_blob) catch |err| {
|
||||
lp.assert(false, "couldn't reset curl easy", .{ .err = err });
|
||||
};
|
||||
@@ -575,8 +575,8 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
|
||||
self.available.append(&conn.node);
|
||||
}
|
||||
|
||||
pub fn newConnection(self: *Network) !http.Connection {
|
||||
return http.Connection.init(self.ca_blob, self.config);
|
||||
pub fn newConnection(self: *Runtime) !net_http.Connection {
|
||||
return net_http.Connection.init(self.ca_blob, self.config);
|
||||
}
|
||||
|
||||
// Wraps lines @ 64 columns. A PEM is basically a base64 encoded DER (which is
|
||||
@@ -8,7 +8,7 @@ const log = @import("../log.zig");
|
||||
const App = @import("../App.zig");
|
||||
const Config = @import("../Config.zig");
|
||||
const telemetry = @import("telemetry.zig");
|
||||
const Network = @import("../network/Network.zig");
|
||||
const Runtime = @import("../network/Runtime.zig");
|
||||
|
||||
const URL = "https://telemetry.lightpanda.io";
|
||||
const BUFFER_SIZE = 1024;
|
||||
@@ -17,7 +17,7 @@ const MAX_BODY_SIZE = 500 * 1024; // 500KB server limit
|
||||
const LightPanda = @This();
|
||||
|
||||
allocator: Allocator,
|
||||
network: *Network,
|
||||
runtime: *Runtime,
|
||||
writer: std.Io.Writer.Allocating,
|
||||
|
||||
/// Protects concurrent producers in send().
|
||||
@@ -36,11 +36,11 @@ pub fn init(self: *LightPanda, app: *App, iid: ?[36]u8, run_mode: Config.RunMode
|
||||
.iid = iid,
|
||||
.run_mode = run_mode,
|
||||
.allocator = app.allocator,
|
||||
.network = &app.network,
|
||||
.runtime = &app.network,
|
||||
.writer = std.Io.Writer.Allocating.init(app.allocator),
|
||||
};
|
||||
|
||||
self.network.onTick(@ptrCast(self), flushCallback);
|
||||
self.runtime.onTick(@ptrCast(self), flushCallback);
|
||||
}
|
||||
|
||||
pub fn deinit(self: *LightPanda) void {
|
||||
@@ -70,17 +70,17 @@ fn flushCallback(ctx: *anyopaque) void {
|
||||
}
|
||||
|
||||
fn postEvent(self: *LightPanda) !void {
|
||||
const conn = self.network.getConnection() orelse {
|
||||
const conn = self.runtime.getConnection() orelse {
|
||||
return;
|
||||
};
|
||||
errdefer self.network.releaseConnection(conn);
|
||||
errdefer self.runtime.releaseConnection(conn);
|
||||
|
||||
const h = self.head.load(.monotonic);
|
||||
const t = self.tail.load(.acquire);
|
||||
const dropped = self.dropped.swap(0, .monotonic);
|
||||
|
||||
if (h == t and dropped == 0) {
|
||||
self.network.releaseConnection(conn);
|
||||
self.runtime.releaseConnection(conn);
|
||||
return;
|
||||
}
|
||||
errdefer _ = self.dropped.fetchAdd(dropped, .monotonic);
|
||||
@@ -104,7 +104,7 @@ fn postEvent(self: *LightPanda) !void {
|
||||
try conn.setBody(self.writer.written());
|
||||
|
||||
self.head.store(h + sent, .release);
|
||||
self.network.submitRequest(conn);
|
||||
self.runtime.submitRequest(conn);
|
||||
}
|
||||
|
||||
fn writeEvent(self: *LightPanda, event: telemetry.Event) !bool {
|
||||
|
||||
Reference in New Issue
Block a user