mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-03-28 15:40:04 +00:00
Compare commits
2 Commits
ci-web-bot
...
wp/mrdimid
| Author | SHA1 | Date | |
|---|---|---|---|
|
98124e97aa | ||
|
|
dd1c758c0e |
92
.github/workflows/e2e-test.yml
vendored
92
.github/workflows/e2e-test.yml
vendored
@@ -117,98 +117,6 @@ jobs:
|
||||
BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
|
||||
kill `cat LPD.pid` `cat PROXY.id`
|
||||
|
||||
# e2e tests w/ web-bot-auth configuration on.
|
||||
wba-demo-scripts:
|
||||
name: wba-demo-scripts
|
||||
needs: zig-build-release
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: 'lightpanda-io/demo'
|
||||
fetch-depth: 0
|
||||
|
||||
- run: npm install
|
||||
|
||||
- name: download artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: lightpanda-build-release
|
||||
|
||||
- run: chmod a+x ./lightpanda
|
||||
|
||||
- run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
|
||||
|
||||
- name: run end to end tests
|
||||
run: |
|
||||
./lightpanda serve \
|
||||
--web_bot_auth_key_file private_key.pem \
|
||||
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
|
||||
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
|
||||
& echo $! > LPD.pid
|
||||
go run runner/main.go
|
||||
kill `cat LPD.pid`
|
||||
|
||||
- name: build proxy
|
||||
run: |
|
||||
cd proxy
|
||||
go build
|
||||
|
||||
- name: run end to end tests through proxy
|
||||
run: |
|
||||
./proxy/proxy & echo $! > PROXY.id
|
||||
./lightpanda serve \
|
||||
--web_bot_auth_key_file private_key.pem \
|
||||
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
|
||||
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
|
||||
--http_proxy 'http://127.0.0.1:3000' \
|
||||
& echo $! > LPD.pid
|
||||
go run runner/main.go
|
||||
kill `cat LPD.pid` `cat PROXY.id`
|
||||
|
||||
- name: run request interception through proxy
|
||||
run: |
|
||||
export PROXY_USERNAME=username PROXY_PASSWORD=password
|
||||
./proxy/proxy & echo $! > PROXY.id
|
||||
./lightpanda serve & echo $! > LPD.pid
|
||||
URL=https://demo-browser.lightpanda.io/campfire-commerce/ node puppeteer/proxy_auth.js
|
||||
BASE_URL=https://demo-browser.lightpanda.io/ node playwright/proxy_auth.js
|
||||
kill `cat LPD.pid` `cat PROXY.id`
|
||||
|
||||
wba-test:
|
||||
name: wba-test
|
||||
needs: zig-build-release
|
||||
|
||||
env:
|
||||
LIGHTPANDA_DISABLE_TELEMETRY: true
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: download artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: lightpanda-build-release
|
||||
|
||||
- run: chmod a+x ./lightpanda
|
||||
|
||||
- run: echo "${{ secrets.WBA_PRIVATE_KEY_PEM }}" > private_key.pem
|
||||
|
||||
- run: |
|
||||
./lightpanda fetch https://crawltest.com/cdn-cgi/web-bot-auth \
|
||||
--log_level error \
|
||||
--web_bot_auth_key_file private_key.pem \
|
||||
--web_bot_auth_keyid ${{ vars.WBA_KEY_ID }} \
|
||||
--web_bot_auth_domain ${{ vars.WBA_DOMAIN }} \
|
||||
--dump markdown \
|
||||
| tee output.log
|
||||
|
||||
- run: cat output.log | grep -q "unknown public key or unknown verified bot ID for keyid"
|
||||
|
||||
cdp-and-hyperfine-bench:
|
||||
name: cdp-and-hyperfine-bench
|
||||
needs: zig-build-release
|
||||
|
||||
@@ -36,6 +36,10 @@ RUN ZIG=$(grep '\.minimum_zig_version = "' "build.zig.zon" | cut -d'"' -f2) && \
|
||||
mv zig-${ARCH}-linux-${ZIG} /usr/local/lib && \
|
||||
ln -s /usr/local/lib/zig-${ARCH}-linux-${ZIG}/zig /usr/local/bin/zig
|
||||
|
||||
# install deps
|
||||
RUN git submodule init && \
|
||||
git submodule update --recursive
|
||||
|
||||
# download and install v8
|
||||
RUN case $TARGETPLATFORM in \
|
||||
"linux/arm64") ARCH="aarch64" ;; \
|
||||
|
||||
14
src/App.zig
14
src/App.zig
@@ -26,7 +26,6 @@ const Snapshot = @import("browser/js/Snapshot.zig");
|
||||
const Platform = @import("browser/js/Platform.zig");
|
||||
const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
|
||||
const RobotStore = @import("browser/Robots.zig").RobotStore;
|
||||
const WebBotAuth = @import("browser/WebBotAuth.zig");
|
||||
|
||||
pub const Http = @import("http/Http.zig");
|
||||
pub const ArenaPool = @import("ArenaPool.zig");
|
||||
@@ -41,7 +40,6 @@ telemetry: Telemetry,
|
||||
allocator: Allocator,
|
||||
arena_pool: ArenaPool,
|
||||
robots: RobotStore,
|
||||
web_bot_auth: ?WebBotAuth,
|
||||
app_dir_path: ?[]const u8,
|
||||
shutdown: bool = false,
|
||||
|
||||
@@ -54,14 +52,7 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
|
||||
|
||||
app.robots = RobotStore.init(allocator);
|
||||
|
||||
if (config.webBotAuth()) |wba_cfg| {
|
||||
app.web_bot_auth = try WebBotAuth.fromConfig(allocator, &wba_cfg);
|
||||
} else {
|
||||
app.web_bot_auth = null;
|
||||
}
|
||||
errdefer if (app.web_bot_auth) |wba| wba.deinit(allocator);
|
||||
|
||||
app.http = try Http.init(allocator, &app.robots, &app.web_bot_auth, config);
|
||||
app.http = try Http.init(allocator, &app.robots, config);
|
||||
errdefer app.http.deinit();
|
||||
|
||||
app.platform = try Platform.init();
|
||||
@@ -93,9 +84,6 @@ pub fn deinit(self: *App) void {
|
||||
}
|
||||
self.telemetry.deinit();
|
||||
self.robots.deinit();
|
||||
if (self.web_bot_auth) |wba| {
|
||||
wba.deinit(allocator);
|
||||
}
|
||||
self.http.deinit();
|
||||
self.snapshot.deinit();
|
||||
self.platform.deinit();
|
||||
|
||||
@@ -23,8 +23,6 @@ const Allocator = std.mem.Allocator;
|
||||
const log = @import("log.zig");
|
||||
const dump = @import("browser/dump.zig");
|
||||
|
||||
const WebBotAuthConfig = @import("browser/WebBotAuth.zig").Config;
|
||||
|
||||
pub const RunMode = enum {
|
||||
help,
|
||||
fetch,
|
||||
@@ -155,17 +153,6 @@ pub fn userAgentSuffix(self: *const Config) ?[]const u8 {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn webBotAuth(self: *const Config) ?WebBotAuthConfig {
|
||||
return switch (self.mode) {
|
||||
inline .serve, .fetch, .mcp => |opts| WebBotAuthConfig{
|
||||
.key_file = opts.common.web_bot_auth_key_file orelse return null,
|
||||
.keyid = opts.common.web_bot_auth_keyid orelse return null,
|
||||
.domain = opts.common.web_bot_auth_domain orelse return null,
|
||||
},
|
||||
.help, .version => null,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn maxConnections(self: *const Config) u16 {
|
||||
return switch (self.mode) {
|
||||
.serve => |opts| opts.cdp_max_connections,
|
||||
@@ -230,10 +217,6 @@ pub const Common = struct {
|
||||
log_format: ?log.Format = null,
|
||||
log_filter_scopes: ?[]log.Scope = null,
|
||||
user_agent_suffix: ?[]const u8 = null,
|
||||
|
||||
web_bot_auth_key_file: ?[]const u8 = null,
|
||||
web_bot_auth_keyid: ?[]const u8 = null,
|
||||
web_bot_auth_domain: ?[]const u8 = null,
|
||||
};
|
||||
|
||||
/// Pre-formatted HTTP headers for reuse across Http and Client.
|
||||
@@ -341,14 +324,6 @@ pub fn printUsageAndExit(self: *const Config, success: bool) void {
|
||||
\\--user_agent_suffix
|
||||
\\ Suffix to append to the Lightpanda/X.Y User-Agent
|
||||
\\
|
||||
\\--web_bot_auth_key_file
|
||||
\\ Path to the Ed25519 private key PEM file.
|
||||
\\
|
||||
\\--web_bot_auth_keyid
|
||||
\\ The JWK thumbprint of your public key.
|
||||
\\
|
||||
\\--web_bot_auth_domain
|
||||
\\ Your domain e.g. yourdomain.com
|
||||
;
|
||||
|
||||
// MAX_HELP_LEN|
|
||||
@@ -870,32 +845,5 @@ fn parseCommonArg(
|
||||
return true;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--web_bot_auth_key_file", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_key_file" });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
common.web_bot_auth_key_file = try allocator.dupe(u8, str);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--web_bot_auth_keyid", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_keyid" });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
common.web_bot_auth_keyid = try allocator.dupe(u8, str);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (std.mem.eql(u8, "--web_bot_auth_domain", opt)) {
|
||||
const str = args.next() orelse {
|
||||
log.fatal(.app, "missing argument value", .{ .arg = "--web_bot_auth_domain" });
|
||||
return error.InvalidArgument;
|
||||
};
|
||||
common.web_bot_auth_domain = try allocator.dupe(u8, str);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
307
src/Net.zig
307
src/Net.zig
@@ -476,9 +476,17 @@ pub const Handles = struct {
|
||||
available: HandleList,
|
||||
multi: *libcurl.CurlM,
|
||||
performing: bool = false,
|
||||
runtime_ctx: ?RuntimeContext = null,
|
||||
|
||||
pub const HandleList = std.DoublyLinkedList;
|
||||
|
||||
const RuntimeContext = struct {
|
||||
handles: *Handles,
|
||||
runtime: *Runtime,
|
||||
cdp_fd: ?posix.fd_t,
|
||||
timer_deadline_ms: ?i64 = null,
|
||||
};
|
||||
|
||||
pub fn init(
|
||||
allocator: Allocator,
|
||||
ca_blob: ?libcurl.CurlBlob,
|
||||
@@ -532,6 +540,9 @@ pub const Handles = struct {
|
||||
|
||||
pub fn add(self: *Handles, conn: *const Connection) !void {
|
||||
try libcurl.curl_multi_add_handle(self.multi, conn.easy);
|
||||
if (self.runtime_ctx != null) {
|
||||
_ = try self.socketActionTimeout();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(self: *Handles, conn: *Connection) void {
|
||||
@@ -554,6 +565,10 @@ pub const Handles = struct {
|
||||
}
|
||||
|
||||
pub fn perform(self: *Handles) !c_int {
|
||||
if (self.runtime_ctx != null) {
|
||||
return self.socketActionTimeout();
|
||||
}
|
||||
|
||||
self.performing = true;
|
||||
defer self.performing = false;
|
||||
|
||||
@@ -579,6 +594,34 @@ pub const Handles = struct {
|
||||
}
|
||||
|
||||
pub fn poll(self: *Handles, extra_fds: []libcurl.CurlWaitFd, timeout_ms: c_int) !void {
|
||||
if (self.runtime_ctx) |*ctx| {
|
||||
var wait_ms: i32 = @intCast(timeout_ms);
|
||||
try self.runDueTimer();
|
||||
|
||||
if (ctx.timer_deadline_ms) |deadline| {
|
||||
const now = std.time.milliTimestamp();
|
||||
if (deadline > now) {
|
||||
const remaining: i32 = @intCast(@min(deadline - now, std.math.maxInt(i32)));
|
||||
wait_ms = @min(wait_ms, remaining);
|
||||
} else {
|
||||
wait_ms = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const watched_fd: ?posix.fd_t = if (extra_fds.len > 0) extra_fds[0].fd else ctx.cdp_fd;
|
||||
const watched_ready = try ctx.runtime.dispatchFor(wait_ms, watched_fd);
|
||||
|
||||
if (extra_fds.len > 0) {
|
||||
extra_fds[0].revents = .{};
|
||||
if (watched_ready) {
|
||||
extra_fds[0].revents.pollin = true;
|
||||
}
|
||||
}
|
||||
|
||||
try self.runDueTimer();
|
||||
return;
|
||||
}
|
||||
|
||||
try libcurl.curl_multi_poll(self.multi, extra_fds, timeout_ms, null);
|
||||
}
|
||||
|
||||
@@ -598,6 +641,128 @@ pub const Handles = struct {
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn attachRuntime(self: *Handles, runtime: *Runtime, cdp_fd: ?posix.fd_t) !void {
|
||||
self.detachRuntime();
|
||||
self.runtime_ctx = .{
|
||||
.handles = self,
|
||||
.runtime = runtime,
|
||||
.cdp_fd = cdp_fd,
|
||||
.timer_deadline_ms = null,
|
||||
};
|
||||
|
||||
const ctx = &self.runtime_ctx.?;
|
||||
try libcurl.curl_multi_setopt(self.multi, .socket_function, onCurlSocket);
|
||||
try libcurl.curl_multi_setopt(self.multi, .socket_data, ctx);
|
||||
try libcurl.curl_multi_setopt(self.multi, .timer_function, onCurlTimer);
|
||||
try libcurl.curl_multi_setopt(self.multi, .timer_data, ctx);
|
||||
_ = try self.socketActionTimeout();
|
||||
}
|
||||
|
||||
pub fn detachRuntime(self: *Handles) void {
|
||||
if (self.runtime_ctx) |*ctx| {
|
||||
ctx.runtime.removeByCtx(ctx);
|
||||
}
|
||||
libcurl.curl_multi_setopt(self.multi, .socket_function, @as(?*const libcurl.CurlSocketCallback, null)) catch {};
|
||||
libcurl.curl_multi_setopt(self.multi, .socket_data, @as(?*anyopaque, null)) catch {};
|
||||
libcurl.curl_multi_setopt(self.multi, .timer_function, @as(?*const libcurl.CurlTimerCallback, null)) catch {};
|
||||
libcurl.curl_multi_setopt(self.multi, .timer_data, @as(?*anyopaque, null)) catch {};
|
||||
self.runtime_ctx = null;
|
||||
}
|
||||
|
||||
fn runDueTimer(self: *Handles) !void {
|
||||
const ctx = &(self.runtime_ctx orelse return);
|
||||
const deadline = ctx.timer_deadline_ms orelse return;
|
||||
if (std.time.milliTimestamp() < deadline) {
|
||||
return;
|
||||
}
|
||||
ctx.timer_deadline_ms = null;
|
||||
_ = try self.socketActionTimeout();
|
||||
}
|
||||
|
||||
fn socketAction(self: *Handles, fd: posix.fd_t, events: u32) !c_int {
|
||||
const select_mask = runtimeEventsToCurlSelect(events).toC();
|
||||
var running: c_int = undefined;
|
||||
self.performing = true;
|
||||
defer self.performing = false;
|
||||
try libcurl.curl_multi_socket_action(self.multi, @intCast(fd), select_mask, &running);
|
||||
return running;
|
||||
}
|
||||
|
||||
fn socketActionTimeout(self: *Handles) !c_int {
|
||||
var running: c_int = undefined;
|
||||
self.performing = true;
|
||||
defer self.performing = false;
|
||||
try libcurl.curl_multi_socket_action(self.multi, libcurl.CURL_SOCKET_TIMEOUT, 0, &running);
|
||||
return running;
|
||||
}
|
||||
|
||||
fn runtimeEventsToCurlSelect(events: u32) libcurl.CurlSelectMask {
|
||||
return .{
|
||||
.in = (events & Runtime.READABLE) != 0,
|
||||
.out = (events & Runtime.WRITABLE) != 0,
|
||||
.err = (events & Runtime.ERROR) != 0,
|
||||
};
|
||||
}
|
||||
|
||||
fn curlPollToRuntimeEvents(what: libcurl.CurlPoll) u32 {
|
||||
return switch (what) {
|
||||
.in => Runtime.READABLE | Runtime.ERROR,
|
||||
.out => Runtime.WRITABLE | Runtime.ERROR,
|
||||
.inout => Runtime.READABLE | Runtime.WRITABLE | Runtime.ERROR,
|
||||
.remove => 0,
|
||||
};
|
||||
}
|
||||
|
||||
fn onRuntimeFdEvent(ctx_ptr: *anyopaque, event: RuntimeEvent) anyerror!void {
|
||||
const ctx: *RuntimeContext = @ptrCast(@alignCast(ctx_ptr));
|
||||
_ = try ctx.handles.socketAction(event.fd, event.events);
|
||||
}
|
||||
|
||||
fn onCurlSocket(
|
||||
easy: ?*libcurl.Curl,
|
||||
s: libcurl.CurlSocket,
|
||||
what_raw: c_int,
|
||||
userp: ?*anyopaque,
|
||||
socketp: ?*anyopaque,
|
||||
) callconv(.c) c_int {
|
||||
_ = easy;
|
||||
_ = socketp;
|
||||
const ctx = userp orelse return 0;
|
||||
const runtime_ctx: *RuntimeContext = @ptrCast(@alignCast(ctx));
|
||||
|
||||
const what = std.meta.intToEnum(libcurl.CurlPoll, what_raw) catch return 0;
|
||||
if (what == .remove) {
|
||||
runtime_ctx.runtime.remove(@intCast(s));
|
||||
return 0;
|
||||
}
|
||||
|
||||
runtime_ctx.runtime.add(
|
||||
@intCast(s),
|
||||
curlPollToRuntimeEvents(what),
|
||||
runtime_ctx,
|
||||
onRuntimeFdEvent,
|
||||
) catch {};
|
||||
return 0;
|
||||
}
|
||||
|
||||
fn onCurlTimer(
|
||||
multi: ?*libcurl.CurlM,
|
||||
timeout_ms: c_long,
|
||||
userp: ?*anyopaque,
|
||||
) callconv(.c) c_int {
|
||||
_ = multi;
|
||||
const ctx = userp orelse return 0;
|
||||
const runtime_ctx: *RuntimeContext = @ptrCast(@alignCast(ctx));
|
||||
|
||||
if (timeout_ms < 0) {
|
||||
runtime_ctx.timer_deadline_ms = null;
|
||||
return 0;
|
||||
}
|
||||
|
||||
runtime_ctx.timer_deadline_ms = std.time.milliTimestamp() + @as(i64, @intCast(timeout_ms));
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
// TODO: on BSD / Linux, we could just read the PEM file directly.
|
||||
@@ -1381,6 +1546,148 @@ pub const WsConnection = struct {
|
||||
}
|
||||
};
|
||||
|
||||
pub const RuntimeEvent = struct {
|
||||
fd: posix.fd_t,
|
||||
events: u32,
|
||||
};
|
||||
|
||||
pub const Runtime = PollRuntime;
|
||||
|
||||
var runtime_active = std.atomic.Value(bool).init(false);
|
||||
|
||||
const PollRuntime = struct {
|
||||
pub const READABLE: u32 = @intCast(posix.POLL.IN);
|
||||
pub const WRITABLE: u32 = @intCast(posix.POLL.OUT);
|
||||
pub const ERROR: u32 = @intCast(posix.POLL.ERR | posix.POLL.HUP | posix.POLL.NVAL);
|
||||
|
||||
allocator: Allocator,
|
||||
watchers: std.AutoHashMapUnmanaged(posix.fd_t, Watcher) = .empty,
|
||||
|
||||
const Self = @This();
|
||||
|
||||
pub const EventCallback = *const fn (ctx: *anyopaque, event: RuntimeEvent) anyerror!void;
|
||||
|
||||
const Watcher = struct {
|
||||
events: u32,
|
||||
ctx: *anyopaque,
|
||||
cb: EventCallback,
|
||||
};
|
||||
|
||||
pub fn init(allocator: Allocator) !Self {
|
||||
if (runtime_active.swap(true, .acq_rel)) {
|
||||
return error.RuntimeAlreadyActive;
|
||||
}
|
||||
errdefer _ = runtime_active.swap(false, .acq_rel);
|
||||
|
||||
return .{
|
||||
.allocator = allocator,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Self) void {
|
||||
self.watchers.deinit(self.allocator);
|
||||
_ = runtime_active.swap(false, .acq_rel);
|
||||
}
|
||||
|
||||
pub fn add(self: *Self, fd: posix.fd_t, events: u32, ctx: *anyopaque, cb: EventCallback) !void {
|
||||
const gop = try self.watchers.getOrPut(self.allocator, fd);
|
||||
gop.value_ptr.* = .{ .events = events, .ctx = ctx, .cb = cb };
|
||||
}
|
||||
|
||||
pub fn remove(self: *Self, fd: posix.fd_t) void {
|
||||
_ = self.watchers.remove(fd);
|
||||
}
|
||||
|
||||
pub fn removeByCtx(self: *Self, ctx: *anyopaque) void {
|
||||
while (true) {
|
||||
var found: ?posix.fd_t = null;
|
||||
var it = self.watchers.iterator();
|
||||
while (it.next()) |entry| {
|
||||
if (entry.value_ptr.ctx == ctx) {
|
||||
found = entry.key_ptr.*;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const fd = found orelse break;
|
||||
self.remove(fd);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(self: *Self, stop_ctx: *anyopaque, should_stop: *const fn (ctx: *anyopaque) bool) !void {
|
||||
while (!should_stop(stop_ctx)) {
|
||||
_ = try self.dispatch(200);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dispatch(self: *Self, timeout_ms: i32) !usize {
|
||||
var triggered: usize = 0;
|
||||
_ = try self.dispatchPoll(timeout_ms, null, &triggered);
|
||||
return triggered;
|
||||
}
|
||||
|
||||
pub fn dispatchFor(self: *Self, timeout_ms: i32, watched_fd: ?posix.fd_t) !bool {
|
||||
var triggered: usize = 0;
|
||||
return self.dispatchPoll(timeout_ms, watched_fd, &triggered);
|
||||
}
|
||||
|
||||
fn dispatchPoll(self: *Self, timeout_ms: i32, watched_fd: ?posix.fd_t, triggered: *usize) !bool {
|
||||
var pollfds: std.ArrayList(posix.pollfd) = .empty;
|
||||
defer pollfds.deinit(self.allocator);
|
||||
|
||||
var it = self.watchers.iterator();
|
||||
while (it.next()) |entry| {
|
||||
try pollfds.append(self.allocator, .{
|
||||
.fd = entry.key_ptr.*,
|
||||
.events = @intCast(entry.value_ptr.events),
|
||||
.revents = 0,
|
||||
});
|
||||
}
|
||||
|
||||
if (watched_fd) |wfd| {
|
||||
if (!self.watchers.contains(wfd)) {
|
||||
try pollfds.append(self.allocator, .{
|
||||
.fd = wfd,
|
||||
.events = @intCast(Self.READABLE | Self.ERROR),
|
||||
.revents = 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (pollfds.items.len == 0) {
|
||||
if (timeout_ms > 0) {
|
||||
std.Thread.sleep(@as(u64, @intCast(timeout_ms)) * std.time.ns_per_ms);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
_ = try posix.poll(pollfds.items, timeout_ms);
|
||||
|
||||
var watched_ready = false;
|
||||
for (pollfds.items) |pfd| {
|
||||
if (pfd.revents == 0) continue;
|
||||
triggered.* += 1;
|
||||
|
||||
const revents_u32: u32 = @intCast(pfd.revents);
|
||||
if (watched_fd) |wfd| {
|
||||
if (pfd.fd == wfd) {
|
||||
watched_ready = true;
|
||||
}
|
||||
}
|
||||
|
||||
const watcher = self.watchers.get(pfd.fd) orelse continue;
|
||||
watcher.cb(watcher.ctx, .{
|
||||
.fd = pfd.fd,
|
||||
.events = revents_u32,
|
||||
}) catch |err| {
|
||||
log.err(.app, "runtime callback", .{ .err = err, .fd = pfd.fd });
|
||||
self.remove(pfd.fd);
|
||||
};
|
||||
}
|
||||
|
||||
return watched_ready;
|
||||
}
|
||||
};
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
test "mask" {
|
||||
|
||||
@@ -121,26 +121,52 @@ pub fn run(self: *Server, address: net.Address, timeout_ms: u32) !void {
|
||||
try posix.listen(listener, self.app.config.maxPendingConnections());
|
||||
|
||||
log.info(.app, "server running", .{ .address = address });
|
||||
while (!self.shutdown.load(.acquire)) {
|
||||
const socket = posix.accept(listener, null, null, posix.SOCK.NONBLOCK) catch |err| {
|
||||
switch (err) {
|
||||
error.SocketNotListening, error.ConnectionAborted => {
|
||||
log.info(.app, "server stopped", .{});
|
||||
break;
|
||||
},
|
||||
error.WouldBlock => {
|
||||
std.Thread.sleep(10 * std.time.ns_per_ms);
|
||||
continue;
|
||||
},
|
||||
else => {
|
||||
log.err(.app, "CDP accept", .{ .err = err });
|
||||
std.Thread.sleep(std.time.ns_per_s);
|
||||
continue;
|
||||
},
|
||||
}
|
||||
|
||||
var runtime = try Net.Runtime.init(self.allocator);
|
||||
defer runtime.deinit();
|
||||
|
||||
var accept_ctx: AcceptCtx = .{
|
||||
.server = self,
|
||||
.timeout_ms = timeout_ms,
|
||||
};
|
||||
|
||||
try runtime.add(listener, Net.Runtime.READABLE, &accept_ctx, onListenerEvent);
|
||||
defer runtime.remove(listener);
|
||||
|
||||
runtime.run(self, shouldStopRuntime) catch |err| {
|
||||
return err;
|
||||
};
|
||||
|
||||
log.info(.app, "server stopped", .{});
|
||||
}
|
||||
|
||||
const AcceptCtx = struct {
|
||||
server: *Server,
|
||||
timeout_ms: u32,
|
||||
};
|
||||
|
||||
fn shouldStopRuntime(ctx: *anyopaque) bool {
|
||||
const self: *Server = @ptrCast(@alignCast(ctx));
|
||||
return self.shutdown.load(.acquire);
|
||||
}
|
||||
|
||||
fn onListenerEvent(ctx: *anyopaque, event: Net.RuntimeEvent) !void {
|
||||
_ = event;
|
||||
const accept_ctx: *AcceptCtx = @ptrCast(@alignCast(ctx));
|
||||
const self = accept_ctx.server;
|
||||
const listener = self.listener orelse return;
|
||||
|
||||
while (true) {
|
||||
const socket = posix.accept(listener, null, null, posix.SOCK.NONBLOCK) catch |err| switch (err) {
|
||||
error.WouldBlock => return,
|
||||
error.SocketNotListening, error.ConnectionAborted => return,
|
||||
else => {
|
||||
log.err(.app, "CDP accept", .{ .err = err });
|
||||
return;
|
||||
},
|
||||
};
|
||||
|
||||
self.spawnWorker(socket, timeout_ms) catch |err| {
|
||||
self.spawnWorker(socket, accept_ctx.timeout_ms) catch |err| {
|
||||
log.err(.app, "CDP spawn", .{ .err = err });
|
||||
posix.close(socket);
|
||||
};
|
||||
|
||||
@@ -261,7 +261,7 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
|
||||
std.debug.assert(http_client.intercepted == 0);
|
||||
}
|
||||
|
||||
var ms: u64 = ms_to_next_task orelse blk: {
|
||||
const ms: u64 = ms_to_next_task orelse blk: {
|
||||
if (wait_ms - ms_remaining < 100) {
|
||||
if (comptime builtin.is_test) {
|
||||
return .done;
|
||||
@@ -288,13 +288,7 @@ fn _wait(self: *Session, page: *Page, wait_ms: u32) !WaitResult {
|
||||
// Same as above, except we have a scheduled task,
|
||||
// it just happens to be too far into the future
|
||||
// compared to how long we were told to wait.
|
||||
if (!browser.hasBackgroundTasks()) {
|
||||
return .done;
|
||||
}
|
||||
// _we_ have nothing to run, but v8 is working on
|
||||
// background tasks. We'll wait for them.
|
||||
browser.waitForBackgroundTasks();
|
||||
ms = 20;
|
||||
return .done;
|
||||
}
|
||||
|
||||
// We have a task to run in the not-so-distant future.
|
||||
|
||||
@@ -1,284 +0,0 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
const std = @import("std");
|
||||
const crypto = @import("../crypto.zig");
|
||||
|
||||
const Http = @import("../http/Http.zig");
|
||||
|
||||
const WebBotAuth = @This();
|
||||
|
||||
pkey: *crypto.EVP_PKEY,
|
||||
keyid: []const u8,
|
||||
directory_url: [:0]const u8,
|
||||
|
||||
pub const Config = struct {
|
||||
key_file: []const u8,
|
||||
keyid: []const u8,
|
||||
domain: []const u8,
|
||||
};
|
||||
|
||||
fn parsePemPrivateKey(pem: []const u8) !*crypto.EVP_PKEY {
|
||||
const begin = "-----BEGIN PRIVATE KEY-----";
|
||||
const end = "-----END PRIVATE KEY-----";
|
||||
const start_idx = std.mem.indexOf(u8, pem, begin) orelse return error.InvalidPem;
|
||||
const end_idx = std.mem.indexOf(u8, pem, end) orelse return error.InvalidPem;
|
||||
|
||||
const b64 = std.mem.trim(u8, pem[start_idx + begin.len .. end_idx], &std.ascii.whitespace);
|
||||
|
||||
// decode base64 into 48-byte DER buffer
|
||||
var der: [48]u8 = undefined;
|
||||
try std.base64.standard.Decoder.decode(der[0..48], b64);
|
||||
|
||||
// Ed25519 PKCS#8 structure always places the 32-byte raw private key at offset 16.
|
||||
const key_bytes = der[16..48];
|
||||
|
||||
const pkey = crypto.EVP_PKEY_new_raw_private_key(crypto.EVP_PKEY_ED25519, null, key_bytes.ptr, 32);
|
||||
return pkey orelse error.InvalidKey;
|
||||
}
|
||||
|
||||
fn signEd25519(pkey: *crypto.EVP_PKEY, message: []const u8, out: *[64]u8) !void {
|
||||
const ctx = crypto.EVP_MD_CTX_new() orelse return error.OutOfMemory;
|
||||
defer crypto.EVP_MD_CTX_free(ctx);
|
||||
|
||||
if (crypto.EVP_DigestSignInit(ctx, null, null, null, pkey) != 1)
|
||||
return error.SignInit;
|
||||
|
||||
var sig_len: usize = 64;
|
||||
if (crypto.EVP_DigestSign(ctx, out.ptr, &sig_len, message.ptr, message.len) != 1)
|
||||
return error.SignFailed;
|
||||
}
|
||||
|
||||
pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAuth {
|
||||
const pem = try std.fs.cwd().readFileAlloc(allocator, config.key_file, 1024 * 4);
|
||||
defer allocator.free(pem);
|
||||
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
errdefer crypto.EVP_PKEY_free(pkey);
|
||||
|
||||
const directory_url = try std.fmt.allocPrintSentinel(
|
||||
allocator,
|
||||
"https://{s}/.well-known/http-message-signatures-directory",
|
||||
.{config.domain},
|
||||
0,
|
||||
);
|
||||
errdefer allocator.free(directory_url);
|
||||
|
||||
return .{
|
||||
.pkey = pkey,
|
||||
// Owned by the Config so it's okay.
|
||||
.keyid = config.keyid,
|
||||
.directory_url = directory_url,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn signRequest(
|
||||
self: *const WebBotAuth,
|
||||
allocator: std.mem.Allocator,
|
||||
headers: *Http.Headers,
|
||||
authority: []const u8,
|
||||
) !void {
|
||||
const now = std.time.timestamp();
|
||||
const expires = now + 60;
|
||||
|
||||
// build the signature-input value (without the sig1= label)
|
||||
const sig_input_value = try std.fmt.allocPrint(
|
||||
allocator,
|
||||
"(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
|
||||
.{ now, expires, self.keyid },
|
||||
);
|
||||
defer allocator.free(sig_input_value);
|
||||
|
||||
// build the canonical string to sign
|
||||
const canonical = try std.fmt.allocPrint(
|
||||
allocator,
|
||||
"\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
|
||||
.{ authority, self.directory_url, sig_input_value },
|
||||
);
|
||||
defer allocator.free(canonical);
|
||||
|
||||
// sign it
|
||||
var sig: [64]u8 = undefined;
|
||||
try signEd25519(self.pkey, canonical, &sig);
|
||||
|
||||
// base64 encode
|
||||
const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
|
||||
const encoded = try allocator.alloc(u8, encoded_len);
|
||||
defer allocator.free(encoded);
|
||||
_ = std.base64.standard.Encoder.encode(encoded, &sig);
|
||||
|
||||
// build the 3 headers and add them
|
||||
const sig_agent = try std.fmt.allocPrintSentinel(
|
||||
allocator,
|
||||
"Signature-Agent: \"{s}\"",
|
||||
.{self.directory_url},
|
||||
0,
|
||||
);
|
||||
defer allocator.free(sig_agent);
|
||||
|
||||
const sig_input = try std.fmt.allocPrintSentinel(
|
||||
allocator,
|
||||
"Signature-Input: sig1={s}",
|
||||
.{sig_input_value},
|
||||
0,
|
||||
);
|
||||
defer allocator.free(sig_input);
|
||||
|
||||
const signature = try std.fmt.allocPrintSentinel(
|
||||
allocator,
|
||||
"Signature: sig1=:{s}:",
|
||||
.{encoded},
|
||||
0,
|
||||
);
|
||||
defer allocator.free(signature);
|
||||
|
||||
try headers.add(sig_agent);
|
||||
try headers.add(sig_input);
|
||||
try headers.add(signature);
|
||||
}
|
||||
|
||||
pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
|
||||
crypto.EVP_PKEY_free(self.pkey);
|
||||
allocator.free(self.directory_url);
|
||||
}
|
||||
|
||||
test "parsePemPrivateKey: valid Ed25519 PKCS#8 PEM" {
|
||||
const pem =
|
||||
\\-----BEGIN PRIVATE KEY-----
|
||||
\\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
|
||||
\\-----END PRIVATE KEY-----
|
||||
\\
|
||||
;
|
||||
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
defer crypto.EVP_PKEY_free(pkey);
|
||||
}
|
||||
|
||||
test "parsePemPrivateKey: missing BEGIN marker returns error" {
|
||||
const bad_pem = "-----END PRIVATE KEY-----\n";
|
||||
try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
|
||||
}
|
||||
|
||||
test "parsePemPrivateKey: missing END marker returns error" {
|
||||
const bad_pem = "-----BEGIN PRIVATE KEY-----\nMC4CAQA=\n";
|
||||
try std.testing.expectError(error.InvalidPem, parsePemPrivateKey(bad_pem));
|
||||
}
|
||||
|
||||
test "signEd25519: signature length is always 64 bytes" {
|
||||
const pem =
|
||||
\\-----BEGIN PRIVATE KEY-----
|
||||
\\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
|
||||
\\-----END PRIVATE KEY-----
|
||||
\\
|
||||
;
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
defer crypto.EVP_PKEY_free(pkey);
|
||||
|
||||
var sig: [64]u8 = @splat(0);
|
||||
try signEd25519(pkey, "hello world", &sig);
|
||||
|
||||
var all_zero = true;
|
||||
for (sig) |b| if (b != 0) {
|
||||
all_zero = false;
|
||||
break;
|
||||
};
|
||||
try std.testing.expect(!all_zero);
|
||||
}
|
||||
|
||||
test "signEd25519: same key + message produces same signature (deterministic)" {
|
||||
const pem =
|
||||
\\-----BEGIN PRIVATE KEY-----
|
||||
\\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
|
||||
\\-----END PRIVATE KEY-----
|
||||
\\
|
||||
;
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
defer crypto.EVP_PKEY_free(pkey);
|
||||
|
||||
var sig1: [64]u8 = undefined;
|
||||
var sig2: [64]u8 = undefined;
|
||||
try signEd25519(pkey, "deterministic test", &sig1);
|
||||
try signEd25519(pkey, "deterministic test", &sig2);
|
||||
|
||||
try std.testing.expectEqualSlices(u8, &sig1, &sig2);
|
||||
}
|
||||
|
||||
test "signEd25519: same key + diff message produces different signature (deterministic)" {
|
||||
const pem =
|
||||
\\-----BEGIN PRIVATE KEY-----
|
||||
\\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
|
||||
\\-----END PRIVATE KEY-----
|
||||
\\
|
||||
;
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
defer crypto.EVP_PKEY_free(pkey);
|
||||
|
||||
var sig1: [64]u8 = undefined;
|
||||
var sig2: [64]u8 = undefined;
|
||||
try signEd25519(pkey, "msg 1", &sig1);
|
||||
try signEd25519(pkey, "msg 2", &sig2);
|
||||
|
||||
try std.testing.expect(!std.mem.eql(u8, &sig1, &sig2));
|
||||
}
|
||||
|
||||
test "signRequest: adds headers with correct names" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const pem =
|
||||
\\-----BEGIN PRIVATE KEY-----
|
||||
\\MC4CAQAwBQYDK2VwBCIEIBuCRBIEFNtXcMBsyOOkFBFTJcEWTkbgSwKExhOjKFHT
|
||||
\\-----END PRIVATE KEY-----
|
||||
\\
|
||||
;
|
||||
const pkey = try parsePemPrivateKey(pem);
|
||||
|
||||
const directory_url = try allocator.dupeZ(
|
||||
u8,
|
||||
"https://example.com/.well-known/http-message-signatures-directory",
|
||||
);
|
||||
|
||||
var auth = WebBotAuth{
|
||||
.pkey = pkey,
|
||||
.keyid = "test-key-id",
|
||||
.directory_url = directory_url,
|
||||
};
|
||||
defer auth.deinit(allocator);
|
||||
|
||||
var headers = try Http.Headers.init("User-Agent: Test-Agent");
|
||||
defer headers.deinit();
|
||||
|
||||
try auth.signRequest(allocator, &headers, "example.com");
|
||||
|
||||
var it = headers.iterator();
|
||||
var found_sig_agent = false;
|
||||
var found_sig_input = false;
|
||||
var found_signature = false;
|
||||
var count: usize = 0;
|
||||
|
||||
while (it.next()) |h| {
|
||||
count += 1;
|
||||
if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
|
||||
if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
|
||||
if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
|
||||
}
|
||||
|
||||
try std.testing.expect(count >= 3);
|
||||
try std.testing.expect(found_sig_agent);
|
||||
try std.testing.expect(found_sig_input);
|
||||
try std.testing.expect(found_signature);
|
||||
}
|
||||
@@ -19,8 +19,6 @@
|
||||
const std = @import("std");
|
||||
|
||||
const Page = @import("Page.zig");
|
||||
const URL = @import("URL.zig");
|
||||
const TreeWalker = @import("webapi/TreeWalker.zig");
|
||||
const CData = @import("webapi/CData.zig");
|
||||
const Element = @import("webapi/Element.zig");
|
||||
const Node = @import("webapi/Node.zig");
|
||||
@@ -105,37 +103,20 @@ fn isVisibleElement(el: *Element) bool {
|
||||
};
|
||||
}
|
||||
|
||||
fn getAnchorLabel(el: *Element) ?[]const u8 {
|
||||
return el.getAttributeSafe(comptime .wrap("aria-label")) orelse el.getAttributeSafe(comptime .wrap("title"));
|
||||
}
|
||||
|
||||
fn isAllWhitespace(text: []const u8) bool {
|
||||
return for (text) |c| {
|
||||
if (!std.ascii.isWhitespace(c)) break false;
|
||||
} else true;
|
||||
}
|
||||
|
||||
fn hasBlockDescendant(root: *Node) bool {
|
||||
var tw = TreeWalker.FullExcludeSelf.Elements.init(root, .{});
|
||||
while (tw.next()) |el| {
|
||||
if (isBlock(el.getTag())) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn hasVisibleContent(root: *Node) bool {
|
||||
var tw = TreeWalker.FullExcludeSelf.init(root, .{});
|
||||
while (tw.next()) |node| {
|
||||
if (isSignificantText(node)) return true;
|
||||
if (node.is(Element)) |el| {
|
||||
if (!isVisibleElement(el)) {
|
||||
tw.skipChildren();
|
||||
} else if (el.getTag() == .img) {
|
||||
return true;
|
||||
}
|
||||
fn hasBlockDescendant(node: *Node) bool {
|
||||
var it = node.childrenIterator();
|
||||
return while (it.next()) |child| {
|
||||
if (child.is(Element)) |el| {
|
||||
if (isBlock(el.getTag())) break true;
|
||||
if (hasBlockDescendant(child)) break true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} else false;
|
||||
}
|
||||
|
||||
fn ensureNewline(state: *State, writer: *std.Io.Writer) !void {
|
||||
@@ -297,29 +278,20 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
|
||||
}
|
||||
try writer.writeAll("](");
|
||||
if (el.getAttributeSafe(comptime .wrap("src"))) |src| {
|
||||
const absolute_src = URL.resolve(page.call_arena, page.base(), src, .{ .encode = true }) catch src;
|
||||
try writer.writeAll(absolute_src);
|
||||
try writer.writeAll(src);
|
||||
}
|
||||
try writer.writeAll(")");
|
||||
state.last_char_was_newline = false;
|
||||
return;
|
||||
},
|
||||
.anchor => {
|
||||
const has_content = hasVisibleContent(el.asNode());
|
||||
const label = getAnchorLabel(el);
|
||||
const href_raw = el.getAttributeSafe(comptime .wrap("href"));
|
||||
|
||||
if (!has_content and label == null and href_raw == null) return;
|
||||
|
||||
const has_block = hasBlockDescendant(el.asNode());
|
||||
const href = if (href_raw) |h| URL.resolve(page.call_arena, page.base(), h, .{ .encode = true }) catch h else null;
|
||||
|
||||
if (has_block) {
|
||||
try renderChildren(el.asNode(), state, writer, page);
|
||||
if (href) |h| {
|
||||
if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
|
||||
if (!state.last_char_was_newline) try writer.writeByte('\n');
|
||||
try writer.writeAll("([](");
|
||||
try writer.writeAll(h);
|
||||
try writer.writeAll("([Link](");
|
||||
try writer.writeAll(href);
|
||||
try writer.writeAll("))\n");
|
||||
state.last_char_was_newline = true;
|
||||
}
|
||||
@@ -329,14 +301,10 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
|
||||
if (isStandaloneAnchor(el)) {
|
||||
if (!state.last_char_was_newline) try writer.writeByte('\n');
|
||||
try writer.writeByte('[');
|
||||
if (has_content) {
|
||||
try renderChildren(el.asNode(), state, writer, page);
|
||||
} else {
|
||||
try writer.writeAll(label orelse "");
|
||||
}
|
||||
try renderChildren(el.asNode(), state, writer, page);
|
||||
try writer.writeAll("](");
|
||||
if (href) |h| {
|
||||
try writer.writeAll(h);
|
||||
if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
|
||||
try writer.writeAll(href);
|
||||
}
|
||||
try writer.writeAll(")\n");
|
||||
state.last_char_was_newline = true;
|
||||
@@ -344,14 +312,10 @@ fn renderElement(el: *Element, state: *State, writer: *std.Io.Writer, page: *Pag
|
||||
}
|
||||
|
||||
try writer.writeByte('[');
|
||||
if (has_content) {
|
||||
try renderChildren(el.asNode(), state, writer, page);
|
||||
} else {
|
||||
try writer.writeAll(label orelse "");
|
||||
}
|
||||
try renderChildren(el.asNode(), state, writer, page);
|
||||
try writer.writeAll("](");
|
||||
if (href) |h| {
|
||||
try writer.writeAll(h);
|
||||
if (el.getAttributeSafe(comptime .wrap("href"))) |href| {
|
||||
try writer.writeAll(href);
|
||||
}
|
||||
try writer.writeByte(')');
|
||||
state.last_char_was_newline = false;
|
||||
@@ -488,8 +452,6 @@ fn testMarkdownHTML(html: []const u8, expected: []const u8) !void {
|
||||
const testing = @import("../testing.zig");
|
||||
const page = try testing.test_session.createPage();
|
||||
defer testing.test_session.removePage();
|
||||
page.url = "http://localhost/";
|
||||
|
||||
const doc = page.window._document;
|
||||
|
||||
const div = try doc.createElement("div", null, page);
|
||||
@@ -558,11 +520,11 @@ test "browser.markdown: blockquote" {
|
||||
}
|
||||
|
||||
test "browser.markdown: links" {
|
||||
try testMarkdownHTML("<a href=\"/relative\">Link</a>", "[Link](http://localhost/relative)\n");
|
||||
try testMarkdownHTML("<a href=\"https://lightpanda.io\">Lightpanda</a>", "[Lightpanda](https://lightpanda.io)\n");
|
||||
}
|
||||
|
||||
test "browser.markdown: images" {
|
||||
try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "\n");
|
||||
try testMarkdownHTML("<img src=\"logo.png\" alt=\"Logo\">", "\n");
|
||||
}
|
||||
|
||||
test "browser.markdown: headings" {
|
||||
@@ -603,7 +565,7 @@ test "browser.markdown: block link" {
|
||||
\\### Title
|
||||
\\
|
||||
\\Description
|
||||
\\([](https://example.com))
|
||||
\\([Link](https://example.com))
|
||||
\\
|
||||
);
|
||||
}
|
||||
@@ -626,8 +588,8 @@ test "browser.markdown: standalone anchors" {
|
||||
\\ <a href="2">Link 2</a>
|
||||
\\</main>
|
||||
,
|
||||
\\[Link 1](http://localhost/1)
|
||||
\\[Link 2](http://localhost/2)
|
||||
\\[Link 1](1)
|
||||
\\[Link 2](2)
|
||||
\\
|
||||
);
|
||||
}
|
||||
@@ -639,58 +601,7 @@ test "browser.markdown: mixed anchors in main" {
|
||||
\\ Welcome <a href="1">Link 1</a>.
|
||||
\\</main>
|
||||
,
|
||||
\\Welcome [Link 1](http://localhost/1).
|
||||
\\Welcome [Link 1](1).
|
||||
\\
|
||||
);
|
||||
}
|
||||
|
||||
test "browser.markdown: skip empty links" {
|
||||
try testMarkdownHTML(
|
||||
\\<a href="/"></a>
|
||||
\\<a href="/"><svg></svg></a>
|
||||
,
|
||||
\\[](http://localhost/)
|
||||
\\[](http://localhost/)
|
||||
\\
|
||||
);
|
||||
}
|
||||
|
||||
test "browser.markdown: resolve links" {
|
||||
const testing = @import("../testing.zig");
|
||||
const page = try testing.test_session.createPage();
|
||||
defer testing.test_session.removePage();
|
||||
page.url = "https://example.com/a/index.html";
|
||||
|
||||
const doc = page.window._document;
|
||||
const div = try doc.createElement("div", null, page);
|
||||
try page.parseHtmlAsChildren(div.asNode(),
|
||||
\\<a href="b">Link</a>
|
||||
\\<img src="../c.png" alt="Img">
|
||||
\\<a href="/my page">Space</a>
|
||||
);
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{}, &aw.writer, page);
|
||||
|
||||
try testing.expectString(
|
||||
\\[Link](https://example.com/a/b)
|
||||
\\
|
||||
\\[Space](https://example.com/my%20page)
|
||||
\\
|
||||
, aw.written());
|
||||
}
|
||||
|
||||
test "browser.markdown: anchor fallback label" {
|
||||
try testMarkdownHTML(
|
||||
\\<a href="/discord" aria-label="Discord Server"><svg></svg></a>
|
||||
, "[Discord Server](http://localhost/discord)\n");
|
||||
|
||||
try testMarkdownHTML(
|
||||
\\<a href="/search" title="Search Site"><svg></svg></a>
|
||||
, "[Search Site](http://localhost/search)\n");
|
||||
|
||||
try testMarkdownHTML(
|
||||
\\<a href="/no-label"><svg></svg></a>
|
||||
, "[](http://localhost/no-label)\n");
|
||||
}
|
||||
|
||||
@@ -31,7 +31,6 @@ const Mode = enum {
|
||||
|
||||
pub fn TreeWalker(comptime mode: Mode) type {
|
||||
return struct {
|
||||
_current: ?*Node = null,
|
||||
_next: ?*Node,
|
||||
_root: *Node,
|
||||
|
||||
@@ -48,46 +47,37 @@ pub fn TreeWalker(comptime mode: Mode) type {
|
||||
|
||||
pub fn next(self: *Self) ?*Node {
|
||||
const node = self._next orelse return null;
|
||||
self._current = node;
|
||||
|
||||
if (comptime mode == .children) {
|
||||
self._next = node.nextSibling();
|
||||
self._next = Node.linkToNodeOrNull(node._child_link.next);
|
||||
return node;
|
||||
}
|
||||
|
||||
if (node.firstChild()) |child| {
|
||||
self._next = child;
|
||||
if (node._children) |children| {
|
||||
self._next = children.first();
|
||||
} else if (node._child_link.next) |n| {
|
||||
self._next = Node.linkToNode(n);
|
||||
} else {
|
||||
var current: *Node = node;
|
||||
while (current != self._root) {
|
||||
if (current.nextSibling()) |sibling| {
|
||||
self._next = sibling;
|
||||
return node;
|
||||
// No children, no next sibling - walk up until we find a next sibling or hit root
|
||||
var current = node._parent;
|
||||
while (current) |parent| {
|
||||
if (parent == self._root) {
|
||||
self._next = null;
|
||||
break;
|
||||
}
|
||||
current = current._parent orelse break;
|
||||
if (parent._child_link.next) |next_sibling| {
|
||||
self._next = Node.linkToNode(next_sibling);
|
||||
break;
|
||||
}
|
||||
current = parent._parent;
|
||||
} else {
|
||||
self._next = null;
|
||||
}
|
||||
self._next = null;
|
||||
}
|
||||
return node;
|
||||
}
|
||||
|
||||
pub fn skipChildren(self: *Self) void {
|
||||
if (comptime mode == .children) return;
|
||||
const current_node = self._current orelse return;
|
||||
|
||||
var current: *Node = current_node;
|
||||
while (current != self._root) {
|
||||
if (current.nextSibling()) |sibling| {
|
||||
self._next = sibling;
|
||||
return;
|
||||
}
|
||||
current = current._parent orelse break;
|
||||
}
|
||||
self._next = null;
|
||||
}
|
||||
|
||||
pub fn reset(self: *Self) void {
|
||||
self._current = null;
|
||||
self._next = firstNext(self._root);
|
||||
}
|
||||
|
||||
@@ -157,38 +147,3 @@ pub fn TreeWalker(comptime mode: Mode) type {
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
test "TreeWalker: skipChildren" {
|
||||
const testing = @import("../../testing.zig");
|
||||
const page = try testing.test_session.createPage();
|
||||
defer testing.test_session.removePage();
|
||||
const doc = page.window._document;
|
||||
|
||||
// <div>
|
||||
// <span>
|
||||
// <b>A</b>
|
||||
// </span>
|
||||
// <p>B</p>
|
||||
// </div>
|
||||
const div = try doc.createElement("div", null, page);
|
||||
const span = try doc.createElement("span", null, page);
|
||||
const b = try doc.createElement("b", null, page);
|
||||
const p = try doc.createElement("p", null, page);
|
||||
_ = try span.asNode().appendChild(b.asNode(), page);
|
||||
_ = try div.asNode().appendChild(span.asNode(), page);
|
||||
_ = try div.asNode().appendChild(p.asNode(), page);
|
||||
|
||||
var tw = Full.init(div.asNode(), .{});
|
||||
|
||||
// root (div)
|
||||
try testing.expect(tw.next() == div.asNode());
|
||||
|
||||
// span
|
||||
try testing.expect(tw.next() == span.asNode());
|
||||
|
||||
// skip children of span (should jump over <b> to <p>)
|
||||
tw.skipChildren();
|
||||
try testing.expect(tw.next() == p.asNode());
|
||||
|
||||
try testing.expect(tw.next() == null);
|
||||
}
|
||||
|
||||
@@ -72,14 +72,6 @@ pub fn init(label_: ?[]const u8, opts_: ?InitOpts, page: *Page) !TextDecoderStre
|
||||
};
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *TextDecoderStream) void {
|
||||
self._transform.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *TextDecoderStream, shutdown: bool, page: *Page) void {
|
||||
self._transform.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
fn decodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value, ignoreBOM: bool) !void {
|
||||
// chunk should be a Uint8Array; decode it as UTF-8 string
|
||||
const typed_array = try chunk.toZig(js.TypedArray(u8));
|
||||
@@ -119,8 +111,6 @@ pub const JsApi = struct {
|
||||
pub const name = "TextDecoderStream";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(TextDecoderStream.deinit);
|
||||
};
|
||||
|
||||
pub const constructor = bridge.constructor(TextDecoderStream.init, .{});
|
||||
|
||||
@@ -34,14 +34,6 @@ pub fn init(page: *Page) !TextEncoderStream {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *TextEncoderStream) void {
|
||||
self._transform.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *TextEncoderStream, shutdown: bool, page: *Page) void {
|
||||
self._transform.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
fn encodeTransform(controller: *TransformStream.DefaultController, chunk: js.Value) !void {
|
||||
// chunk should be a JS string; encode it as UTF-8 bytes (Uint8Array)
|
||||
const str = chunk.isString() orelse return error.InvalidChunk;
|
||||
@@ -64,8 +56,6 @@ pub const JsApi = struct {
|
||||
pub const name = "TextEncoderStream";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(TextEncoderStream.deinit);
|
||||
};
|
||||
|
||||
pub const constructor = bridge.constructor(TextEncoderStream.init, .{});
|
||||
|
||||
@@ -52,8 +52,6 @@ _pull_fn: ?js.Function.Global = null,
|
||||
_pulling: bool = false,
|
||||
_pull_again: bool = false,
|
||||
_cancel: ?Cancel = null,
|
||||
_arena: std.mem.Allocator,
|
||||
_rc: usize = 0,
|
||||
|
||||
const UnderlyingSource = struct {
|
||||
start: ?js.Function = null,
|
||||
@@ -70,18 +68,13 @@ const QueueingStrategy = struct {
|
||||
pub fn init(src_: ?UnderlyingSource, strategy_: ?QueueingStrategy, page: *Page) !*ReadableStream {
|
||||
const strategy: QueueingStrategy = strategy_ orelse .{};
|
||||
|
||||
const arena = try page.getArena(.{ .debug = "ReadableStream" });
|
||||
errdefer page.releaseArena(arena);
|
||||
|
||||
const self = try arena.create(ReadableStream);
|
||||
self.* = .{
|
||||
const self = try page._factory.create(ReadableStream{
|
||||
._page = page,
|
||||
._state = .readable,
|
||||
._arena = arena,
|
||||
._reader = null,
|
||||
._controller = undefined,
|
||||
._stored_error = null,
|
||||
};
|
||||
});
|
||||
|
||||
self._controller = try ReadableStreamDefaultController.init(self, strategy.highWaterMark, page);
|
||||
|
||||
@@ -115,23 +108,6 @@ pub fn initWithData(data: []const u8, page: *Page) !*ReadableStream {
|
||||
return stream;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *ReadableStream, _: bool, page: *Page) void {
|
||||
const rc = self._rc;
|
||||
if (comptime IS_DEBUG) {
|
||||
std.debug.assert(rc != 0);
|
||||
}
|
||||
|
||||
if (rc == 1) {
|
||||
page.releaseArena(self._arena);
|
||||
} else {
|
||||
self._rc = rc - 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *ReadableStream) void {
|
||||
self._rc += 1;
|
||||
}
|
||||
|
||||
pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader {
|
||||
if (self.getLocked()) {
|
||||
return error.ReaderLocked;
|
||||
@@ -144,12 +120,6 @@ pub fn getReader(self: *ReadableStream, page: *Page) !*ReadableStreamDefaultRead
|
||||
|
||||
pub fn releaseReader(self: *ReadableStream) void {
|
||||
self._reader = null;
|
||||
|
||||
const rc = self._rc;
|
||||
if (comptime IS_DEBUG) {
|
||||
std.debug.assert(rc != 0);
|
||||
}
|
||||
self._rc = rc - 1;
|
||||
}
|
||||
|
||||
pub fn getAsyncIterator(self: *ReadableStream, page: *Page) !*AsyncIterator {
|
||||
@@ -397,8 +367,6 @@ pub const JsApi = struct {
|
||||
pub const name = "ReadableStream";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(ReadableStream.deinit);
|
||||
};
|
||||
|
||||
pub const constructor = bridge.constructor(ReadableStream.init, .{});
|
||||
@@ -422,14 +390,6 @@ pub const AsyncIterator = struct {
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *AsyncIterator) void {
|
||||
self._stream.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *AsyncIterator, shutdown: bool, page: *Page) void {
|
||||
self._stream.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
pub fn next(self: *AsyncIterator, page: *Page) !js.Promise {
|
||||
return self._reader.read(page);
|
||||
}
|
||||
@@ -446,8 +406,6 @@ pub const AsyncIterator = struct {
|
||||
pub const name = "ReadableStreamAsyncIterator";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(AsyncIterator.deinit);
|
||||
};
|
||||
|
||||
pub const next = bridge.function(ReadableStream.AsyncIterator.next, .{});
|
||||
|
||||
@@ -27,8 +27,6 @@ const ReadableStreamDefaultReader = @import("ReadableStreamDefaultReader.zig");
|
||||
|
||||
const IS_DEBUG = @import("builtin").mode == .Debug;
|
||||
|
||||
/// ReadableStreamDefaultController uses ReadableStream's arena to make
|
||||
/// allocation. Indeed, the controller is owned by its ReadableStream.
|
||||
const ReadableStreamDefaultController = @This();
|
||||
|
||||
pub const Chunk = union(enum) {
|
||||
@@ -48,6 +46,7 @@ pub const Chunk = union(enum) {
|
||||
|
||||
_page: *Page,
|
||||
_stream: *ReadableStream,
|
||||
_arena: std.mem.Allocator,
|
||||
_queue: std.ArrayList(Chunk),
|
||||
_pending_reads: std.ArrayList(js.PromiseResolver.Global),
|
||||
_high_water_mark: u32,
|
||||
@@ -57,22 +56,15 @@ pub fn init(stream: *ReadableStream, high_water_mark: u32, page: *Page) !*Readab
|
||||
._page = page,
|
||||
._queue = .empty,
|
||||
._stream = stream,
|
||||
._arena = page.arena,
|
||||
._pending_reads = .empty,
|
||||
._high_water_mark = high_water_mark,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *ReadableStreamDefaultController) void {
|
||||
self._stream.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *ReadableStreamDefaultController, shutdown: bool, page: *Page) void {
|
||||
self._stream.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
pub fn addPendingRead(self: *ReadableStreamDefaultController, page: *Page) !js.Promise {
|
||||
const resolver = page.js.local.?.createPromiseResolver();
|
||||
try self._pending_reads.append(self._stream._arena, try resolver.persist());
|
||||
try self._pending_reads.append(self._arena, try resolver.persist());
|
||||
return resolver.promise();
|
||||
}
|
||||
|
||||
@@ -82,8 +74,8 @@ pub fn enqueue(self: *ReadableStreamDefaultController, chunk: Chunk) !void {
|
||||
}
|
||||
|
||||
if (self._pending_reads.items.len == 0) {
|
||||
const chunk_copy = try chunk.dupe(self._stream._arena);
|
||||
return self._queue.append(self._stream._arena, chunk_copy);
|
||||
const chunk_copy = try chunk.dupe(self._page.arena);
|
||||
return self._queue.append(self._arena, chunk_copy);
|
||||
}
|
||||
|
||||
// I know, this is ouch! But we expect to have very few (if any)
|
||||
@@ -117,7 +109,7 @@ pub fn enqueueValue(self: *ReadableStreamDefaultController, value: js.Value) !vo
|
||||
|
||||
if (self._pending_reads.items.len == 0) {
|
||||
const persisted = try value.persist();
|
||||
try self._queue.append(self._stream._arena, .{ .js_value = persisted });
|
||||
try self._queue.append(self._arena, .{ .js_value = persisted });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -178,7 +170,7 @@ pub fn doError(self: *ReadableStreamDefaultController, err: []const u8) !void {
|
||||
}
|
||||
|
||||
self._stream._state = .errored;
|
||||
self._stream._stored_error = try self._stream._arena.dupe(u8, err);
|
||||
self._stream._stored_error = try self._page.arena.dupe(u8, err);
|
||||
|
||||
// Reject all pending reads
|
||||
for (self._pending_reads.items) |resolver| {
|
||||
@@ -218,8 +210,6 @@ pub const JsApi = struct {
|
||||
pub const name = "ReadableStreamDefaultController";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(ReadableStreamDefaultController.deinit);
|
||||
};
|
||||
|
||||
pub const enqueue = bridge.function(ReadableStreamDefaultController.enqueueValue, .{});
|
||||
|
||||
@@ -19,8 +19,6 @@
|
||||
const std = @import("std");
|
||||
const js = @import("../../js/js.zig");
|
||||
|
||||
const IS_DEBUG = @import("builtin").mode == .Debug;
|
||||
|
||||
const Page = @import("../../Page.zig");
|
||||
const ReadableStream = @import("ReadableStream.zig");
|
||||
const ReadableStreamDefaultController = @import("ReadableStreamDefaultController.zig");
|
||||
@@ -37,21 +35,6 @@ pub fn init(stream: *ReadableStream, page: *Page) !*ReadableStreamDefaultReader
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *ReadableStreamDefaultReader) void {
|
||||
const stream = self._stream orelse {
|
||||
if (comptime IS_DEBUG) {
|
||||
std.debug.assert(false);
|
||||
}
|
||||
return;
|
||||
};
|
||||
stream.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *ReadableStreamDefaultReader, shutdown: bool, page: *Page) void {
|
||||
const stream = self._stream orelse return;
|
||||
stream.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
pub const ReadResult = struct {
|
||||
done: bool,
|
||||
value: Chunk,
|
||||
@@ -127,8 +110,6 @@ pub const JsApi = struct {
|
||||
pub const name = "ReadableStreamDefaultReader";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(ReadableStreamDefaultReader.deinit);
|
||||
};
|
||||
|
||||
pub const read = bridge.function(ReadableStreamDefaultReader.read, .{});
|
||||
|
||||
@@ -85,14 +85,6 @@ pub fn initWithZigTransform(zig_transform: ZigTransformFn, page: *Page) !*Transf
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *TransformStream) void {
|
||||
self._readable.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *TransformStream, shutdown: bool, page: *Page) void {
|
||||
self._readable.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
pub fn transformWrite(self: *TransformStream, chunk: js.Value, page: *Page) !void {
|
||||
if (self._controller._zig_transform_fn) |zig_fn| {
|
||||
// Zig-level transform (used by TextEncoderStream etc.)
|
||||
@@ -138,8 +130,6 @@ pub const JsApi = struct {
|
||||
pub const name = "TransformStream";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(TransformStream.deinit);
|
||||
};
|
||||
|
||||
pub const constructor = bridge.constructor(TransformStream.init, .{});
|
||||
@@ -175,14 +165,6 @@ pub const TransformStreamDefaultController = struct {
|
||||
});
|
||||
}
|
||||
|
||||
pub fn acquireRef(self: *TransformStreamDefaultController) void {
|
||||
self._stream.acquireRef();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *TransformStreamDefaultController, shutdown: bool, page: *Page) void {
|
||||
self._stream.deinit(shutdown, page);
|
||||
}
|
||||
|
||||
pub fn enqueue(self: *TransformStreamDefaultController, chunk: ReadableStreamDefaultController.Chunk) !void {
|
||||
try self._stream._readable._controller.enqueue(chunk);
|
||||
}
|
||||
@@ -207,8 +189,6 @@ pub const TransformStreamDefaultController = struct {
|
||||
pub const name = "TransformStreamDefaultController";
|
||||
pub const prototype_chain = bridge.prototypeChain();
|
||||
pub var class_id: bridge.ClassId = undefined;
|
||||
pub const weak = true;
|
||||
pub const finalizer = bridge.finalizer(TransformStreamDefaultController.deinit);
|
||||
};
|
||||
|
||||
pub const enqueue = bridge.function(TransformStreamDefaultController.enqueueValue, .{});
|
||||
|
||||
@@ -228,8 +228,6 @@ pub extern fn X25519_keypair(out_public_value: *[32]u8, out_private_key: *[32]u8
|
||||
|
||||
pub const NID_X25519 = @as(c_int, 948);
|
||||
pub const EVP_PKEY_X25519 = NID_X25519;
|
||||
pub const NID_ED25519 = 949;
|
||||
pub const EVP_PKEY_ED25519 = NID_ED25519;
|
||||
|
||||
pub extern fn EVP_PKEY_new_raw_private_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
|
||||
pub extern fn EVP_PKEY_new_raw_public_key(@"type": c_int, unused: ?*ENGINE, in: [*c]const u8, len: usize) [*c]EVP_PKEY;
|
||||
@@ -238,11 +236,3 @@ pub extern fn EVP_PKEY_CTX_free(ctx: ?*EVP_PKEY_CTX) void;
|
||||
pub extern fn EVP_PKEY_derive_init(ctx: ?*EVP_PKEY_CTX) c_int;
|
||||
pub extern fn EVP_PKEY_derive(ctx: ?*EVP_PKEY_CTX, key: [*c]u8, out_key_len: [*c]usize) c_int;
|
||||
pub extern fn EVP_PKEY_derive_set_peer(ctx: ?*EVP_PKEY_CTX, peer: [*c]EVP_PKEY) c_int;
|
||||
pub extern fn EVP_PKEY_free(pkey: ?*EVP_PKEY) void;
|
||||
|
||||
pub extern fn EVP_DigestSignInit(ctx: ?*EVP_MD_CTX, pctx: ?*?*EVP_PKEY_CTX, typ: ?*const EVP_MD, e: ?*ENGINE, pkey: ?*EVP_PKEY) c_int;
|
||||
pub extern fn EVP_DigestSign(ctx: ?*EVP_MD_CTX, sig: [*c]u8, sig_len: *usize, data: [*c]const u8, data_len: usize) c_int;
|
||||
pub extern fn EVP_MD_CTX_new() ?*EVP_MD_CTX;
|
||||
pub extern fn EVP_MD_CTX_free(ctx: ?*EVP_MD_CTX) void;
|
||||
pub const struct_evp_md_ctx_st = opaque {};
|
||||
pub const EVP_MD_CTX = struct_evp_md_ctx_st;
|
||||
|
||||
@@ -29,8 +29,6 @@ const Notification = @import("../Notification.zig");
|
||||
const CookieJar = @import("../browser/webapi/storage/Cookie.zig").Jar;
|
||||
const Robots = @import("../browser/Robots.zig");
|
||||
const RobotStore = Robots.RobotStore;
|
||||
const WebBotAuth = @import("../browser/WebBotAuth.zig");
|
||||
|
||||
const posix = std.posix;
|
||||
|
||||
const Allocator = std.mem.Allocator;
|
||||
@@ -85,9 +83,6 @@ robot_store: *RobotStore,
|
||||
// Allows us to fetch the robots.txt just once.
|
||||
pending_robots_queue: std.StringHashMapUnmanaged(std.ArrayList(Request)) = .empty,
|
||||
|
||||
// Reference to the App-owned WebBotAuth.
|
||||
web_bot_auth: *const ?WebBotAuth,
|
||||
|
||||
// Once we have a handle/easy to process a request with, we create a Transfer
|
||||
// which contains the Request as well as any state we need to process the
|
||||
// request. These wil come and go with each request.
|
||||
@@ -126,13 +121,7 @@ pub const CDPClient = struct {
|
||||
|
||||
const TransferQueue = std.DoublyLinkedList;
|
||||
|
||||
pub fn init(
|
||||
allocator: Allocator,
|
||||
ca_blob: ?Net.Blob,
|
||||
robot_store: *RobotStore,
|
||||
web_bot_auth: *const ?WebBotAuth,
|
||||
config: *const Config,
|
||||
) !*Client {
|
||||
pub fn init(allocator: Allocator, ca_blob: ?Net.Blob, robot_store: *RobotStore, config: *const Config) !*Client {
|
||||
var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator);
|
||||
errdefer transfer_pool.deinit();
|
||||
|
||||
@@ -156,7 +145,6 @@ pub fn init(
|
||||
.handles = handles,
|
||||
.allocator = allocator,
|
||||
.robot_store = robot_store,
|
||||
.web_bot_auth = web_bot_auth,
|
||||
.http_proxy = http_proxy,
|
||||
.use_proxy = http_proxy != null,
|
||||
.config = config,
|
||||
@@ -185,6 +173,14 @@ pub fn newHeaders(self: *const Client) !Net.Headers {
|
||||
return Net.Headers.init(self.config.http_headers.user_agent_header);
|
||||
}
|
||||
|
||||
pub fn attachRuntime(self: *Client, runtime: *Net.Runtime, cdp_fd: ?posix.fd_t) !void {
|
||||
try self.handles.attachRuntime(runtime, cdp_fd);
|
||||
}
|
||||
|
||||
pub fn detachRuntime(self: *Client) void {
|
||||
self.handles.detachRuntime();
|
||||
}
|
||||
|
||||
pub fn abort(self: *Client) void {
|
||||
self._abort(true, 0);
|
||||
}
|
||||
@@ -721,12 +717,6 @@ fn makeRequest(self: *Client, conn: *Net.Connection, transfer: *Transfer) anyerr
|
||||
try conn.secretHeaders(&header_list, &self.config.http_headers); // Add headers that must be hidden from intercepts
|
||||
try conn.setHeaders(&header_list);
|
||||
|
||||
// If we have WebBotAuth, sign our request.
|
||||
if (self.web_bot_auth.*) |wba| {
|
||||
const authority = URL.getHost(req.url);
|
||||
try wba.signRequest(transfer.arena.allocator(), &header_list, authority);
|
||||
}
|
||||
|
||||
// Add cookies.
|
||||
if (header_list.cookies) |cookies| {
|
||||
try conn.setCookies(cookies);
|
||||
@@ -1320,7 +1310,7 @@ pub const Transfer = struct {
|
||||
}
|
||||
transfer._redirecting = false;
|
||||
|
||||
if ((status == 401 or status == 407) and transfer.client.use_proxy) {
|
||||
if (status == 401 or status == 407) {
|
||||
// The auth challenge must be parsed from a following
|
||||
// WWW-Authenticate or Proxy-Authenticate header.
|
||||
transfer._auth_challenge = .{
|
||||
|
||||
@@ -29,7 +29,6 @@ pub const Headers = Net.Headers;
|
||||
|
||||
const Config = @import("../Config.zig");
|
||||
const RobotStore = @import("../browser/Robots.zig").RobotStore;
|
||||
const WebBotAuth = @import("../browser/WebBotAuth.zig");
|
||||
|
||||
const Allocator = std.mem.Allocator;
|
||||
const ArenaAllocator = std.heap.ArenaAllocator;
|
||||
@@ -46,14 +45,8 @@ allocator: Allocator,
|
||||
config: *const Config,
|
||||
ca_blob: ?Net.Blob,
|
||||
robot_store: *RobotStore,
|
||||
web_bot_auth: *const ?WebBotAuth,
|
||||
|
||||
pub fn init(
|
||||
allocator: Allocator,
|
||||
robot_store: *RobotStore,
|
||||
web_bot_auth: *const ?WebBotAuth,
|
||||
config: *const Config,
|
||||
) !Http {
|
||||
pub fn init(allocator: Allocator, robot_store: *RobotStore, config: *const Config) !Http {
|
||||
try Net.globalInit();
|
||||
errdefer Net.globalDeinit();
|
||||
|
||||
@@ -75,7 +68,6 @@ pub fn init(
|
||||
.config = config,
|
||||
.ca_blob = ca_blob,
|
||||
.robot_store = robot_store,
|
||||
.web_bot_auth = web_bot_auth,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -89,7 +81,7 @@ pub fn deinit(self: *Http) void {
|
||||
}
|
||||
|
||||
pub fn createClient(self: *Http, allocator: Allocator) !*Client {
|
||||
return Client.init(allocator, self.ca_blob, self.robot_store, self.web_bot_auth, self.config);
|
||||
return Client.init(allocator, self.ca_blob, self.robot_store, self.config);
|
||||
}
|
||||
|
||||
pub fn newConnection(self: *Http) !Net.Connection {
|
||||
|
||||
@@ -24,29 +24,22 @@ mutex: std.Thread.Mutex = .{},
|
||||
aw: std.io.Writer.Allocating,
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator, app: *App, writer: *std.io.Writer) !*Self {
|
||||
const http_client = try app.http.createClient(allocator);
|
||||
errdefer http_client.deinit();
|
||||
|
||||
const notification = try lp.Notification.init(allocator);
|
||||
errdefer notification.deinit();
|
||||
|
||||
const self = try allocator.create(Self);
|
||||
errdefer allocator.destroy(self);
|
||||
|
||||
var browser = try lp.Browser.init(app, .{ .http_client = http_client });
|
||||
errdefer browser.deinit();
|
||||
self.allocator = allocator;
|
||||
self.app = app;
|
||||
self.writer = writer;
|
||||
self.aw = .init(allocator);
|
||||
|
||||
self.* = .{
|
||||
.allocator = allocator,
|
||||
.app = app,
|
||||
.writer = writer,
|
||||
.browser = browser,
|
||||
.aw = .init(allocator),
|
||||
.http_client = http_client,
|
||||
.notification = notification,
|
||||
.session = undefined,
|
||||
.page = undefined,
|
||||
};
|
||||
self.http_client = try app.http.createClient(allocator);
|
||||
errdefer self.http_client.deinit();
|
||||
|
||||
self.notification = try .init(allocator);
|
||||
errdefer self.notification.deinit();
|
||||
|
||||
self.browser = try lp.Browser.init(app, .{ .http_client = self.http_client });
|
||||
errdefer self.browser.deinit();
|
||||
|
||||
self.session = try self.browser.newSession(self.notification);
|
||||
self.page = try self.session.createPage();
|
||||
|
||||
@@ -39,7 +39,32 @@ pub const CurlOffT = c.curl_off_t;
|
||||
pub const CurlDebugFunction = fn (*Curl, CurlInfoType, [*c]u8, usize, *anyopaque) c_int;
|
||||
pub const CurlHeaderFunction = fn ([*]const u8, usize, usize, *anyopaque) usize;
|
||||
pub const CurlWriteFunction = fn ([*]const u8, usize, usize, *anyopaque) usize;
|
||||
pub const CurlSocketCallback = fn (?*Curl, CurlSocket, c_int, ?*anyopaque, ?*anyopaque) callconv(.c) c_int;
|
||||
pub const CurlTimerCallback = fn (?*CurlM, c_long, ?*anyopaque) callconv(.c) c_int;
|
||||
pub const curl_writefunc_error: usize = c.CURL_WRITEFUNC_ERROR;
|
||||
pub const CURL_SOCKET_TIMEOUT: CurlSocket = c.CURL_SOCKET_TIMEOUT;
|
||||
|
||||
pub const CurlPoll = enum(c_int) {
|
||||
in = c.CURL_POLL_IN,
|
||||
out = c.CURL_POLL_OUT,
|
||||
inout = c.CURL_POLL_INOUT,
|
||||
remove = c.CURL_POLL_REMOVE,
|
||||
};
|
||||
|
||||
pub const CurlSelectMask = packed struct(c_int) {
|
||||
in: bool = false,
|
||||
out: bool = false,
|
||||
err: bool = false,
|
||||
_reserved: std.meta.Int(.unsigned, @bitSizeOf(c_int) - 3) = 0,
|
||||
|
||||
pub fn toC(self: @This()) c_int {
|
||||
var mask: c_int = 0;
|
||||
if (self.in) mask |= c.CURL_CSELECT_IN;
|
||||
if (self.out) mask |= c.CURL_CSELECT_OUT;
|
||||
if (self.err) mask |= c.CURL_CSELECT_ERR;
|
||||
return mask;
|
||||
}
|
||||
};
|
||||
|
||||
pub const CurlGlobalFlags = packed struct(u8) {
|
||||
ssl: bool = false,
|
||||
@@ -156,6 +181,10 @@ pub const CurlOption = enum(c.CURLoption) {
|
||||
|
||||
pub const CurlMOption = enum(c.CURLMoption) {
|
||||
max_host_connections = c.CURLMOPT_MAX_HOST_CONNECTIONS,
|
||||
socket_function = c.CURLMOPT_SOCKETFUNCTION,
|
||||
socket_data = c.CURLMOPT_SOCKETDATA,
|
||||
timer_function = c.CURLMOPT_TIMERFUNCTION,
|
||||
timer_data = c.CURLMOPT_TIMERDATA,
|
||||
};
|
||||
|
||||
pub const CurlInfo = enum(c.CURLINFO) {
|
||||
@@ -675,6 +704,10 @@ pub fn curl_multi_setopt(multi: *CurlM, comptime option: CurlMOption, value: any
|
||||
};
|
||||
break :blk c.curl_multi_setopt(multi, opt, n);
|
||||
},
|
||||
.socket_function => c.curl_multi_setopt(multi, opt, value),
|
||||
.socket_data => c.curl_multi_setopt(multi, opt, value),
|
||||
.timer_function => c.curl_multi_setopt(multi, opt, value),
|
||||
.timer_data => c.curl_multi_setopt(multi, opt, value),
|
||||
};
|
||||
try errorMCheck(code);
|
||||
}
|
||||
@@ -701,6 +734,15 @@ pub fn curl_multi_poll(
|
||||
try errorMCheck(c.curl_multi_poll(multi, raw_fds, @intCast(extra_fds.len), timeout_ms, numfds));
|
||||
}
|
||||
|
||||
pub fn curl_multi_socket_action(
|
||||
multi: *CurlM,
|
||||
s: CurlSocket,
|
||||
ev_bitmask: c_int,
|
||||
running_handles: *c_int,
|
||||
) ErrorMulti!void {
|
||||
try errorMCheck(c.curl_multi_socket_action(multi, s, ev_bitmask, running_handles));
|
||||
}
|
||||
|
||||
pub fn curl_multi_info_read(multi: *CurlM, msgs_in_queue: *c_int) ?CurlMsg {
|
||||
const ptr = c.curl_multi_info_read(multi, msgs_in_queue);
|
||||
if (ptr == null) return null;
|
||||
|
||||
Reference in New Issue
Block a user