From b0fe5d60ab72574d21a0a0e3ecc305609e9a9805 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 29 Jul 2025 21:26:59 +0800 Subject: [PATCH 01/44] Initial work on integrating libcurl and making all http nonblocking --- .gitignore | 1 + .gitmodules | 12 + build.zig | 615 ++++- build.zig.zon | 4 - src/app.zig | 13 +- src/browser/ScriptManager.zig | 543 ++++ src/browser/browser.zig | 3 +- src/browser/mime.zig | 85 +- src/browser/netsurf.zig | 41 +- src/browser/page.zig | 923 +++---- src/browser/session.zig | 1 + src/browser/storage/cookie.zig | 24 +- src/browser/xhr/xhr.zig | 42 +- src/cdp/domains/network.zig | 33 +- src/cdp/domains/page.zig | 4 +- src/data/public_suffix_list.zig | 3 +- src/http/ca_certs.zig | 93 + src/http/client.zig | 4351 ++++--------------------------- src/http/errors.zig | 238 ++ src/main.zig | 89 +- src/notification.zig | 6 +- src/runtime/loop.zig | 4 + src/telemetry/lightpanda.zig | 4 +- src/testing.zig | 4 +- src/url.zig | 153 +- vendor/curl | 1 + vendor/mbedtls | 1 + vendor/nghttp2 | 1 + vendor/zlib | 1 + 29 files changed, 2570 insertions(+), 4723 deletions(-) create mode 100644 src/browser/ScriptManager.zig create mode 100644 src/http/ca_certs.zig create mode 100644 src/http/errors.zig create mode 160000 vendor/curl create mode 160000 vendor/mbedtls create mode 160000 vendor/nghttp2 create mode 160000 vendor/zlib diff --git a/.gitignore b/.gitignore index ad9ae7b4..579000fd 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ zig-out /vendor/libiconv/ lightpanda.id /v8/ +/vendor/nghttp2/lib/includes/nghttp2/nghttp2ver.h diff --git a/.gitmodules b/.gitmodules index f025f0bd..01e8ca81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,15 @@ [submodule "vendor/mimalloc"] path = vendor/mimalloc url = https://github.com/microsoft/mimalloc.git/ +[submodule "vendor/nghttp2"] + path = vendor/nghttp2 + url = https://github.com/nghttp2/nghttp2.git +[submodule "vendor/mbedtls"] + path = vendor/mbedtls + url = https://github.com/Mbed-TLS/mbedtls.git +[submodule "vendor/zlib"] + path = vendor/zlib + url = https://github.com/madler/zlib.git +[submodule "vendor/curl"] + path = vendor/curl + url = https://github.com/curl/curl.git diff --git a/build.zig b/build.zig index 6f3906f3..e3c025d9 100644 --- a/build.zig +++ b/build.zig @@ -19,11 +19,13 @@ const std = @import("std"); const builtin = @import("builtin"); +const Build = std.Build; + /// Do not rename this constant. It is scanned by some scripts to determine /// which zig version to install. const recommended_zig_version = "0.14.1"; -pub fn build(b: *std.Build) !void { +pub fn build(b: *Build) !void { switch (comptime builtin.zig_version.order(std.SemanticVersion.parse(recommended_zig_version) catch unreachable)) { .eq => {}, .lt => { @@ -138,29 +140,28 @@ pub fn build(b: *std.Build) !void { } } -fn common(b: *std.Build, opts: *std.Build.Step.Options, step: *std.Build.Step.Compile) !void { +fn common(b: *Build, opts: *Build.Step.Options, step: *Build.Step.Compile) !void { const mod = step.root_module; const target = mod.resolved_target.?; const optimize = mod.optimize.?; const dep_opts = .{ .target = target, .optimize = optimize }; try moduleNetSurf(b, step, target); - mod.addImport("tls", b.dependency("tls", dep_opts).module("tls")); + mod.addImport("build_config", opts.createModule()); mod.addImport("tigerbeetle-io", b.dependency("tigerbeetle_io", .{}).module("tigerbeetle_io")); { // v8 + mod.link_libcpp = true; + const v8_opts = b.addOptions(); v8_opts.addOption(bool, "inspector_subtype", false); const v8_mod = b.dependency("v8", dep_opts).module("v8"); v8_mod.addOptions("default_exports", v8_opts); mod.addImport("v8", v8_mod); - } - mod.link_libcpp = true; - { const release_dir = if (mod.optimize.? == .Debug) "debug" else "release"; const os = switch (target.result.os.tag) { .linux => "linux", @@ -181,21 +182,210 @@ fn common(b: *std.Build, opts: *std.Build.Step.Options, step: *std.Build.Step.Co ); }; mod.addObjectFile(mod.owner.path(lib_path)); + + switch (target.result.os.tag) { + .macos => { + // v8 has a dependency, abseil-cpp, which, on Mac, uses CoreFoundation + mod.addSystemFrameworkPath(.{ .cwd_relative = "/System/Library/Frameworks" }); + mod.linkFramework("CoreFoundation", .{}); + }, + else => {}, + } } - switch (target.result.os.tag) { - .macos => { - // v8 has a dependency, abseil-cpp, which, on Mac, uses CoreFoundation - mod.addSystemFrameworkPath(.{ .cwd_relative = "/System/Library/Frameworks" }); - mod.linkFramework("CoreFoundation", .{}); - }, - else => {}, - } + { + //curl + { + const is_linux = target.result.os.tag == .linux; + if (is_linux) { + mod.addCMacro("HAVE_LINUX_TCP_H", "1"); + mod.addCMacro("HAVE_MSG_NOSIGNAL", "1"); + mod.addCMacro("HAVE_GETHOSTBYNAME_R", "1"); + } + mod.addCMacro("_FILE_OFFSET_BITS", "64"); + mod.addCMacro("BUILDING_LIBCURL", "1"); + mod.addCMacro("CURL_DISABLE_AWS", "1"); + mod.addCMacro("CURL_DISABLE_DICT", "1"); + mod.addCMacro("CURL_DISABLE_DOH", "1"); + mod.addCMacro("CURL_DISABLE_FILE", "1"); + mod.addCMacro("CURL_DISABLE_FTP", "1"); + mod.addCMacro("CURL_DISABLE_GOPHER", "1"); + mod.addCMacro("CURL_DISABLE_KERBEROS", "1"); + mod.addCMacro("CURL_DISABLE_IMAP", "1"); + mod.addCMacro("CURL_DISABLE_IPFS", "1"); + mod.addCMacro("CURL_DISABLE_LDAP", "1"); + mod.addCMacro("CURL_DISABLE_LDAPS", "1"); + mod.addCMacro("CURL_DISABLE_MQTT", "1"); + mod.addCMacro("CURL_DISABLE_NTLM", "1"); + mod.addCMacro("CURL_DISABLE_PROGRESS_METER", "1"); + mod.addCMacro("CURL_DISABLE_POP3", "1"); + mod.addCMacro("CURL_DISABLE_RTSP", "1"); + mod.addCMacro("CURL_DISABLE_SMB", "1"); + mod.addCMacro("CURL_DISABLE_SMTP", "1"); + mod.addCMacro("CURL_DISABLE_TELNET", "1"); + mod.addCMacro("CURL_DISABLE_TFTP", "1"); + mod.addCMacro("CURL_EXTERN_SYMBOL", "__attribute__ ((__visibility__ (\"default\"))"); + mod.addCMacro("CURL_OS", if (is_linux) "\"Linux\"" else "\"mac\""); + mod.addCMacro("CURL_STATICLIB", "1"); + mod.addCMacro("ENABLE_IPV6", "1"); + mod.addCMacro("HAVE_ALARM", "1"); + mod.addCMacro("HAVE_ALLOCA_H", "1"); + mod.addCMacro("HAVE_ARPA_INET_H", "1"); + mod.addCMacro("HAVE_ARPA_TFTP_H", "1"); + mod.addCMacro("HAVE_ASSERT_H", "1"); + mod.addCMacro("HAVE_BASENAME", "1"); + mod.addCMacro("HAVE_BOOL_T", "1"); + mod.addCMacro("HAVE_BUILTIN_AVAILABLE", "1"); + mod.addCMacro("HAVE_CLOCK_GETTIME_MONOTONIC", "1"); + mod.addCMacro("HAVE_DLFCN_H", "1"); + mod.addCMacro("HAVE_ERRNO_H", "1"); + mod.addCMacro("HAVE_FCNTL", "1"); + mod.addCMacro("HAVE_FCNTL_H", "1"); + mod.addCMacro("HAVE_FCNTL_O_NONBLOCK", "1"); + mod.addCMacro("HAVE_FREEADDRINFO", "1"); + mod.addCMacro("HAVE_FSETXATTR", "1"); + mod.addCMacro("HAVE_FSETXATTR_5", "1"); + mod.addCMacro("HAVE_FTRUNCATE", "1"); + mod.addCMacro("HAVE_GETADDRINFO", "1"); + mod.addCMacro("HAVE_GETEUID", "1"); + mod.addCMacro("HAVE_GETHOSTBYNAME", "1"); + mod.addCMacro("HAVE_GETHOSTBYNAME_R_6", "1"); + mod.addCMacro("HAVE_GETHOSTNAME", "1"); + mod.addCMacro("HAVE_GETPEERNAME", "1"); + mod.addCMacro("HAVE_GETPPID", "1"); + mod.addCMacro("HAVE_GETPPID", "1"); + mod.addCMacro("HAVE_GETPROTOBYNAME", "1"); + mod.addCMacro("HAVE_GETPWUID", "1"); + mod.addCMacro("HAVE_GETPWUID_R", "1"); + mod.addCMacro("HAVE_GETRLIMIT", "1"); + mod.addCMacro("HAVE_GETSOCKNAME", "1"); + mod.addCMacro("HAVE_GETTIMEOFDAY", "1"); + mod.addCMacro("HAVE_GMTIME_R", "1"); + mod.addCMacro("HAVE_IDN2_H", "1"); + mod.addCMacro("HAVE_IF_NAMETOINDEX", "1"); + mod.addCMacro("HAVE_IFADDRS_H", "1"); + mod.addCMacro("HAVE_INET_ADDR", "1"); + mod.addCMacro("HAVE_INET_PTON", "1"); + mod.addCMacro("HAVE_INTTYPES_H", "1"); + mod.addCMacro("HAVE_IOCTL", "1"); + mod.addCMacro("HAVE_IOCTL_FIONBIO", "1"); + mod.addCMacro("HAVE_IOCTL_SIOCGIFADDR", "1"); + mod.addCMacro("HAVE_LDAP_URL_PARSE", "1"); + mod.addCMacro("HAVE_LIBGEN_H", "1"); + mod.addCMacro("HAVE_LIBZ", "1"); + mod.addCMacro("HAVE_LL", "1"); + mod.addCMacro("HAVE_LOCALE_H", "1"); + mod.addCMacro("HAVE_LOCALTIME_R", "1"); + mod.addCMacro("HAVE_LONGLONG", "1"); + mod.addCMacro("HAVE_MALLOC_H", "1"); + mod.addCMacro("HAVE_MEMORY_H", "1"); + mod.addCMacro("HAVE_NET_IF_H", "1"); + mod.addCMacro("HAVE_NETDB_H", "1"); + mod.addCMacro("HAVE_NETINET_IN_H", "1"); + mod.addCMacro("HAVE_NETINET_TCP_H", "1"); + mod.addCMacro("HAVE_PIPE", "1"); + mod.addCMacro("HAVE_POLL", "1"); + mod.addCMacro("HAVE_POLL_FINE", "1"); + mod.addCMacro("HAVE_POLL_H", "1"); + mod.addCMacro("HAVE_POSIX_STRERROR_R", "1"); + mod.addCMacro("HAVE_PTHREAD_H", "1"); + mod.addCMacro("HAVE_PWD_H", "1"); + mod.addCMacro("HAVE_RECV", "1"); + mod.addCMacro("HAVE_SA_FAMILY_T", "1"); + mod.addCMacro("HAVE_SELECT", "1"); + mod.addCMacro("HAVE_SEND", "1"); + mod.addCMacro("HAVE_SETJMP_H", "1"); + mod.addCMacro("HAVE_SETLOCALE", "1"); + mod.addCMacro("HAVE_SETRLIMIT", "1"); + mod.addCMacro("HAVE_SETSOCKOPT", "1"); + mod.addCMacro("HAVE_SIGACTION", "1"); + mod.addCMacro("HAVE_SIGINTERRUPT", "1"); + mod.addCMacro("HAVE_SIGNAL", "1"); + mod.addCMacro("HAVE_SIGNAL_H", "1"); + mod.addCMacro("HAVE_SIGSETJMP", "1"); + mod.addCMacro("HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID", "1"); + mod.addCMacro("HAVE_SOCKET", "1"); + mod.addCMacro("HAVE_STDBOOL_H", "1"); + mod.addCMacro("HAVE_STDINT_H", "1"); + mod.addCMacro("HAVE_STDIO_H", "1"); + mod.addCMacro("HAVE_STDLIB_H", "1"); + mod.addCMacro("HAVE_STRCASECMP", "1"); + mod.addCMacro("HAVE_STRDUP", "1"); + mod.addCMacro("HAVE_STRERROR_R", "1"); + mod.addCMacro("HAVE_STRING_H", "1"); + mod.addCMacro("HAVE_STRINGS_H", "1"); + mod.addCMacro("HAVE_STRSTR", "1"); + mod.addCMacro("HAVE_STRTOK_R", "1"); + mod.addCMacro("HAVE_STRTOLL", "1"); + mod.addCMacro("HAVE_STRUCT_SOCKADDR_STORAGE", "1"); + mod.addCMacro("HAVE_STRUCT_TIMEVAL", "1"); + mod.addCMacro("HAVE_SYS_IOCTL_H", "1"); + mod.addCMacro("HAVE_SYS_PARAM_H", "1"); + mod.addCMacro("HAVE_SYS_POLL_H", "1"); + mod.addCMacro("HAVE_SYS_RESOURCE_H", "1"); + mod.addCMacro("HAVE_SYS_SELECT_H", "1"); + mod.addCMacro("HAVE_SYS_SOCKET_H", "1"); + mod.addCMacro("HAVE_SYS_STAT_H", "1"); + mod.addCMacro("HAVE_SYS_TIME_H", "1"); + mod.addCMacro("HAVE_SYS_TYPES_H", "1"); + mod.addCMacro("HAVE_SYS_UIO_H", "1"); + mod.addCMacro("HAVE_SYS_UN_H", "1"); + mod.addCMacro("HAVE_TERMIO_H", "1"); + mod.addCMacro("HAVE_TERMIOS_H", "1"); + mod.addCMacro("HAVE_TIME_H", "1"); + mod.addCMacro("HAVE_UNAME", "1"); + mod.addCMacro("HAVE_UNISTD_H", "1"); + mod.addCMacro("HAVE_UTIME", "1"); + mod.addCMacro("HAVE_UTIME_H", "1"); + mod.addCMacro("HAVE_UTIMES", "1"); + mod.addCMacro("HAVE_VARIADIC_MACROS_C99", "1"); + mod.addCMacro("HAVE_VARIADIC_MACROS_GCC", "1"); + mod.addCMacro("HAVE_ZLIB_H", "1"); + mod.addCMacro("RANDOM_FILE", "\"/dev/urandom\""); + mod.addCMacro("RECV_TYPE_ARG1", "int"); + mod.addCMacro("RECV_TYPE_ARG2", "void *"); + mod.addCMacro("RECV_TYPE_ARG3", "size_t"); + mod.addCMacro("RECV_TYPE_ARG4", "int"); + mod.addCMacro("RECV_TYPE_RETV", "ssize_t"); + mod.addCMacro("SEND_QUAL_ARG2", "const"); + mod.addCMacro("SEND_TYPE_ARG1", "int"); + mod.addCMacro("SEND_TYPE_ARG2", "void *"); + mod.addCMacro("SEND_TYPE_ARG3", "size_t"); + mod.addCMacro("SEND_TYPE_ARG4", "int"); + mod.addCMacro("SEND_TYPE_RETV", "ssize_t"); + mod.addCMacro("SIZEOF_CURL_OFF_T", "8"); + mod.addCMacro("SIZEOF_INT", "4"); + mod.addCMacro("SIZEOF_LONG", "8"); + mod.addCMacro("SIZEOF_OFF_T", "8"); + mod.addCMacro("SIZEOF_SHORT", "2"); + mod.addCMacro("SIZEOF_SIZE_T", "8"); + mod.addCMacro("SIZEOF_TIME_T", "8"); + mod.addCMacro("STDC_HEADERS", "1"); + mod.addCMacro("TIME_WITH_SYS_TIME", "1"); + mod.addCMacro("USE_NGHTTP2", "1"); + mod.addCMacro("USE_MBEDTLS", "1"); + mod.addCMacro("USE_THREADS_POSIX", "1"); + mod.addCMacro("USE_UNIX_SOCKETS", "1"); + } - mod.addImport("build_config", opts.createModule()); + try buildZlib(b, mod); + try buildMbedtls(b, mod); + try buildNghttp2(b, mod); + try buildCurl(b, mod); + + switch (target.result.os.tag) { + .macos => { + // needed for proxying on mac + mod.addSystemFrameworkPath(.{ .cwd_relative = "/System/Library/Frameworks" }); + mod.linkFramework("CoreFoundation", .{}); + mod.linkFramework("SystemConfiguration", .{}); + }, + else => {}, + } + } } -fn moduleNetSurf(b: *std.Build, step: *std.Build.Step.Compile, target: std.Build.ResolvedTarget) !void { +fn moduleNetSurf(b: *Build, step: *Build.Step.Compile, target: std.Build.ResolvedTarget) !void { const os = target.result.os.tag; const arch = target.result.cpu.arch; @@ -250,3 +440,396 @@ fn moduleNetSurf(b: *std.Build, step: *std.Build.Step.Compile, target: std.Build step.addIncludePath(b.path(ns ++ "/" ++ lib ++ "/src")); } } + +fn buildZlib(b: *Build, m: *Build.Module) !void { + const zlib = b.addLibrary(.{ + .name = "zlib", + .root_module = m, + }); + + const root = "vendor/zlib/"; + zlib.installHeader(b.path(root ++ "zlib.h"), "zlib.h"); + zlib.installHeader(b.path(root ++ "zconf.h"), "zconf.h"); + zlib.addCSourceFiles(.{ + .flags = &.{ + "-DHAVE_SYS_TYPES_H", + "-DHAVE_STDINT_H", + "-DHAVE_STDDEF_H", + }, + .files = &.{ + root ++ "adler32.c", + root ++ "compress.c", + root ++ "crc32.c", + root ++ "deflate.c", + root ++ "gzclose.c", + root ++ "gzlib.c", + root ++ "gzread.c", + root ++ "gzwrite.c", + root ++ "inflate.c", + root ++ "infback.c", + root ++ "inftrees.c", + root ++ "inffast.c", + root ++ "trees.c", + root ++ "uncompr.c", + root ++ "zutil.c", + } + }); +} + +fn buildMbedtls(b: *Build, m: *Build.Module) !void { + const mbedtls = b.addLibrary(.{ + .name = "mbedtls", + .root_module = m, + }); + + const root = "vendor/mbedtls/"; + mbedtls.addIncludePath(b.path(root ++ "include")); + mbedtls.addIncludePath(b.path(root ++ "library")); + + mbedtls.addCSourceFiles(.{ + .flags = &.{ + }, + .files = &.{ + root ++ "library/aes.c", + root ++ "library/aesni.c", + root ++ "library/aesce.c", + root ++ "library/aria.c", + root ++ "library/asn1parse.c", + root ++ "library/asn1write.c", + root ++ "library/base64.c", + root ++ "library/bignum.c", + root ++ "library/bignum_core.c", + root ++ "library/bignum_mod.c", + root ++ "library/bignum_mod_raw.c", + root ++ "library/camellia.c", + root ++ "library/ccm.c", + root ++ "library/chacha20.c", + root ++ "library/chachapoly.c", + root ++ "library/cipher.c", + root ++ "library/cipher_wrap.c", + root ++ "library/constant_time.c", + root ++ "library/cmac.c", + root ++ "library/ctr_drbg.c", + root ++ "library/des.c", + root ++ "library/dhm.c", + root ++ "library/ecdh.c", + root ++ "library/ecdsa.c", + root ++ "library/ecjpake.c", + root ++ "library/ecp.c", + root ++ "library/ecp_curves.c", + root ++ "library/entropy.c", + root ++ "library/entropy_poll.c", + root ++ "library/error.c", + root ++ "library/gcm.c", + root ++ "library/hkdf.c", + root ++ "library/hmac_drbg.c", + root ++ "library/lmots.c", + root ++ "library/lms.c", + root ++ "library/md.c", + root ++ "library/md5.c", + root ++ "library/memory_buffer_alloc.c", + root ++ "library/nist_kw.c", + root ++ "library/oid.c", + root ++ "library/padlock.c", + root ++ "library/pem.c", + root ++ "library/pk.c", + root ++ "library/pk_ecc.c", + root ++ "library/pk_wrap.c", + root ++ "library/pkcs12.c", + root ++ "library/pkcs5.c", + root ++ "library/pkparse.c", + root ++ "library/pkwrite.c", + root ++ "library/platform.c", + root ++ "library/platform_util.c", + root ++ "library/poly1305.c", + root ++ "library/psa_crypto.c", + root ++ "library/psa_crypto_aead.c", + root ++ "library/psa_crypto_cipher.c", + root ++ "library/psa_crypto_client.c", + root ++ "library/psa_crypto_ffdh.c", + root ++ "library/psa_crypto_driver_wrappers_no_static.c", + root ++ "library/psa_crypto_ecp.c", + root ++ "library/psa_crypto_hash.c", + root ++ "library/psa_crypto_mac.c", + root ++ "library/psa_crypto_pake.c", + root ++ "library/psa_crypto_rsa.c", + root ++ "library/psa_crypto_se.c", + root ++ "library/psa_crypto_slot_management.c", + root ++ "library/psa_crypto_storage.c", + root ++ "library/psa_its_file.c", + root ++ "library/psa_util.c", + root ++ "library/ripemd160.c", + root ++ "library/rsa.c", + root ++ "library/rsa_alt_helpers.c", + root ++ "library/sha1.c", + root ++ "library/sha3.c", + root ++ "library/sha256.c", + root ++ "library/sha512.c", + root ++ "library/threading.c", + root ++ "library/timing.c", + root ++ "library/version.c", + root ++ "library/version_features.c", + root ++ "library/pkcs7.c", + root ++ "library/x509.c", + root ++ "library/x509_create.c", + root ++ "library/x509_crl.c", + root ++ "library/x509_crt.c", + root ++ "library/x509_csr.c", + root ++ "library/x509write.c", + root ++ "library/x509write_crt.c", + root ++ "library/x509write_csr.c", + root ++ "library/debug.c", + root ++ "library/mps_reader.c", + root ++ "library/mps_trace.c", + root ++ "library/net_sockets.c", + root ++ "library/ssl_cache.c", + root ++ "library/ssl_ciphersuites.c", + root ++ "library/ssl_client.c", + root ++ "library/ssl_cookie.c", + root ++ "library/ssl_debug_helpers_generated.c", + root ++ "library/ssl_msg.c", + root ++ "library/ssl_ticket.c", + root ++ "library/ssl_tls.c", + root ++ "library/ssl_tls12_client.c", + root ++ "library/ssl_tls12_server.c", + root ++ "library/ssl_tls13_keys.c", + root ++ "library/ssl_tls13_server.c", + root ++ "library/ssl_tls13_client.c", + root ++ "library/ssl_tls13_generic.c", + } + }); +} + +fn buildNghttp2(b: *Build, m: *Build.Module) !void { + const nghttp2 = b.addLibrary(.{ + .name = "nghttp2", + .root_module = m, + }); + + const nghttp2_version_file = b.addWriteFile( + "vendor/nghttp2/lib/includes/nghttp2/nghttp2ver.h", + \\ #ifndef NGHTTP2VER_H + \\ #define NGHTTP2VER_H + \\ #define NGHTTP2_VERSION "1.66" + \\ #define NGHTTP2_VERSION_NUM 0x014300 + \\ #endif /* NGHTTP2VER_H */ + ); + nghttp2.step.dependOn(&nghttp2_version_file.step); + + const root = "vendor/nghttp2/"; + nghttp2.addIncludePath(b.path(root ++ "lib")); + nghttp2.addIncludePath(b.path(root ++ "lib/includes")); + nghttp2.addCSourceFiles(.{ + .flags = &.{ + "-DNGHTTP2_STATICLIB", + "-DHAVE_NETINET_IN", + "-DHAVE_TIME_H", + }, + .files = &.{ + root ++ "lib/sfparse.c", + root ++ "lib/nghttp2_alpn.c", + root ++ "lib/nghttp2_buf.c", + root ++ "lib/nghttp2_callbacks.c", + root ++ "lib/nghttp2_debug.c", + root ++ "lib/nghttp2_extpri.c", + root ++ "lib/nghttp2_frame.c", + root ++ "lib/nghttp2_hd.c", + root ++ "lib/nghttp2_hd_huffman.c", + root ++ "lib/nghttp2_hd_huffman_data.c", + root ++ "lib/nghttp2_helper.c", + root ++ "lib/nghttp2_http.c", + root ++ "lib/nghttp2_map.c", + root ++ "lib/nghttp2_mem.c", + root ++ "lib/nghttp2_option.c", + root ++ "lib/nghttp2_outbound_item.c", + root ++ "lib/nghttp2_pq.c", + root ++ "lib/nghttp2_priority_spec.c", + root ++ "lib/nghttp2_queue.c", + root ++ "lib/nghttp2_rcbuf.c", + root ++ "lib/nghttp2_session.c", + root ++ "lib/nghttp2_stream.c", + root ++ "lib/nghttp2_submit.c", + root ++ "lib/nghttp2_version.c", + root ++ "lib/nghttp2_ratelim.c", + root ++ "lib/nghttp2_time.c", + } + }); +} + +fn buildCurl(b: *Build, m: *Build.Module) !void { + const curl = b.addLibrary(.{ + .name = "curl", + .root_module = m, + }); + + const root = "vendor/curl/"; + + curl.addIncludePath(b.path(root ++ "lib")); + curl.addIncludePath(b.path(root ++ "include")); + curl.addCSourceFiles(.{ + .flags = &.{ + }, + .files = &.{ + root ++ "lib/altsvc.c", + root ++ "lib/amigaos.c", + root ++ "lib/asyn-ares.c", + root ++ "lib/asyn-base.c", + root ++ "lib/asyn-thrdd.c", + root ++ "lib/bufq.c", + root ++ "lib/bufref.c", + root ++ "lib/cf-h1-proxy.c", + root ++ "lib/cf-h2-proxy.c", + root ++ "lib/cf-haproxy.c", + root ++ "lib/cf-https-connect.c", + root ++ "lib/cf-socket.c", + root ++ "lib/cfilters.c", + root ++ "lib/conncache.c", + root ++ "lib/connect.c", + root ++ "lib/content_encoding.c", + root ++ "lib/cookie.c", + root ++ "lib/cshutdn.c", + root ++ "lib/curl_addrinfo.c", + root ++ "lib/curl_des.c", + root ++ "lib/curl_endian.c", + root ++ "lib/curl_fnmatch.c", + root ++ "lib/curl_get_line.c", + root ++ "lib/curl_gethostname.c", + root ++ "lib/curl_gssapi.c", + root ++ "lib/curl_memrchr.c", + root ++ "lib/curl_ntlm_core.c", + root ++ "lib/curl_range.c", + root ++ "lib/curl_rtmp.c", + root ++ "lib/curl_sasl.c", + root ++ "lib/curl_sha512_256.c", + root ++ "lib/curl_sspi.c", + root ++ "lib/curl_threads.c", + root ++ "lib/curl_trc.c", + root ++ "lib/cw-out.c", + root ++ "lib/cw-pause.c", + root ++ "lib/dict.c", + root ++ "lib/doh.c", + root ++ "lib/dynhds.c", + root ++ "lib/easy.c", + root ++ "lib/easygetopt.c", + root ++ "lib/easyoptions.c", + root ++ "lib/escape.c", + root ++ "lib/fake_addrinfo.c", + root ++ "lib/file.c", + root ++ "lib/fileinfo.c", + root ++ "lib/fopen.c", + root ++ "lib/formdata.c", + root ++ "lib/ftp.c", + root ++ "lib/ftplistparser.c", + root ++ "lib/getenv.c", + root ++ "lib/getinfo.c", + root ++ "lib/gopher.c", + root ++ "lib/hash.c", + root ++ "lib/headers.c", + root ++ "lib/hmac.c", + root ++ "lib/hostip.c", + root ++ "lib/hostip4.c", + root ++ "lib/hostip6.c", + root ++ "lib/hsts.c", + root ++ "lib/http.c", + root ++ "lib/http1.c", + root ++ "lib/http2.c", + root ++ "lib/http_aws_sigv4.c", + root ++ "lib/http_chunks.c", + root ++ "lib/http_digest.c", + root ++ "lib/http_negotiate.c", + root ++ "lib/http_ntlm.c", + root ++ "lib/http_proxy.c", + root ++ "lib/httpsrr.c", + root ++ "lib/idn.c", + root ++ "lib/if2ip.c", + root ++ "lib/imap.c", + root ++ "lib/krb5.c", + root ++ "lib/ldap.c", + root ++ "lib/llist.c", + root ++ "lib/macos.c", + root ++ "lib/md4.c", + root ++ "lib/md5.c", + root ++ "lib/memdebug.c", + root ++ "lib/mime.c", + root ++ "lib/mprintf.c", + root ++ "lib/mqtt.c", + root ++ "lib/multi.c", + root ++ "lib/multi_ev.c", + root ++ "lib/netrc.c", + root ++ "lib/noproxy.c", + root ++ "lib/openldap.c", + root ++ "lib/parsedate.c", + root ++ "lib/pingpong.c", + root ++ "lib/pop3.c", + root ++ "lib/progress.c", + root ++ "lib/psl.c", + root ++ "lib/rand.c", + root ++ "lib/rename.c", + root ++ "lib/request.c", + root ++ "lib/rtsp.c", + root ++ "lib/select.c", + root ++ "lib/sendf.c", + root ++ "lib/setopt.c", + root ++ "lib/sha256.c", + root ++ "lib/share.c", + root ++ "lib/slist.c", + root ++ "lib/smb.c", + root ++ "lib/smtp.c", + root ++ "lib/socketpair.c", + root ++ "lib/socks.c", + root ++ "lib/socks_gssapi.c", + root ++ "lib/socks_sspi.c", + root ++ "lib/speedcheck.c", + root ++ "lib/splay.c", + root ++ "lib/strcase.c", + root ++ "lib/strdup.c", + root ++ "lib/strequal.c", + root ++ "lib/strerror.c", + root ++ "lib/system_win32.c", + root ++ "lib/telnet.c", + root ++ "lib/tftp.c", + root ++ "lib/transfer.c", + root ++ "lib/uint-bset.c", + root ++ "lib/uint-hash.c", + root ++ "lib/uint-spbset.c", + root ++ "lib/uint-table.c", + root ++ "lib/url.c", + root ++ "lib/urlapi.c", + root ++ "lib/version.c", + root ++ "lib/ws.c", + root ++ "lib/curlx/base64.c", + root ++ "lib/curlx/dynbuf.c", + root ++ "lib/curlx/inet_ntop.c", + root ++ "lib/curlx/nonblock.c", + root ++ "lib/curlx/strparse.c", + root ++ "lib/curlx/timediff.c", + root ++ "lib/curlx/timeval.c", + root ++ "lib/curlx/wait.c", + root ++ "lib/curlx/warnless.c", + root ++ "lib/vquic/curl_ngtcp2.c", + root ++ "lib/vquic/curl_osslq.c", + root ++ "lib/vquic/curl_quiche.c", + root ++ "lib/vquic/vquic.c", + root ++ "lib/vquic/vquic-tls.c", + root ++ "lib/vauth/cleartext.c", + root ++ "lib/vauth/cram.c", + root ++ "lib/vauth/digest.c", + root ++ "lib/vauth/digest_sspi.c", + root ++ "lib/vauth/gsasl.c", + root ++ "lib/vauth/krb5_gssapi.c", + root ++ "lib/vauth/krb5_sspi.c", + root ++ "lib/vauth/ntlm.c", + root ++ "lib/vauth/ntlm_sspi.c", + root ++ "lib/vauth/oauth2.c", + root ++ "lib/vauth/spnego_gssapi.c", + root ++ "lib/vauth/spnego_sspi.c", + root ++ "lib/vauth/vauth.c", + root ++ "lib/vtls/cipher_suite.c", + root ++ "lib/vtls/mbedtls.c", + root ++ "lib/vtls/mbedtls_threadlock.c", + root ++ "lib/vtls/vtls.c", + root ++ "lib/vtls/vtls_scache.c", + root ++ "lib/vtls/x509asn1.c", + }, + }); +} diff --git a/build.zig.zon b/build.zig.zon index b51e82da..6a45c2d8 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -4,10 +4,6 @@ .version = "0.0.0", .fingerprint = 0xda130f3af836cea0, .dependencies = .{ - .tls = .{ - .url = "https://github.com/ianic/tls.zig/archive/55845f755d9e2e821458ea55693f85c737cd0c7a.tar.gz", - .hash = "tls-0.1.0-ER2e0m43BQAshi8ixj1qf3w2u2lqKtXtkrxUJ4AGZDcl", - }, .tigerbeetle_io = .{ .url = "https://github.com/lightpanda-io/tigerbeetle-io/archive/61d9652f1a957b7f4db723ea6aa0ce9635e840ce.tar.gz", .hash = "tigerbeetle_io-0.0.0-ViLgxpyRBAB5BMfIcj3KMXfbJzwARs9uSl8aRy2OXULd", diff --git a/src/app.zig b/src/app.zig index ad66f73a..80954855 100644 --- a/src/app.zig +++ b/src/app.zig @@ -1,10 +1,11 @@ const std = @import("std"); + const Allocator = std.mem.Allocator; const log = @import("log.zig"); const Loop = @import("runtime/loop.zig").Loop; -const http = @import("http/client.zig"); const Platform = @import("runtime/js.zig").Platform; +const http = @import("http/client.zig"); const Telemetry = @import("telemetry/telemetry.zig").Telemetry; const Notification = @import("notification.zig").Notification; @@ -17,7 +18,7 @@ pub const App = struct { platform: ?*const Platform, allocator: Allocator, telemetry: Telemetry, - http_client: http.Client, + http_client: *http.Client, app_dir_path: ?[]const u8, notification: *Notification, @@ -59,12 +60,8 @@ pub const App = struct { .platform = config.platform, .app_dir_path = app_dir_path, .notification = notification, - .http_client = try http.Client.init(allocator, loop, .{ - .max_concurrent = 3, - .http_proxy = config.http_proxy, - .proxy_type = config.proxy_type, - .proxy_auth = config.proxy_auth, - .tls_verify_host = config.tls_verify_host, + .http_client = try http.Client.init(allocator, .{ + .max_concurrent_transfers = 3, }), .config = config, }; diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig new file mode 100644 index 00000000..895fe065 --- /dev/null +++ b/src/browser/ScriptManager.zig @@ -0,0 +1,543 @@ +// Copyright (C) 2023-2025 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); + + +const log = @import("../log.zig"); +const parser = @import("netsurf.zig"); +const http = @import("../http/client.zig"); + +const App = @import("../app.zig").App; +const Env = @import("env.zig").Env; +const Page = @import("page.zig").Page; +const URL = @import("../url.zig").URL; + +const Allocator = std.mem.Allocator; +const ArrayListUnmanaged = std.ArrayListUnmanaged; + +const ScriptManager = @This(); + +page: *Page, + +// Only once this is true can deferred scripts be run +static_scripts_done: bool, + +// Normal scripts (non-deffered & non-async). These must be executed ni order +scripts: OrderList, + +// List of deferred scripts. These must be executed in order, but only once +// dom_loaded == true, +deferred: OrderList, + +client: *http.Client, +allocator: Allocator, +buffer_pool: BufferPool, +script_pool: std.heap.MemoryPool(PendingScript), + +const OrderList = std.DoublyLinkedList(*PendingScript); + +pub fn init(app: *App, page: *Page) ScriptManager { + const allocator = app.allocator; + return .{ + .page = page, + .scripts = .{}, + .deferred = .{}, + .allocator = allocator, + .client = app.http_client, + .static_scripts_done = false, + .buffer_pool = BufferPool.init(allocator, 5), + .script_pool = std.heap.MemoryPool(PendingScript).init(allocator), + }; +} + +pub fn deinit(self: *ScriptManager) void { + self.buffer_pool.deinit(); + self.script_pool.deinit(); +} + +pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { + if (try parser.elementGetAttribute(element, "nomodule") != null) { + // these scripts should only be loaded if we don't support modules + // but since we do support modules, we can just skip them. + return; + } + + // If a script tag gets dynamically created and added to the dom: + // document.getElementsByTagName('head')[0].appendChild(script) + // that script tag will immediately get executed by our scriptAddedCallback. + // However, if the location where the script tag is inserted happens to be + // below where processHTMLDoc curently is, then we'll re-run that same script + // again in processHTMLDoc. This flag is used to let us know if a specific + // +// Unlike external modules which can only ever be executed after releasing an +// http handle, these are executed without there necessarily being a free handle. +// Thus, Http/Client.zig maintains a dedicated handle for these calls. pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { var blocking = Blocking{ .allocator = self.allocator, @@ -232,10 +270,11 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { }; var client = self.client; - try client.request(.{ + try client.blockingRequest(.{ .url = url, .method = .GET, .ctx = &blocking, + .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null, .header_done_callback = Blocking.headerCallback, .data_callback = Blocking.dataCallback, .done_callback = Blocking.doneCallback, @@ -244,7 +283,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { // rely on http's timeout settings to avoid an endless/long loop. while (true) { - try client.tick(1000); + try client.tick(200); switch (blocking.state) { .running => {}, .done => |result| return result, diff --git a/src/browser/page.zig b/src/browser/page.zig index 51129e88..3811c1c9 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -292,13 +292,10 @@ pub const Page = struct { // The HTML page was parsed. We now either have JS scripts to // download, or timeouts to execute, or both. - // If we have active http transfers, we might as well run - // any "secondary" task, since we won't be exiting this loop - // anyways. // scheduler.run could trigger new http transfers, so do not // store http_client.active BEFORE this call and then use // it AFTER. - const ms_to_next_task = try scheduler.run(http_client.active > 0); + const ms_to_next_task = try scheduler.runHighPriority(); if (try_catch.hasCaught()) { const msg = (try try_catch.err(self.arena)) orelse "unknown"; @@ -316,7 +313,7 @@ pub const Page = struct { // we'd wait to long, might as well exit early. return; } - + _ = try scheduler.runLowPriority(); std.time.sleep(std.time.ns_per_ms * ms); break :SW; } @@ -326,6 +323,8 @@ pub const Page = struct { return; } + _ = try scheduler.runLowPriority(); + // We'll block here, waiting for network IO. We know // when the next timeout is scheduled, and we know how long // the caller wants to wait for, so we can pick a good wait @@ -344,7 +343,7 @@ pub const Page = struct { } const ms_elapsed = timer.lap() / 1_000_000; - if (ms_elapsed > ms_remaining) { + if (ms_elapsed >= ms_remaining) { return; } ms_remaining -= ms_elapsed; @@ -354,7 +353,27 @@ pub const Page = struct { fn printWaitAnalysis(self: *Page) void { std.debug.print("mode: {s}\n", .{@tagName(std.meta.activeTag(self.mode))}); std.debug.print("load: {s}\n", .{@tagName(self.load_state)}); - std.debug.print("active requests: {d}\n", .{self.http_client.active}); + { + std.debug.print("\nactive requests: {d}\n", .{self.http_client.active}); + var n_ = self.http_client.handles.in_use.first; + while (n_) |n| { + const transfer = HttpClient.Transfer.fromEasy(n.data.conn.easy) catch |err| { + std.debug.print(" - failed to load transfer: {any}\n", .{err}); + break; + }; + std.debug.print(" - {s}\n", .{transfer}); + n_ = n.next; + } + } + + { + std.debug.print("\nqueued requests: {d}\n", .{self.http_client.queue.len}); + var n_ = self.http_client.queue.first; + while (n_) |n| { + std.debug.print(" - {s}\n", .{n.data.url}); + n_ = n.next; + } + } { std.debug.print("\nscripts: {d}\n", .{self.script_manager.scripts.len}); diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index bd237c6d..f59f9cee 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -363,7 +363,7 @@ pub const XMLHttpRequest = struct { if (self.state != .opened) return DOMError.InvalidState; if (self.send_flag) return DOMError.InvalidState; - log.debug(.http, "request", .{ .method = self.method, .url = self.url, .source = "xhr" }); + log.debug(.http, "request queued", .{ .method = self.method, .url = self.url, .source = "xhr" }); self.send_flag = true; if (body) |b| { @@ -394,6 +394,8 @@ pub const XMLHttpRequest = struct { try transfer.addHeader(hdr); } + log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" }); + // @newhttp // { // var arr: std.ArrayListUnmanaged(u8) = .{}; diff --git a/src/http/Client.zig b/src/http/Client.zig index 1da6d857..a34a0ff9 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -531,7 +531,8 @@ pub const Transfer = struct { return chunk_len; } - fn fromEasy(easy: *c.CURL) !*Transfer { + // pub because Page.printWaitAnalysis uses it + pub fn fromEasy(easy: *c.CURL) !*Transfer { var private: *anyopaque = undefined; try errorCheck(c.curl_easy_getinfo(easy, c.CURLINFO_PRIVATE, &private)); return @alignCast(@ptrCast(private)); diff --git a/src/log.zig b/src/log.zig index 751d67c3..9ac06d46 100644 --- a/src/log.zig +++ b/src/log.zig @@ -320,22 +320,18 @@ fn timestamp() i64 { return std.time.milliTimestamp(); } -var last_log: i64 = 0; +var first_log: i64 = 0; fn elapsed() i64 { const now = timestamp(); last_log_lock.lock(); - const previous = last_log; - last_log = now; - last_log_lock.unlock(); + defer last_log_lock.unlock(); - if (previous == 0) { - return 0; + if (first_log == 0) { + first_log = now; } - if (previous > now) { - return 0; - } - return now - previous; + + return now - first_log; } const testing = @import("testing.zig"); From ddb549cb45ac2a0f40254d70f0bac36074660e75 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 5 Aug 2025 17:54:29 +0800 Subject: [PATCH 18/44] cookie support --- src/browser/ScriptManager.zig | 2 + src/browser/html/document.zig | 5 +- src/browser/page.zig | 22 +++++++- src/browser/storage/cookie.zig | 37 ++++++------- src/browser/xhr/xhr.zig | 20 +------ src/data/public_suffix_list.zig | 3 +- src/http/Client.zig | 97 +++++++++++++++++++++++++++++++-- 7 files changed, 138 insertions(+), 48 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index bb69ef1f..47dfb354 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -222,6 +222,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { .url = remote_url.?, .ctx = pending_script, .method = .GET, + .cookie = page.requestCookie(.{}), .start_callback = if (log.enabled(.http, .debug)) startCallback else null, .header_done_callback = headerCallback, .data_callback = dataCallback, @@ -274,6 +275,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { .url = url, .method = .GET, .ctx = &blocking, + .cookie = self.page.requestCookie(.{}), .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null, .header_done_callback = Blocking.headerCallback, .data_callback = Blocking.dataCallback, diff --git a/src/browser/html/document.zig b/src/browser/html/document.zig index cf2ddd7e..ee9eb846 100644 --- a/src/browser/html/document.zig +++ b/src/browser/html/document.zig @@ -85,7 +85,10 @@ pub const HTMLDocument = struct { pub fn get_cookie(_: *parser.DocumentHTML, page: *Page) ![]const u8 { var buf: std.ArrayListUnmanaged(u8) = .{}; - try page.cookie_jar.forRequest(&page.url.uri, buf.writer(page.arena), .{ .navigation = true, .is_http = false }); + try page.cookie_jar.forRequest(&page.url.uri, buf.writer(page.arena), .{ + .is_http = false, + .is_navigation = true, + }); return buf.items; } diff --git a/src/browser/page.zig b/src/browser/page.zig index 3811c1c9..384ea1f6 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -426,6 +426,19 @@ pub const Page = struct { return arr.items; } + const RequestCookieOpts = struct { + is_http: bool = true, + is_navigation: bool = false, + }; + pub fn requestCookie(self: *const Page, opts: RequestCookieOpts) HttpClient.RequestCookie { + return .{ + .jar = self.cookie_jar, + .origin = &self.url.uri, + .is_http = opts.is_http, + .is_navigation = opts.is_navigation, + }; + } + // spec reference: https://html.spec.whatwg.org/#document-lifecycle pub fn navigate(self: *Page, request_url: []const u8, opts: NavigateOpts) !void { if (self.mode != .pre) { @@ -453,17 +466,22 @@ pub const Page = struct { } const owned_url = try self.arena.dupeZ(u8, request_url); + self.url = try URL.parse(owned_url, null); - try self.http_client.request(.{ + self.http_client.request(.{ .ctx = self, .url = owned_url, .method = opts.method, .body = opts.body, + .cookie = self.requestCookie(.{ .is_navigation = true }), .header_done_callback = pageHeaderDoneCallback, .data_callback = pageDataCallback, .done_callback = pageDoneCallback, .error_callback = pageErrorCallback, - }); + }) catch |err| { + log.err(.http, "navigate request", .{ .url = owned_url, .err = err }); + return err; + }; self.session.browser.notification.dispatch(.page_navigate, &.{ .opts = opts, diff --git a/src/browser/storage/cookie.zig b/src/browser/storage/cookie.zig index 8c061576..98104c0e 100644 --- a/src/browser/storage/cookie.zig +++ b/src/browser/storage/cookie.zig @@ -10,8 +10,8 @@ const public_suffix_list = @import("../../data/public_suffix_list.zig").lookup; pub const LookupOpts = struct { request_time: ?i64 = null, origin_uri: ?*const Uri = null, - navigation: bool = true, is_http: bool, + is_navigation: bool = true, }; pub const Jar = struct { @@ -91,7 +91,7 @@ pub const Jar = struct { var first = true; for (self.cookies.items) |*cookie| { - if (!cookie.appliesTo(&target, same_site, opts.navigation, opts.is_http)) continue; + if (!cookie.appliesTo(&target, same_site, opts.is_navigation, opts.is_http)) continue; // we have a match! if (first) { @@ -103,18 +103,15 @@ pub const Jar = struct { } } - // @newhttp - // pub fn populateFromResponse(self: *Jar, uri: *const Uri, header: *const http.ResponseHeader) !void { - // const now = std.time.timestamp(); - // var it = header.iterate("set-cookie"); - // while (it.next()) |set_cookie| { - // const c = Cookie.parse(self.allocator, uri, set_cookie) catch |err| { - // log.warn(.web_api, "cookie parse failed", .{ .raw = set_cookie, .err = err }); - // continue; - // }; - // try self.add(c, now); - // } - // } + pub fn populateFromResponse(self: *Jar, uri: *const Uri, set_cookie: []const u8) !void { + const c = Cookie.parse(self.allocator, uri, set_cookie) catch |err| { + log.warn(.web_api, "cookie parse failed", .{ .raw = set_cookie, .err = err }); + return; + }; + + const now = std.time.timestamp(); + try self.add(c, now); + } fn writeCookie(cookie: *const Cookie, writer: anytype) !void { if (cookie.name.len > 0) { @@ -429,7 +426,7 @@ pub const Cookie = struct { return .{ name, value, rest }; } - pub fn appliesTo(self: *const Cookie, url: *const PreparedUri, same_site: bool, navigation: bool, is_http: bool) bool { + pub fn appliesTo(self: *const Cookie, url: *const PreparedUri, same_site: bool, is_navigation: bool, is_http: bool) bool { if (self.http_only and is_http == false) { // http only cookies can be accessed from Javascript return false; @@ -448,7 +445,7 @@ pub const Cookie = struct { // and cookie.same_site == .lax switch (self.same_site) { .strict => return false, - .lax => if (navigation == false) return false, + .lax => if (is_navigation == false) return false, .none => {}, } } @@ -619,7 +616,7 @@ test "Jar: forRequest" { // nothing fancy here try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .is_http = true }); - try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .origin_uri = &test_uri, .navigation = false, .is_http = true }); + try expectCookies("global1=1; global2=2", &jar, test_uri, .{ .origin_uri = &test_uri, .is_navigation = false, .is_http = true }); // We have a cookie where Domain=lightpanda.io // This should _not_ match xyxlightpanda.io @@ -685,22 +682,22 @@ test "Jar: forRequest" { // non-navigational cross domain, insecure try expectCookies("", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{ .origin_uri = &(try std.Uri.parse("https://example.com/")), - .navigation = false, .is_http = true, + .is_navigation = false, }); // non-navigational cross domain, secure try expectCookies("sitenone=6", &jar, try std.Uri.parse("https://lightpanda.io/x/"), .{ .origin_uri = &(try std.Uri.parse("https://example.com/")), - .navigation = false, .is_http = true, + .is_navigation = false, }); // non-navigational same origin try expectCookies("global1=1; global2=2; sitelax=7; sitestrict=8", &jar, try std.Uri.parse("http://lightpanda.io/x/"), .{ .origin_uri = &(try std.Uri.parse("https://lightpanda.io/")), - .navigation = false, .is_http = true, + .is_navigation = false, }); // exact domain match + suffix diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index f59f9cee..a91967e7 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -81,7 +81,6 @@ pub const XMLHttpRequest = struct { proto: XMLHttpRequestEventTarget = XMLHttpRequestEventTarget{}, arena: Allocator, transfer: ?*HttpClient.Transfer = null, - cookie_jar: *CookieJar, err: ?anyerror = null, last_dispatch: i64 = 0, send_flag: bool = false, @@ -169,7 +168,6 @@ pub const XMLHttpRequest = struct { .headers = .{}, .method = undefined, .state = .unsent, - .cookie_jar = page.cookie_jar, }; } @@ -378,6 +376,7 @@ pub const XMLHttpRequest = struct { .method = self.method, .body = self.request_body, .content_type = "Content-Type: text/plain; charset=UTF-8", // @newhttp TODO + .cookie = page.requestCookie(.{}), .start_callback = httpStartCallback, .header_callback = httpHeaderCallback, .header_done_callback = httpHeaderDoneCallback, @@ -395,20 +394,6 @@ pub const XMLHttpRequest = struct { } log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" }); - - // @newhttp - // { - // var arr: std.ArrayListUnmanaged(u8) = .{}; - // try self.cookie_jar.forRequest(&self.url.?.uri, arr.writer(self.arena), .{ - // .navigation = false, - // .origin_uri = &self.origin_url.uri, - // .is_http = true, - // }); - - // if (arr.items.len > 0) { - // try request.addHeader("Cookie", arr.items, .{}); - // } - // } self.transfer = transfer; } @@ -445,9 +430,6 @@ pub const XMLHttpRequest = struct { self.state = .loading; self.dispatchEvt("readystatechange"); - - // @newhttp - // try self.cookie_jar.populateFromResponse(self.request.?.request_uri, &header); } fn httpDataCallback(transfer: *HttpClient.Transfer, data: []const u8) !void { diff --git a/src/data/public_suffix_list.zig b/src/data/public_suffix_list.zig index dcab218a..16eb145d 100644 --- a/src/data/public_suffix_list.zig +++ b/src/data/public_suffix_list.zig @@ -8,8 +8,7 @@ pub fn lookup(value: []const u8) bool { const public_suffix_list = std.StaticStringMap(void).initComptime(entries); const entries: []const struct { []const u8, void } = - // @newhttp - if (builtin.is_test or true) &.{ + if (builtin.is_test) &.{ .{ "api.gov.uk", {} }, .{ "gov.uk", {} }, } else &.{ diff --git a/src/http/Client.zig b/src/http/Client.zig index a34a0ff9..5d4e337f 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -24,6 +24,7 @@ const Http = @import("Http.zig"); const c = Http.c; const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; const errorCheck = Http.errorCheck; const errorMCheck = Http.errorMCheck; @@ -43,18 +44,49 @@ pub const Method = Http.Method; // those other http requests. pub const Client = @This(); +// count of active requests active: usize, + +// curl has 2 APIs: easy and multi. Multi is like a combination of some I/O block +// (e.g. epoll) and a bunch of pools. You add/remove easys to the multiple and +// then poll the multi. multi: *c.CURLM, + +// Our easy handles. Although the multi contains buffer pools and connections +// pools, re-using the easys is still recommended. This acts as our own pool +// of easys. handles: Handles, + +// When handles has no more available easys, requests get queued. queue: RequestQueue, -allocator: Allocator, -transfer_pool: std.heap.MemoryPool(Transfer), + +// Memory pool for Queue nodes. queue_node_pool: std.heap.MemoryPool(RequestQueue.Node), + +// The main app allocator +allocator: Allocator, + +// Once we have a handle/easy to process a request with, we create a Transfer +// which contains the Request as well as any state we need to process the +// request. These wil come and go with each request. +transfer_pool: std.heap.MemoryPool(Transfer), + //@newhttp http_proxy: ?std.Uri = null, + +// see ScriptManager.blockingGet blocking: Handle, + +// Boolean to check that we don't make a blocking request while an existing +// blocking request is already being processed. blocking_active: if (builtin.mode == .Debug) bool else void = if (builtin.mode == .Debug) false else {}, +// The only place this is meant to be used is in `makeRequest` BEFORE `perform` +// is called. It is used to generate our Cookie header. It can be used for other +// purposes, but keep in mind that, while single-threaded, calls like makeRequest +// can result in makeRequest being re-called (from a doneCallback). +arena: ArenaAllocator, + const RequestQueue = std.DoublyLinkedList(Request); pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Client { @@ -85,6 +117,7 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie .allocator = allocator, .transfer_pool = transfer_pool, .queue_node_pool = queue_node_pool, + .arena = ArenaAllocator.init(allocator), }; return client; @@ -99,6 +132,7 @@ pub fn deinit(self: *Client) void { self.transfer_pool.deinit(); self.queue_node_pool.deinit(); + self.arena.deinit(); self.allocator.destroy(self); } @@ -176,6 +210,13 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { const conn = handle.conn; const easy = conn.easy; + // we need this for cookies + const uri = std.Uri.parse(req.url) catch |err| { + self.handles.release(handle); + log.warn(.http, "invalid url", .{ .err = err, .url = req.url }); + return; + }; + const header_list = blk: { errdefer self.handles.release(handle); try conn.setMethod(req.method); @@ -192,6 +233,23 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { header_list = c.curl_slist_append(header_list, ct); } + { + const COOKIE_HEADER = "Cookie: "; + const aa = self.arena.allocator(); + defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); + + var arr: std.ArrayListUnmanaged(u8) = .{}; + try arr.appendSlice(aa, COOKIE_HEADER); + try req.cookie.forRequest(&uri, arr.writer(aa)); + + if (arr.items.len > COOKIE_HEADER.len) { + try arr.append(aa, 0); //null terminate + + // copies the value + header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr)); + } + } + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list)); break :blk header_list; @@ -203,12 +261,14 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { const transfer = try self.transfer_pool.create(); transfer.* = .{ .id = 0, + .uri = uri, .req = req, .ctx = req.ctx, .handle = handle, ._request_header_list = header_list, }; errdefer self.transfer_pool.destroy(transfer); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer)); try errorMCheck(c.curl_multi_add_handle(self.multi, easy)); @@ -370,11 +430,27 @@ const Handle = struct { } }; +pub const RequestCookie = struct { + is_http: bool, + is_navigation: bool, + origin: *const std.Uri, + jar: *@import("../browser/storage/cookie.zig").Jar, + + fn forRequest(self: *const RequestCookie, uri: *const std.Uri, writer: anytype) !void { + return self.jar.forRequest(uri, writer, .{ + .is_http = self.is_http, + .is_navigation = self.is_navigation, + .origin_uri = self.origin, + }); + } +}; + pub const Request = struct { method: Method, url: [:0]const u8, body: ?[]const u8 = null, content_type: ?[:0]const u8 = null, + cookie: RequestCookie, // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, @@ -391,6 +467,7 @@ pub const Transfer = struct { id: usize, req: Request, ctx: *anyopaque, + uri: std.Uri, // used for setting/getting the cookie // We'll store the response header here response_header: ?Header = null, @@ -479,10 +556,10 @@ pub const Transfer = struct { return buf_len; } - const CONTENT_TYPE_LEN = "content-type:".len; - var hdr = &transfer.response_header.?; + if (hdr._content_type_len == 0) { + const CONTENT_TYPE_LEN = "content-type:".len; if (buf_len > CONTENT_TYPE_LEN) { if (std.ascii.eqlIgnoreCase(header[0..CONTENT_TYPE_LEN], "content-type:")) { const value = std.mem.trimLeft(u8, header[CONTENT_TYPE_LEN..], " "); @@ -493,6 +570,18 @@ pub const Transfer = struct { } } + { + const SET_COOKIE_LEN = "set-cookie:".len; + if (buf_len > SET_COOKIE_LEN) { + if (std.ascii.eqlIgnoreCase(header[0..SET_COOKIE_LEN], "set-cookie:")) { + const value = std.mem.trimLeft(u8, header[SET_COOKIE_LEN..], " "); + transfer.req.cookie.jar.populateFromResponse(&transfer.uri, value) catch |err| { + log.err(.http, "set cookie", .{ .err = err, .req = transfer }); + }; + } + } + } + if (buf_len == 2) { transfer.req.header_done_callback(transfer) catch |err| { log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); From cabd4fa718b1800dbc7f81367dd12ffa43d70c6a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 5 Aug 2025 19:00:56 +0800 Subject: [PATCH 19/44] re-enable datauris --- src/browser/DataURI.zig | 52 +++++++++++++++++++++++ src/browser/ScriptManager.zig | 4 ++ src/browser/datauri.zig | 79 ----------------------------------- src/browser/page.zig | 1 - 4 files changed, 56 insertions(+), 80 deletions(-) create mode 100644 src/browser/DataURI.zig delete mode 100644 src/browser/datauri.zig diff --git a/src/browser/DataURI.zig b/src/browser/DataURI.zig new file mode 100644 index 00000000..00d3792f --- /dev/null +++ b/src/browser/DataURI.zig @@ -0,0 +1,52 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; + +// Parses data:[][;base64], +pub fn parse(allocator: Allocator, src: []const u8) !?[]const u8 { + if (!std.mem.startsWith(u8, src, "data:")) { + return null; + } + + const uri = src[5..]; + const data_starts = std.mem.indexOfScalar(u8, uri, ',') orelse return null; + + var data = uri[data_starts + 1 ..]; + + // Extract the encoding. + const metadata = uri[0..data_starts]; + if (std.mem.endsWith(u8, metadata, ";base64")) { + const decoder = std.base64.standard.Decoder; + const decoded_size = try decoder.calcSizeForSlice(data); + + const buffer = try allocator.alloc(u8, decoded_size); + errdefer allocator.free(buffer); + + try decoder.decode(buffer, data); + data = buffer; + } + + return data; +} + +const testing = @import("../testing.zig"); +test "DataURI: parse valid" { + try test_valid("data:text/javascript; charset=utf-8;base64,Zm9v", "foo"); + try test_valid("data:text/javascript; charset=utf-8;,foo", "foo"); + try test_valid("data:,foo", "foo"); +} + +test "DataURI: parse invalid" { + try test_cannot_parse("atad:,foo"); + try test_cannot_parse("data:foo"); + try test_cannot_parse("data:"); +} + +fn test_valid(uri: []const u8, expected: []const u8) !void { + defer testing.reset(); + const data_uri = try parse(testing.arena_allocator, uri) orelse return error.TestFailed; + try testing.expectEqual(expected, data_uri); +} + +fn test_cannot_parse(uri: []const u8) !void { + try testing.expectEqual(null, parse(undefined, uri)); +} diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 47dfb354..e5108dbd 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -23,6 +23,7 @@ const parser = @import("netsurf.zig"); const Env = @import("env.zig").Env; const Page = @import("page.zig").Page; +const DataURI = @import("DataURI.zig"); const Browser = @import("browser.zig").Browser; const HttpClient = @import("../http/Client.zig"); const URL = @import("../url.zig").URL; @@ -168,6 +169,9 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { var source: Script.Source = undefined; var remote_url: ?[:0]const u8 = null; if (try parser.elementGetAttribute(element, "src")) |src| { + if (try DataURI.parse(page.arena, src)) |data_uri| { + source = .{ .@"inline" = data_uri }; + } remote_url = try URL.stitch(page.arena, src, page.url.raw, .{ .null_terminated = true }); source = .{ .remote = .{} }; } else { diff --git a/src/browser/datauri.zig b/src/browser/datauri.zig deleted file mode 100644 index d600a255..00000000 --- a/src/browser/datauri.zig +++ /dev/null @@ -1,79 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; - -// Represents https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data -pub const DataURI = struct { - was_base64_encoded: bool, - // The contents in the uri. It will be base64 decoded but not prepared in - // any way for mime.charset. - data: []const u8, - - // Parses data:[][;base64], - pub fn parse(allocator: Allocator, src: []const u8) !?DataURI { - if (!std.mem.startsWith(u8, src, "data:")) { - return null; - } - - const uri = src[5..]; - const data_starts = std.mem.indexOfScalar(u8, uri, ',') orelse return null; - - // Extract the encoding. - var metadata = uri[0..data_starts]; - var base64_encoded = false; - if (std.mem.endsWith(u8, metadata, ";base64")) { - base64_encoded = true; - metadata = metadata[0 .. metadata.len - 7]; - } - - // TODO: Extract mime type. This not trivial because Mime.parse requires - // a []u8 and might mutate the src. And, the DataURI.parse references atm - // do not have deinit calls. - - // Prepare the data. - var data = uri[data_starts + 1 ..]; - if (base64_encoded) { - const decoder = std.base64.standard.Decoder; - const decoded_size = try decoder.calcSizeForSlice(data); - - const buffer = try allocator.alloc(u8, decoded_size); - errdefer allocator.free(buffer); - - try decoder.decode(buffer, data); - data = buffer; - } - - return .{ - .was_base64_encoded = base64_encoded, - .data = data, - }; - } - - pub fn deinit(self: *const DataURI, allocator: Allocator) void { - if (self.was_base64_encoded) { - allocator.free(self.data); - } - } -}; - -const testing = std.testing; -test "DataURI: parse valid" { - try test_valid("data:text/javascript; charset=utf-8;base64,Zm9v", "foo"); - try test_valid("data:text/javascript; charset=utf-8;,foo", "foo"); - try test_valid("data:,foo", "foo"); -} - -test "DataURI: parse invalid" { - try test_cannot_parse("atad:,foo"); - try test_cannot_parse("data:foo"); - try test_cannot_parse("data:"); -} - -fn test_valid(uri: []const u8, expected: []const u8) !void { - const data_uri = try DataURI.parse(std.testing.allocator, uri) orelse return error.TestFailed; - defer data_uri.deinit(testing.allocator); - try testing.expectEqualStrings(expected, data_uri.data); -} - -fn test_cannot_parse(uri: []const u8) !void { - try testing.expectEqual(null, DataURI.parse(std.testing.allocator, uri)); -} diff --git a/src/browser/page.zig b/src/browser/page.zig index 384ea1f6..30c959d4 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -25,7 +25,6 @@ const Dump = @import("dump.zig"); const State = @import("State.zig"); const Env = @import("env.zig").Env; const Mime = @import("mime.zig").Mime; -const DataURI = @import("datauri.zig").DataURI; const Session = @import("session.zig").Session; const Renderer = @import("renderer.zig").Renderer; const Window = @import("html/window.zig").Window; From 06984ace21a32a57ca2dbb5ff741672ca1fd990c Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 5 Aug 2025 20:16:54 +0800 Subject: [PATCH 20/44] fix overflow and debug units --- src/browser/page.zig | 4 ++-- src/http/Client.zig | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/browser/page.zig b/src/browser/page.zig index 30c959d4..54571262 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -406,7 +406,7 @@ pub const Page = struct { std.debug.print("\nprimary schedule: {d}\n", .{self.scheduler.primary.count()}); var it = self.scheduler.primary.iterator(); while (it.next()) |task| { - std.debug.print(" - {s} complete: {any}\n", .{ task.name, task.ms - now }); + std.debug.print(" - {s} schedule: {d}ms\n", .{ task.name, task.ms - now }); } } @@ -414,7 +414,7 @@ pub const Page = struct { std.debug.print("\nsecondary schedule: {d}\n", .{self.scheduler.secondary.count()}); var it = self.scheduler.secondary.iterator(); while (it.next()) |task| { - std.debug.print(" - {s} complete: {any}\n", .{ task.name, task.ms - now }); + std.debug.print(" - {s} schedule: {d}ms\n", .{ task.name, task.ms - now }); } } } diff --git a/src/http/Client.zig b/src/http/Client.zig index 5d4e337f..d0677625 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -560,7 +560,7 @@ pub const Transfer = struct { if (hdr._content_type_len == 0) { const CONTENT_TYPE_LEN = "content-type:".len; - if (buf_len > CONTENT_TYPE_LEN) { + if (header.len > CONTENT_TYPE_LEN) { if (std.ascii.eqlIgnoreCase(header[0..CONTENT_TYPE_LEN], "content-type:")) { const value = std.mem.trimLeft(u8, header[CONTENT_TYPE_LEN..], " "); const len = @min(value.len, hdr._content_type.len); @@ -572,7 +572,7 @@ pub const Transfer = struct { { const SET_COOKIE_LEN = "set-cookie:".len; - if (buf_len > SET_COOKIE_LEN) { + if (header.len > SET_COOKIE_LEN) { if (std.ascii.eqlIgnoreCase(header[0..SET_COOKIE_LEN], "set-cookie:")) { const value = std.mem.trimLeft(u8, header[SET_COOKIE_LEN..], " "); transfer.req.cookie.jar.populateFromResponse(&transfer.uri, value) catch |err| { From 1e612e4166ac3f9ed6a3e3886b447440eb02f4e5 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 6 Aug 2025 10:31:30 +0800 Subject: [PATCH 21/44] Add command line options to control HTTP client http_timeout_ms http_connect_timeout_ms http_max_host_open http_max_concurrent --- src/app.zig | 9 +++- src/http/Client.zig | 5 +- src/http/Http.zig | 7 +-- src/main.zig | 126 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 131 insertions(+), 16 deletions(-) diff --git a/src/app.zig b/src/app.zig index 078b23ce..a1d7ee30 100644 --- a/src/app.zig +++ b/src/app.zig @@ -35,6 +35,10 @@ pub const App = struct { tls_verify_host: bool = true, http_proxy: ?[:0]const u8 = null, proxy_bearer_token: ?[:0]const u8 = null, + http_timeout_ms: ?u31 = null, + http_connect_timeout_ms: ?u31 = null, + http_max_host_open: ?u8 = null, + http_max_concurrent: ?u8 = null, }; pub fn init(allocator: Allocator, config: Config) !*App { @@ -51,7 +55,10 @@ pub const App = struct { errdefer notification.deinit(); var http = try Http.init(allocator, .{ - .max_concurrent_transfers = 10, + .max_host_open = config.http_max_host_open orelse 4, + .max_concurrent = config.http_max_concurrent orelse 10, + .timeout_ms = config.http_timeout_ms orelse 5000, + .connect_timeout_ms = config.http_connect_timeout_ms orelse 0, .http_proxy = config.http_proxy, .tls_verify_host = config.tls_verify_host, .proxy_bearer_token = config.proxy_bearer_token, diff --git a/src/http/Client.zig b/src/http/Client.zig index d0677625..84b10e7e 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -102,6 +102,8 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie const multi = c.curl_multi_init() orelse return error.FailedToInitializeMulti; errdefer _ = c.curl_multi_cleanup(multi); + try errorMCheck(c.curl_multi_setopt(multi, c.CURLMOPT_MAX_HOST_CONNECTIONS, @as(c_long, opts.max_host_open))); + var handles = try Handles.init(allocator, client, ca_blob, &opts); errdefer handles.deinit(allocator); @@ -346,8 +348,7 @@ const Handles = struct { // pointer to opts is not stable, don't hold a reference to it! fn init(allocator: Allocator, client: *Client, ca_blob: ?c.curl_blob, opts: *const Http.Opts) !Handles { - const count = opts.max_concurrent_transfers; - std.debug.assert(count > 0); + const count = if (opts.max_concurrent == 0) 1 else opts.max_concurrent; const handles = try allocator.alloc(Handle, count); errdefer allocator.free(handles); diff --git a/src/http/Http.zig b/src/http/Http.zig index c30de564..52176568 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -230,11 +230,12 @@ pub fn errorMCheck(code: c.CURLMcode) errors.Multi!void { } pub const Opts = struct { - timeout_ms: u31 = 0, + timeout_ms: u31, + max_host_open: u8, + max_concurrent: u8, + connect_timeout_ms: u31, max_redirects: u8 = 10, tls_verify_host: bool = true, - connect_timeout_ms: u31 = 5000, - max_concurrent_transfers: u8 = 5, http_proxy: ?[:0]const u8 = null, proxy_bearer_token: ?[:0]const u8 = null, }; diff --git a/src/main.zig b/src/main.zig index 4a28a805..168fab00 100644 --- a/src/main.zig +++ b/src/main.zig @@ -87,6 +87,10 @@ fn run(alloc: Allocator) !void { .http_proxy = args.httpProxy(), .proxy_bearer_token = args.proxyBearerToken(), .tls_verify_host = args.tlsVerifyHost(), + .http_timeout_ms = args.httpTimeout(), + .http_connect_timeout_ms = args.httpConnectTiemout(), + .http_max_host_open = args.httpMaxHostOpen(), + .http_max_concurrent = args.httpMaxConcurrent(), }); defer app.deinit(); app.telemetry.record(.{ .run = {} }); @@ -169,6 +173,34 @@ const Command = struct { }; } + fn httpMaxConcurrent(self: *const Command) ?u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_max_concurrent, + else => unreachable, + }; + } + + fn httpMaxHostOpen(self: *const Command) ?u8 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_max_host_open, + else => unreachable, + }; + } + + fn httpConnectTiemout(self: *const Command) ?u31 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_connect_timeout, + else => unreachable, + }; + } + + fn httpTimeout(self: *const Command) ?u31 { + return switch (self.mode) { + inline .serve, .fetch => |opts| opts.common.http_timeout, + else => unreachable, + }; + } + fn logLevel(self: *const Command) ?log.Level { return switch (self.mode) { inline .serve, .fetch => |opts| opts.common.log_level, @@ -213,8 +245,12 @@ const Command = struct { }; const Common = struct { - http_proxy: ?[:0]const u8 = null, proxy_bearer_token: ?[:0]const u8 = null, + http_proxy: ?[:0]const u8 = null, + http_max_concurrent: ?u8 = null, + http_max_host_open: ?u8 = null, + http_timeout: ?u31 = null, + http_connect_timeout: ?u31 = null, tls_verify_host: bool = true, log_level: ?log.Level = null, log_format: ?log.Format = null, @@ -222,22 +258,39 @@ const Command = struct { }; fn printUsageAndExit(self: *const Command, success: bool) void { + // MAX_HELP_LEN| const common_options = \\ \\--insecure_disable_tls_host_verification - \\ Disables host verification on all HTTP requests. - \\ This is an advanced option which should only be - \\ set if you understand and accept the risk of - \\ disabling host verification. + \\ Disables host verification on all HTTP requests. This is an + \\ advanced option which should only be set if you understand + \\ and accept the risk of disabling host verification. \\ \\--http_proxy The HTTP proxy to use for all HTTP requests. - \\ A username:password can be included to use basic - \\ authentication. + \\ A username:password can be included for basic authentication. \\ Defaults to none. \\ \\--proxy_bearer_token - \\ The to send for bearer authentication with the proxy - \\ Proxy-Authorization: Bearer + \\ The to send for bearer authentication with the proxy + \\ Proxy-Authorization: Bearer + \\ + \\--http_max_concurrent + \\ The maximum number of concurrent HTTP requests. + \\ Defaults to 10. + \\ + \\--http_max_host_open + \\ The maximum number of open connection to a given host:port. + \\ Defaults to 4. + \\ + \\--http_connect_timeout + \\ The time, in milliseconds, for establishing an HTTP connection + \\ before timing out. 0 means it never times out. + \\ Defaults to 0. + \\ + \\--http_timeout + \\ The maximum time, in milliseconds, the transfer is allowed + \\ to complete. 0 means it never times out. + \\ Defaults to 10000. \\ \\--log_level The log level: debug, info, warn, error or fatal. \\ Defaults to @@ -248,9 +301,9 @@ const Command = struct { \\ Defaults to ++ (if (builtin.mode == .Debug) " pretty." else " logfmt.") ++ \\ - \\ ; + // MAX_HELP_LEN| const usage = \\usage: {s} command [options] [URL] \\ @@ -516,6 +569,58 @@ fn parseCommonArg( return true; } + if (std.mem.eql(u8, "--http_max_concurrent", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_max_concurrent" }); + return error.InvalidArgument; + }; + + common.http_max_concurrent = std.fmt.parseInt(u8, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_concurrent", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_max_host_open", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_max_host_open" }); + return error.InvalidArgument; + }; + + common.http_max_host_open = std.fmt.parseInt(u8, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_max_host_open", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_connect_timeout", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_connect_timeout" }); + return error.InvalidArgument; + }; + + common.http_connect_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_connect_timeout", .err = err }); + return error.InvalidArgument; + }; + return true; + } + + if (std.mem.eql(u8, "--http_timeout", opt)) { + const str = args.next() orelse { + log.fatal(.app, "missing argument value", .{ .arg = "--http_timeout" }); + return error.InvalidArgument; + }; + + common.http_timeout = std.fmt.parseInt(u31, str, 10) catch |err| { + log.fatal(.app, "invalid argument value", .{ .arg = "--http_timeout", .err = err }); + return error.InvalidArgument; + }; + return true; + } + if (std.mem.eql(u8, "--log_level", opt)) { const str = args.next() orelse { log.fatal(.app, "missing argument value", .{ .arg = "--log_level" }); @@ -663,6 +768,7 @@ fn serveCDP(address: std.net.Address, platform: *const Platform) !void { .run_mode = .serve, .tls_verify_host = false, .platform = platform, + .max_concurrent_transfers = 2, }); defer app.deinit(); From c96fb3c2f2ec26b7aedc91fd82d55f6e031276f7 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 6 Aug 2025 11:49:57 +0800 Subject: [PATCH 22/44] support CDP proxy override --- src/cdp/cdp.zig | 13 ++++++---- src/cdp/domains/target.zig | 7 +++--- src/http/Client.zig | 49 +++++++++++++++++++++++++++++++++++--- src/main.zig | 6 ++--- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 052b8af9..d859f925 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -338,10 +338,7 @@ pub fn BrowserContext(comptime CDP_T: type) type { inspector: Inspector, isolated_world: ?IsolatedWorld, - // Used to restore the proxy after the CDP session ends. If CDP never over-wrote it, it won't restore it (the first null). - // If the CDP is restoring it, but the original value was null, that's the 2nd null. - // If you only have 1 null it would be ambiguous, does null mean it shouldn't be restored, or should it be restored to null? - http_proxy_before: ??std.Uri = null, + http_proxy_changed: bool = false, const Self = @This(); @@ -397,7 +394,13 @@ pub fn BrowserContext(comptime CDP_T: type) type { self.node_search_list.deinit(); self.cdp.browser.notification.unregisterAll(self); - if (self.http_proxy_before) |prev_proxy| self.cdp.browser.http_client.http_proxy = prev_proxy; + if (self.http_proxy_changed) { + // has to be called after browser.closeSession, since it won't + // work if there are active connections. + self.cdp.browser.http_client.restoreOriginalProxy() catch |err| { + log.warn(.http, "restoreOriginalProxy", .{ .err = err }); + }; + } } pub fn reset(self: *Self) void { diff --git a/src/cdp/domains/target.zig b/src/cdp/domains/target.zig index 5b2a4f73..0a3ed2e0 100644 --- a/src/cdp/domains/target.zig +++ b/src/cdp/domains/target.zig @@ -68,7 +68,7 @@ fn getBrowserContexts(cmd: anytype) !void { fn createBrowserContext(cmd: anytype) !void { const params = try cmd.params(struct { disposeOnDetach: bool = false, - proxyServer: ?[]const u8 = null, + proxyServer: ?[:0]const u8 = null, proxyBypassList: ?[]const u8 = null, originsWithUniversalNetworkAccess: ?[]const []const u8 = null, }); @@ -84,9 +84,8 @@ fn createBrowserContext(cmd: anytype) !void { if (params) |p| { if (p.proxyServer) |proxy| { // For now the http client is not in the browser context so we assume there is just 1. - bc.http_proxy_before = cmd.cdp.browser.http_client.http_proxy; - const proxy_cp = try cmd.cdp.browser.http_client.allocator.dupe(u8, proxy); - cmd.cdp.browser.http_client.http_proxy = try std.Uri.parse(proxy_cp); + try cmd.cdp.browser.http_client.changeProxy(proxy); + bc.http_proxy_changed = true; } } diff --git a/src/http/Client.zig b/src/http/Client.zig index 84b10e7e..0493279d 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -71,9 +71,6 @@ allocator: Allocator, // request. These wil come and go with each request. transfer_pool: std.heap.MemoryPool(Transfer), -//@newhttp -http_proxy: ?std.Uri = null, - // see ScriptManager.blockingGet blocking: Handle, @@ -87,6 +84,10 @@ blocking_active: if (builtin.mode == .Debug) bool else void = if (builtin.mode = // can result in makeRequest being re-called (from a doneCallback). arena: ArenaAllocator, +// only needed for CDP which can change the proxy and then restore it. When +// restoring, this originally-configured value is what it goes to. +http_proxy: ?[:0]const u8 = null, + const RequestQueue = std.DoublyLinkedList(Request); pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Client { @@ -117,6 +118,7 @@ pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Clie .handles = handles, .blocking = blocking, .allocator = allocator, + .http_proxy = opts.http_proxy, .transfer_pool = transfer_pool, .queue_node_pool = queue_node_pool, .arena = ArenaAllocator.init(allocator), @@ -208,6 +210,36 @@ pub fn blockingRequest(self: *Client, req: Request) !void { return self.makeRequest(&self.blocking, req); } +// Restrictive since it'll only work if there are no inflight requests. In some +// cases, the libcurl documentation is clear that changing settings while a +// connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY, +// but better to be safe than sorry. +// For now, this restriction is ok, since it's only called by CDP on +// createBrowserContext, at which point, if we do have an active connection, +// that's probably a bug (a previous abort failed?). But if we need to call this +// at any point in time, it could be worth digging into libcurl to see if this +// can be changed at any point in the easy's lifecycle. +pub fn changeProxy(self: *Client, proxy: [:0]const u8) !void { + try self.ensureNoActiveConnection(); + + for (self.handles.handles) |h| { + try errorCheck(c.curl_easy_setopt(h.conn.easy, c.CURLOPT_PROXY, proxy.ptr)); + } + try errorCheck(c.curl_easy_setopt(self.blocking.conn.easy, c.CURLOPT_PROXY, proxy.ptr)); +} + +// Same restriction as changeProxy. Should be ok since this is only called on +// BrowserContext deinit. +pub fn restoreOriginalProxy(self: *Client) !void { + try self.ensureNoActiveConnection(); + + const proxy = if (self.http_proxy) |p| p.ptr else null; + for (self.handles.handles) |h| { + try errorCheck(c.curl_easy_setopt(h.conn.easy, c.CURLOPT_PROXY, proxy)); + } + try errorCheck(c.curl_easy_setopt(self.blocking.conn.easy, c.CURLOPT_PROXY, proxy)); +} + fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { const conn = handle.conn; const easy = conn.easy; @@ -339,6 +371,17 @@ fn endTransfer(self: *Client, transfer: *Transfer) void { self.active -= 1; } +fn ensureNoActiveConnection(self: *const Client) !void { + if (self.active > 0) { + return error.InflightConnection; + } + if (comptime builtin.mode == .Debug) { + if (self.blocking_active) { + return error.InflightConnection; + } + } +} + const Handles = struct { handles: []Handle, in_use: HandleList, diff --git a/src/main.zig b/src/main.zig index 168fab00..9b95cf5c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -258,7 +258,7 @@ const Command = struct { }; fn printUsageAndExit(self: *const Command, success: bool) void { - // MAX_HELP_LEN| + // MAX_HELP_LEN| const common_options = \\ \\--insecure_disable_tls_host_verification @@ -303,7 +303,7 @@ const Command = struct { \\ ; - // MAX_HELP_LEN| + // MAX_HELP_LEN| const usage = \\usage: {s} command [options] [URL] \\ @@ -768,7 +768,7 @@ fn serveCDP(address: std.net.Address, platform: *const Platform) !void { .run_mode = .serve, .tls_verify_host = false, .platform = platform, - .max_concurrent_transfers = 2, + .http_max_concurrent = 2, }); defer app.deinit(); From 3554634c1c1a6851aaedf26f23ca193b46a2d01d Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 6 Aug 2025 13:14:58 +0800 Subject: [PATCH 23/44] cleanup optional request headers --- src/browser/page.zig | 4 ++++ src/browser/storage/cookie.zig | 8 +++++++- src/browser/xhr/xhr.zig | 1 - src/http/Client.zig | 14 ++++++-------- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/browser/page.zig b/src/browser/page.zig index 54571262..a368409d 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -472,6 +472,7 @@ pub const Page = struct { .url = owned_url, .method = opts.method, .body = opts.body, + .header = opts.header, .cookie = self.requestCookie(.{ .is_navigation = true }), .header_done_callback = pageHeaderDoneCallback, .data_callback = pageDataCallback, @@ -931,6 +932,8 @@ pub const Page = struct { if (std.ascii.eqlIgnoreCase(method, "post")) { opts.method = .POST; opts.body = buf.items; + // form_data.write currently only supports this encoding, so we know this has to be the content type + opts.header = "Content-Type: application/x-www-form-urlencoded"; } else { action = try URL.concatQueryString(transfer_arena, action, buf.items); } @@ -986,6 +989,7 @@ pub const NavigateOpts = struct { reason: NavigateReason = .address_bar, method: HttpClient.Method = .GET, body: ?[]const u8 = null, + header: ?[:0]const u8 = null, }; fn timestamp() u32 { diff --git a/src/browser/storage/cookie.zig b/src/browser/storage/cookie.zig index 98104c0e..25c96071 100644 --- a/src/browser/storage/cookie.zig +++ b/src/browser/storage/cookie.zig @@ -12,6 +12,7 @@ pub const LookupOpts = struct { origin_uri: ?*const Uri = null, is_http: bool, is_navigation: bool = true, + prefix: ?[]const u8 = null, }; pub const Jar = struct { @@ -91,10 +92,15 @@ pub const Jar = struct { var first = true; for (self.cookies.items) |*cookie| { - if (!cookie.appliesTo(&target, same_site, opts.is_navigation, opts.is_http)) continue; + if (!cookie.appliesTo(&target, same_site, opts.is_navigation, opts.is_http)) { + continue; + } // we have a match! if (first) { + if (opts.prefix) |prefix| { + try writer.writeAll(prefix); + } first = false; } else { try writer.writeAll("; "); diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index a91967e7..94c54461 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -375,7 +375,6 @@ pub const XMLHttpRequest = struct { .url = self.url.?, .method = self.method, .body = self.request_body, - .content_type = "Content-Type: text/plain; charset=UTF-8", // @newhttp TODO .cookie = page.requestCookie(.{}), .start_callback = httpStartCallback, .header_callback = httpHeaderCallback, diff --git a/src/http/Client.zig b/src/http/Client.zig index 0493279d..c9aa6651 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -263,24 +263,21 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { var header_list = conn.commonHeaders(); errdefer c.curl_slist_free_all(header_list); - if (req.content_type) |ct| { - header_list = c.curl_slist_append(header_list, ct); + if (req.header) |hdr| { + header_list = c.curl_slist_append(header_list, hdr); } { - const COOKIE_HEADER = "Cookie: "; const aa = self.arena.allocator(); - defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); - var arr: std.ArrayListUnmanaged(u8) = .{}; - try arr.appendSlice(aa, COOKIE_HEADER); try req.cookie.forRequest(&uri, arr.writer(aa)); - if (arr.items.len > COOKIE_HEADER.len) { + if (arr.items.len > 0) { try arr.append(aa, 0); //null terminate // copies the value header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr)); + defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); } } @@ -485,6 +482,7 @@ pub const RequestCookie = struct { .is_http = self.is_http, .is_navigation = self.is_navigation, .origin_uri = self.origin, + .prefix = "Cookie: ", }); } }; @@ -493,7 +491,7 @@ pub const Request = struct { method: Method, url: [:0]const u8, body: ?[]const u8 = null, - content_type: ?[:0]const u8 = null, + header: ?[:0]const u8 = null, cookie: RequestCookie, // arbitrary data that can be associated with this request From 332e264437af638311df6acdf116debb96953df8 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 6 Aug 2025 15:23:27 +0800 Subject: [PATCH 24/44] remove unimportant todos --- src/browser/ScriptManager.zig | 6 ------ src/browser/page.zig | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index e5108dbd..0dff31f5 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -477,9 +477,6 @@ const PendingScript = struct { // will fail. This assertion exists to catch incorrect assumptions about // how libcurl works, or about how we've configured it. std.debug.assert(self.script.source.remote.capacity == 0); - - // @newhttp TODO: pre size based on content-length - // @newhttp TODO: max-length enfocement self.script.source = .{ .remote = self.manager.buffer_pool.get() }; } @@ -491,7 +488,6 @@ const PendingScript = struct { // .len = data.len, // }); - // @newhttp TODO: max-length enforcement ?? try self.script.source.remote.appendSlice(self.manager.allocator, data); } @@ -704,8 +700,6 @@ const BufferPool = struct { } fn release(self: *BufferPool, buffer: ArrayListUnmanaged(u8)) void { - // @newhttp TODO: discard buffers that are larger than some configured max? - // create mutable copy var b = buffer; diff --git a/src/browser/page.zig b/src/browser/page.zig index a368409d..633947fe 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -537,6 +537,7 @@ pub const Page = struct { .timestamp = timestamp(), }); } + fn _documentIsComplete(self: *Page) !void { try HTMLDocument.documentIsComplete(self.window.document, self); @@ -641,7 +642,8 @@ pub const Page = struct { while (try walker.get_next(root, next)) |n| { next = n; const node = next.?; - const tag = (try parser.nodeHTMLGetTagType(node)) orelse continue; + const e = parser.nodeToElement(node); + const tag = try parser.elementTag(e); if (tag != .script) { // ignore non-js script. continue; From ff742c01697a94500c4438e0d29d77d7a6fc9d36 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 7 Aug 2025 08:21:53 +0800 Subject: [PATCH 25/44] don't allow concurrent blocking calls --- src/browser/ScriptManager.zig | 22 ++++++++++++++++++++++ src/browser/session.zig | 8 +++++--- src/browser/xhr/xhr.zig | 12 ++++++------ src/http/Client.zig | 23 +++-------------------- 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 0dff31f5..44d3ef93 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -38,6 +38,9 @@ page: *Page, // used to prevent recursive evalution is_evaluating: bool, +// used to prevent executing scripts while we're doing a blocking load +is_blocking: bool = false, + // Only once this is true can deferred scripts be run static_scripts_done: bool, @@ -54,6 +57,7 @@ deferreds: OrderList, shutdown: bool = false, + client: *HttpClient, allocator: Allocator, buffer_pool: BufferPool, @@ -269,6 +273,17 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { // http handle, these are executed without there necessarily being a free handle. // Thus, Http/Client.zig maintains a dedicated handle for these calls. pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { + std.debug.assert(self.is_blocking == false); + + self.is_blocking = true; + defer { + self.is_blocking = false; + + // we blocked evaluation while loading this script, there could be + // scripts ready to process. + self.evaluate(); + } + var blocking = Blocking{ .allocator = self.allocator, .buffer_pool = &self.buffer_pool, @@ -314,6 +329,13 @@ fn evaluate(self: *ScriptManager) void { return; } + if (self.is_blocking) { + // Cannot evaluate scripts while a blocking-load is in progress. Not + // only could that result in incorrect evaluation order, it could + // triger another blocking request, while we're doing a blocking request. + return; + } + const page = self.page; self.is_evaluating = true; defer self.is_evaluating = false; diff --git a/src/browser/session.zig b/src/browser/session.zig index a6f83e9c..21b5f67f 100644 --- a/src/browser/session.zig +++ b/src/browser/session.zig @@ -118,13 +118,15 @@ pub const Session = struct { std.debug.assert(self.page != null); - self.page.?.deinit(); - self.page = null; - // RemoveJsContext() will execute the destructor of any type that // registered a destructor (e.g. XMLHttpRequest). + // Should be called before we deinit the page, because these objects + // could be referencing it. self.executor.removeJsContext(); + self.page.?.deinit(); + self.page = null; + // clear netsurf memory arena. parser.deinit(); diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index 94c54461..51393b61 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -171,12 +171,12 @@ pub const XMLHttpRequest = struct { }; } - // pub fn destructor(self: *XMLHttpRequest) void { - // if (self.transfer) |transfer| { - // transfer.abort(); - // self.transfer = null; - // } - // } + pub fn destructor(self: *XMLHttpRequest) void { + if (self.transfer) |transfer| { + transfer.abort(); + self.transfer = null; + } + } pub fn reset(self: *XMLHttpRequest) void { self.url = null; diff --git a/src/http/Client.zig b/src/http/Client.zig index c9aa6651..f2a26dfc 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -74,10 +74,6 @@ transfer_pool: std.heap.MemoryPool(Transfer), // see ScriptManager.blockingGet blocking: Handle, -// Boolean to check that we don't make a blocking request while an existing -// blocking request is already being processed. -blocking_active: if (builtin.mode == .Debug) bool else void = if (builtin.mode == .Debug) false else {}, - // The only place this is meant to be used is in `makeRequest` BEFORE `perform` // is called. It is used to generate our Cookie header. It can be used for other // purposes, but keep in mind that, while single-threaded, calls like makeRequest @@ -199,14 +195,6 @@ pub fn request(self: *Client, req: Request) !void { // See ScriptManager.blockingGet pub fn blockingRequest(self: *Client, req: Request) !void { - if (comptime builtin.mode == .Debug) { - std.debug.assert(self.blocking_active == false); - self.blocking_active = true; - } - defer if (comptime builtin.mode == .Debug) { - self.blocking_active = false; - }; - return self.makeRequest(&self.blocking, req); } @@ -361,7 +349,7 @@ fn endTransfer(self: *Client, transfer: *Transfer) void { self.transfer_pool.destroy(transfer); errorMCheck(c.curl_multi_remove_handle(self.multi, handle.conn.easy)) catch |err| { - log.fatal(.http, "Failed to abort", .{ .err = err }); + log.fatal(.http, "Failed to remove handle", .{ .err = err }); }; self.handles.release(handle); @@ -372,11 +360,6 @@ fn ensureNoActiveConnection(self: *const Client) !void { if (self.active > 0) { return error.InflightConnection; } - if (comptime builtin.mode == .Debug) { - if (self.blocking_active) { - return error.InflightConnection; - } - } } const Handles = struct { @@ -429,8 +412,8 @@ const Handles = struct { } fn release(self: *Handles, handle: *Handle) void { - // client.blocking is a handle without a node, it doesn't exist in the - // eitehr the in_use or available lists. + // client.blocking is a handle without a node, it doesn't exist in + // either the in_use or available lists. const node = &(handle.node orelse return); self.in_use.remove(node); From 079ce5e9dee1c0c8c2c81ba8188215ed832f9588 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 8 Aug 2025 08:36:11 +0800 Subject: [PATCH 26/44] whitelist application/ld+json --- src/browser/ScriptManager.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 44d3ef93..e94320ba 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -140,6 +140,10 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { if (std.ascii.eqlIgnoreCase(script_type, "application/json")) { return; } + if (std.ascii.eqlIgnoreCase(script_type, "application/ld+json")) { + return; + } + log.warn(.user_script, "unknown script type", .{ .type = script_type }); return; }; From 05192b6850ae920f08ef0e3f38bac03abb823c0b Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Mon, 11 Aug 2025 12:09:22 -0700 Subject: [PATCH 27/44] update flake --- flake.lock | 6 +++--- flake.nix | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index f593473f..1c140996 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1748964450, - "narHash": "sha256-ZouDiXkUk8mkMnah10QcoQ9Nu6UW6AFAHLScS3En6aI=", + "lastModified": 1754919767, + "narHash": "sha256-bc9tjR2ymbmbtYlnOcksjI7tQtDDEEJFGm41t0msXsg=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9ff500cd9e123f46c55855eca64beccead29b152", + "rev": "8c0c41355297485b39d6f6a6d722c8cdfe0257df", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 95efc02e..d902ef89 100644 --- a/flake.nix +++ b/flake.nix @@ -49,6 +49,7 @@ glib.dev glibc.dev zlib + zlib.dev ]; }; in From 19c908035b9f00cfd3b1dbc50962997e1e80f226 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 12 Aug 2025 11:13:19 +0800 Subject: [PATCH 28/44] Treat pending requests as active This ensures that page.wait won't unblock too early. As-is, this isn't an issue since active can only be 0 if there are no active OR pending requests. However, with request interception (https://github.com/lightpanda-io/browser/pull/930) it's possible to have no active requests and no pending requests - from the http client's point of view - but still have pending-on-intercept requests. An alternative to this would be to undo these changes, and instead change Page.wait to be intercept-aware. That is, Page.wait would continue to block on http activity and scheduled tasks, as well as intercepted requests. However, since the Page doesn't know anything about CDP right now, and it does know about the http client, maybe doing this in the client is fine. --- src/http/Client.zig | 15 ++++++++++++--- vendor/netsurf/libdom | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/http/Client.zig b/src/http/Client.zig index f2a26dfc..91efd30b 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -185,12 +185,14 @@ pub fn tick(self: *Client, timeout_ms: usize) !void { pub fn request(self: *Client, req: Request) !void { if (self.handles.getFreeHandle()) |handle| { + self.active += 1; return self.makeRequest(handle, req); } const node = try self.queue_node_pool.create(); node.data = req; self.queue.append(node); + self.active += 1; } // See ScriptManager.blockingGet @@ -234,13 +236,18 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { // we need this for cookies const uri = std.Uri.parse(req.url) catch |err| { + self.active -= 1; self.handles.release(handle); log.warn(.http, "invalid url", .{ .err = err, .url = req.url }); return; }; const header_list = blk: { - errdefer self.handles.release(handle); + errdefer { + self.active -= 1; + self.handles.release(handle); + } + try conn.setMethod(req.method); try conn.setURL(req.url); @@ -275,7 +282,10 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { }; { - errdefer self.handles.release(handle); + errdefer { + self.active -= 1; + self.handles.release(handle); + } const transfer = try self.transfer_pool.create(); transfer.* = .{ @@ -299,7 +309,6 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { } } - self.active += 1; return self.perform(0); } diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index 0c590b26..c0df4581 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit 0c590b265a65b937042d68ad34902c9b4a05839a +Subproject commit c0df458132162aba136d57ce1ba2179122a9e717 From ea0bbaf332aa6f9cf390522380685a258f2f0d77 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Tue, 12 Aug 2025 11:27:28 +0800 Subject: [PATCH 29/44] Revert "Treat pending requests as active" This reverts commit 19c908035b9f00cfd3b1dbc50962997e1e80f226. --- src/http/Client.zig | 15 +++------------ vendor/netsurf/libdom | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/http/Client.zig b/src/http/Client.zig index 91efd30b..f2a26dfc 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -185,14 +185,12 @@ pub fn tick(self: *Client, timeout_ms: usize) !void { pub fn request(self: *Client, req: Request) !void { if (self.handles.getFreeHandle()) |handle| { - self.active += 1; return self.makeRequest(handle, req); } const node = try self.queue_node_pool.create(); node.data = req; self.queue.append(node); - self.active += 1; } // See ScriptManager.blockingGet @@ -236,18 +234,13 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { // we need this for cookies const uri = std.Uri.parse(req.url) catch |err| { - self.active -= 1; self.handles.release(handle); log.warn(.http, "invalid url", .{ .err = err, .url = req.url }); return; }; const header_list = blk: { - errdefer { - self.active -= 1; - self.handles.release(handle); - } - + errdefer self.handles.release(handle); try conn.setMethod(req.method); try conn.setURL(req.url); @@ -282,10 +275,7 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { }; { - errdefer { - self.active -= 1; - self.handles.release(handle); - } + errdefer self.handles.release(handle); const transfer = try self.transfer_pool.create(); transfer.* = .{ @@ -309,6 +299,7 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { } } + self.active += 1; return self.perform(0); } diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index c0df4581..0c590b26 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit c0df458132162aba136d57ce1ba2179122a9e717 +Subproject commit 0c590b265a65b937042d68ad34902c9b4a05839a From 971524fa3bfa8aff13a25764a78f1ecb9c4d8f39 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Tue, 12 Aug 2025 12:55:44 +0200 Subject: [PATCH 30/44] finalize document loading with non-HTML pages Avoid infinite the loop of loading non-HTML documents with CDP. --- src/browser/page.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/browser/page.zig b/src/browser/page.zig index 633947fe..cb181f5b 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -608,7 +608,10 @@ pub const Page = struct { self.clearTransferArena(); switch (self.mode) { - .raw => |buf| self.mode = .{ .raw_done = buf.items }, + .raw => |buf| { + self.mode = .{ .raw_done = buf.items }; + self.documentIsComplete(); + }, .html => |*p| { const html_doc = p.html_doc; p.deinit(); // don't need the parser anymore From 03694b54f07b54ea98c8b8f5e2677a2edfb75ba9 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Tue, 12 Aug 2025 13:49:20 +0200 Subject: [PATCH 31/44] 3# This is a combination of 3 commits. intercept continue and abort feedback First version of headers, no cookies yet --- src/browser/ScriptManager.zig | 3 +- src/browser/browser.zig | 3 + src/browser/page.zig | 5 +- src/browser/xhr/xhr.zig | 11 +- src/cdp/cdp.zig | 21 +++- src/cdp/domains/fetch.zig | 202 +++++++++++++++++++++++++++++++++- src/cdp/domains/network.zig | 25 ++--- src/http/Client.zig | 70 +++++++----- src/http/Http.zig | 67 +++++++++-- src/notification.zig | 14 ++- 10 files changed, 357 insertions(+), 64 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index e94320ba..4d6c3e62 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -57,7 +57,6 @@ deferreds: OrderList, shutdown: bool = false, - client: *HttpClient, allocator: Allocator, buffer_pool: BufferPool, @@ -234,6 +233,7 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { .url = remote_url.?, .ctx = pending_script, .method = .GET, + .headers = try HttpClient.Headers.init(), .cookie = page.requestCookie(.{}), .start_callback = if (log.enabled(.http, .debug)) startCallback else null, .header_done_callback = headerCallback, @@ -297,6 +297,7 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { try client.blockingRequest(.{ .url = url, .method = .GET, + .headers = try HttpClient.Headers.init(), .ctx = &blocking, .cookie = self.page.requestCookie(.{}), .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null, diff --git a/src/browser/browser.zig b/src/browser/browser.zig index f836ef2d..1b5636e9 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -52,6 +52,8 @@ pub const Browser = struct { errdefer env.deinit(); const notification = try Notification.init(allocator, app.notification); + app.http.client.notification = notification; + app.http.client.next_request_id = 0; // Should we track ids in CDP only? errdefer notification.deinit(); return .{ @@ -74,6 +76,7 @@ pub const Browser = struct { self.page_arena.deinit(); self.session_arena.deinit(); self.transfer_arena.deinit(); + self.http_client.notification = null; self.notification.deinit(); self.state_pool.deinit(); } diff --git a/src/browser/page.zig b/src/browser/page.zig index cb181f5b..b2c89de2 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -467,12 +467,15 @@ pub const Page = struct { const owned_url = try self.arena.dupeZ(u8, request_url); self.url = try URL.parse(owned_url, null); + var headers = try HttpClient.Headers.init(); + if (opts.header) |hdr| try headers.add(hdr); + self.http_client.request(.{ .ctx = self, .url = owned_url, .method = opts.method, + .headers = headers, .body = opts.body, - .header = opts.header, .cookie = self.requestCookie(.{ .is_navigation = true }), .header_done_callback = pageHeaderDoneCallback, .data_callback = pageDataCallback, diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index 51393b61..f22796e5 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -370,10 +370,16 @@ pub const XMLHttpRequest = struct { } } + var headers = try HttpClient.Headers.init(); + for (self.headers.items) |hdr| { + try headers.add(hdr); + } + try page.http_client.request(.{ .ctx = self, .url = self.url.?, .method = self.method, + .headers = headers, .body = self.request_body, .cookie = page.requestCookie(.{}), .start_callback = httpStartCallback, @@ -387,11 +393,6 @@ pub const XMLHttpRequest = struct { fn httpStartCallback(transfer: *HttpClient.Transfer) !void { const self: *XMLHttpRequest = @alignCast(@ptrCast(transfer.ctx)); - - for (self.headers.items) |hdr| { - try transfer.addHeader(hdr); - } - log.debug(.http, "request start", .{ .method = self.method, .url = self.url, .source = "xhr" }); self.transfer = transfer; } diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index d859f925..696448c2 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -29,6 +29,7 @@ const Page = @import("../browser/page.zig").Page; const Inspector = @import("../browser/env.zig").Env.Inspector; const Incrementing = @import("../id.zig").Incrementing; const Notification = @import("../notification.zig").Notification; +const InterceptState = @import("domains/fetch.zig").InterceptState; const polyfill = @import("../browser/polyfill/polyfill.zig"); @@ -75,6 +76,8 @@ pub fn CDPT(comptime TypeProvider: type) type { // Extra headers to add to all requests. TBD under which conditions this should be reset. extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty, + intercept_state: InterceptState, + const Self = @This(); pub fn init(app: *App, client: TypeProvider.Client) !Self { @@ -89,6 +92,7 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), + .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -96,6 +100,7 @@ pub fn CDPT(comptime TypeProvider: type) type { if (self.browser_context) |*bc| { bc.deinit(); } + self.intercept_state.deinit(); // TBD Should this live in BC? self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -451,6 +456,14 @@ pub fn BrowserContext(comptime CDP_T: type) type { self.cdp.browser.notification.unregister(.http_request_complete, self); } + pub fn fetchEnable(self: *Self) !void { + try self.cdp.browser.notification.register(.http_request_intercept, self, onHttpRequestIntercept); + } + + pub fn fetchDisable(self: *Self) void { + self.cdp.browser.notification.unregister(.http_request_intercept, self); + } + pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void { const self: *Self = @alignCast(@ptrCast(ctx)); return @import("domains/page.zig").pageRemove(self); @@ -475,7 +488,13 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onHttpRequestStart(ctx: *anyopaque, data: *const Notification.RequestStart) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - return @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data); + try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, data); + } + + pub fn onHttpRequestIntercept(ctx: *anyopaque, data: *const Notification.RequestIntercept) !void { + const self: *Self = @alignCast(@ptrCast(ctx)); + defer self.resetNotificationArena(); + try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); } pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index ea87cc92..708e3d9a 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -1,4 +1,4 @@ -// Copyright (C) 2023-2024 Lightpanda (Selecy SAS) +// Copyright (C) 2023-2025 Lightpanda (Selecy SAS) // // Francis Bouvier // Pierre Tachoire @@ -17,13 +17,211 @@ // along with this program. If not, see . const std = @import("std"); +const Allocator = std.mem.Allocator; +const Notification = @import("../../notification.zig").Notification; +const log = @import("../../log.zig"); +const Request = @import("../../http/Client.zig").Request; +const Method = @import("../../http/Client.zig").Method; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { disable, + enable, + continueRequest, + failRequest, }, cmd.input.action) orelse return error.UnknownMethod; switch (action) { - .disable => return cmd.sendResult(null, .{}), + .disable => return disable(cmd), + .enable => return enable(cmd), + .continueRequest => return continueRequest(cmd), + .failRequest => return failRequest(cmd), } } + +// Stored in CDP +pub const InterceptState = struct { + const Self = @This(); + waiting: std.AutoArrayHashMap(u64, Request), + + pub fn init(allocator: Allocator) !InterceptState { + return .{ + .waiting = std.AutoArrayHashMap(u64, Request).init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.waiting.deinit(); + } +}; + +const RequestPattern = struct { + urlPattern: []const u8 = "*", // Wildcards ('*' -> zero or more, '?' -> exactly one) are allowed. Escape character is backslash. Omitting is equivalent to "*". + resourceType: ?ResourceType = null, + requestStage: RequestStage = .Request, +}; +const ResourceType = enum { + Document, + Stylesheet, + Image, + Media, + Font, + Script, + TextTrack, + XHR, + Fetch, + Prefetch, + EventSource, + WebSocket, + Manifest, + SignedExchange, + Ping, + CSPViolationReport, + Preflight, + FedCM, + Other, +}; +const RequestStage = enum { + Request, + Response, +}; + +const EnableParam = struct { + patterns: []RequestPattern = &.{}, + handleAuthRequests: bool = false, +}; +const ErrorReason = enum { + Failed, + Aborted, + TimedOut, + AccessDenied, + ConnectionClosed, + ConnectionReset, + ConnectionRefused, + ConnectionAborted, + ConnectionFailed, + NameNotResolved, + InternetDisconnected, + AddressUnreachable, + BlockedByClient, + BlockedByResponse, +}; + +fn disable(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + bc.fetchDisable(); + return cmd.sendResult(null, .{}); +} + +fn enable(cmd: anytype) !void { + const params = (try cmd.params(EnableParam)) orelse EnableParam{}; + if (params.patterns.len != 0) log.warn(.cdp, "Fetch.enable No patterns yet", .{}); + if (params.handleAuthRequests) log.warn(.cdp, "Fetch.enable No auth yet", .{}); + + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + try bc.fetchEnable(); + + return cmd.sendResult(null, .{}); +} + +pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notification.RequestIntercept) !void { + var cdp = bc.cdp; + + // unreachable because we _have_ to have a page. + const session_id = bc.session_id orelse unreachable; + const target_id = bc.target_id orelse unreachable; + + // We keep it around to wait for modifications to the request. + // NOTE: we assume whomever created the request created it with a lifetime of the Page. + // TODO: What to do when receiving replies for a previous page's requests? + + try cdp.intercept_state.waiting.put(intercept.request.id.?, intercept.request.*); + + // NOTE: .request data preparation is duped from network.zig + const full_request_url = try std.Uri.parse(intercept.request.url); + const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{ + .scheme = true, + .authentication = true, + .authority = true, + .path = true, + .query = true, + }); + const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{ + .fragment = true, + }); + const headers = try intercept.request.headers.asHashMap(arena); + // End of duped code + + try cdp.sendEvent("Fetch.requestPaused", .{ + .requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{intercept.request.id.?}), + .request = .{ + .url = request_url, + .urlFragment = request_fragment, + .method = @tagName(intercept.request.method), + .hasPostData = intercept.request.body != null, + .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, + }, + .frameId = target_id, + .resourceType = ResourceType.Document, // TODO! + .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{intercept.request.id.?}), + }, .{ .session_id = session_id }); + + // Await either continueRequest, failRequest or fulfillRequest + intercept.wait_for_interception.* = true; +} + +const HeaderEntry = struct { + name: []const u8, + value: []const u8, +}; + +fn continueRequest(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + const params = (try cmd.params(struct { + requestId: []const u8, // "INTERCEPT-{d}" + url: ?[]const u8 = null, + method: ?[]const u8 = null, + postData: ?[]const u8 = null, + headers: ?[]const HeaderEntry = null, + interceptResponse: bool = false, + })) orelse return error.InvalidParams; + if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams; + + const request_id = try idFromRequestId(params.requestId); + var waiting_request = (bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound).value; + + // Update the request with the new parameters + if (params.url) |url| { + // The request url must be modified in a way that's not observable by page. So page.url is not updated. + waiting_request.url = try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url); + } + if (params.method) |method| { + waiting_request.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams; + } + + log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId }); + try bc.cdp.browser.http_client.request(waiting_request); + + return cmd.sendResult(null, .{}); +} + +fn failRequest(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; + var state = &bc.cdp.intercept_state; + const params = (try cmd.params(struct { + requestId: []const u8, // "INTERCEPT-{d}" + errorReason: ErrorReason, + })) orelse return error.InvalidParams; + + const request_id = try idFromRequestId(params.requestId); + if (state.waiting.fetchSwapRemove(request_id) == null) return error.RequestNotFound; + + log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason }); + return cmd.sendResult(null, .{}); +} + +// Get u64 from requestId which is formatted as: "INTERCEPT-{d}" +fn idFromRequestId(request_id: []const u8) !u64 { + if (!std.mem.startsWith(u8, request_id, "INTERCEPT-")) return error.InvalidParams; + return std.fmt.parseInt(u64, request_id[10..], 10) catch return error.InvalidParams; +} diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 44701fae..164cc745 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -223,7 +223,7 @@ pub fn httpRequestFail(arena: Allocator, bc: anytype, request: *const Notificati }, .{ .session_id = session_id }); } -pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notification.RequestStart) !void { +pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification.RequestStart) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -251,36 +251,31 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, request: *const Notificat .query = true, }); - const request_url = try urlToString(arena, request.url, .{ + const full_request_url = try std.Uri.parse(data.request.url); + const request_url = try urlToString(arena, &full_request_url, .{ .scheme = true, .authentication = true, .authority = true, .path = true, .query = true, }); - - const request_fragment = try urlToString(arena, request.url, .{ - .fragment = true, + const request_fragment = try urlToString(arena, &full_request_url, .{ + .fragment = true, // TODO since path is false, this likely does not work as intended }); - // @newhttp - const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - // try headers.ensureTotalCapacity(arena, request.headers.items.len); - // for (request.headers.items) |header| { - // headers.putAssumeCapacity(header.name, header.value); - // } + const headers = try data.request.headers.asHashMap(arena); // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.requestWillBeSent", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.request.id.?}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = document_url, .request = .{ .url = request_url, .urlFragment = request_fragment, - .method = @tagName(request.method), - .hasPostData = request.has_body, + .method = @tagName(data.request.method), + .hasPostData = data.request.body != null, .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, }, }, .{ .session_id = session_id }); @@ -326,7 +321,7 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi }, .{ .session_id = session_id }); } -fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 { +pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToStreamOptions) ![]const u8 { var buf: std.ArrayListUnmanaged(u8) = .empty; try url.writeToStream(opts, buf.writer(arena)); return buf.items; diff --git a/src/http/Client.zig b/src/http/Client.zig index f2a26dfc..56673d22 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -20,6 +20,8 @@ const std = @import("std"); const log = @import("../log.zig"); const builtin = @import("builtin"); const Http = @import("Http.zig"); +pub const Headers = Http.Headers; +const Notification = @import("../notification.zig").Notification; const c = Http.c; @@ -57,6 +59,9 @@ multi: *c.CURLM, // of easys. handles: Handles, +// Use to generate the next request ID +next_request_id: u64 = 0, + // When handles has no more available easys, requests get queued. queue: RequestQueue, @@ -74,6 +79,9 @@ transfer_pool: std.heap.MemoryPool(Transfer), // see ScriptManager.blockingGet blocking: Handle, +// To notify registered subscribers of events, the browser sets/nulls this for us. +notification: ?*Notification = null, + // The only place this is meant to be used is in `makeRequest` BEFORE `perform` // is called. It is used to generate our Cookie header. It can be used for other // purposes, but keep in mind that, while single-threaded, calls like makeRequest @@ -184,12 +192,26 @@ pub fn tick(self: *Client, timeout_ms: usize) !void { } pub fn request(self: *Client, req: Request) !void { + var req_copy = req; // We need it mutable + + if (req_copy.id == null) { // If the ID has already been set that means the request was previously intercepted + req_copy.id = self.next_request_id; + self.next_request_id += 1; + if (self.notification) |notification| { + notification.dispatch(.http_request_start, &.{ .request = &req_copy }); + + var wait_for_interception = false; + notification.dispatch(.http_request_intercept, &.{ .request = &req_copy, .wait_for_interception = &wait_for_interception }); + if (wait_for_interception) return; // The user is send an invitation to intercept this request. + } + } + if (self.handles.getFreeHandle()) |handle| { - return self.makeRequest(handle, req); + return self.makeRequest(handle, req_copy); } const node = try self.queue_node_pool.create(); - node.data = req; + node.data = req_copy; self.queue.append(node); } @@ -239,7 +261,8 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { return; }; - const header_list = blk: { + var header_list = req.headers; + { errdefer self.handles.release(handle); try conn.setMethod(req.method); try conn.setURL(req.url); @@ -248,31 +271,23 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { try conn.setBody(b); } - var header_list = conn.commonHeaders(); - errdefer c.curl_slist_free_all(header_list); + // { // TODO move up to `fn request()` + // const aa = self.arena.allocator(); + // var arr: std.ArrayListUnmanaged(u8) = .{}; + // try req.cookie.forRequest(&uri, arr.writer(aa)); - if (req.header) |hdr| { - header_list = c.curl_slist_append(header_list, hdr); - } + // if (arr.items.len > 0) { + // try arr.append(aa, 0); //null terminate - { - const aa = self.arena.allocator(); - var arr: std.ArrayListUnmanaged(u8) = .{}; - try req.cookie.forRequest(&uri, arr.writer(aa)); + // // copies the value + // header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr)); + // defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); + // } + // } - if (arr.items.len > 0) { - try arr.append(aa, 0); //null terminate - - // copies the value - header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr)); - defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); - } - } - - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list)); - - break :blk header_list; - }; + try conn.secretHeaders(&header_list); // Add headers that must be hidden from intercepts + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); + } { errdefer self.handles.release(handle); @@ -284,7 +299,7 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { .req = req, .ctx = req.ctx, .handle = handle, - ._request_header_list = header_list, + ._request_header_list = header_list.headers, }; errdefer self.transfer_pool.destroy(transfer); @@ -471,10 +486,11 @@ pub const RequestCookie = struct { }; pub const Request = struct { + id: ?u64 = null, method: Method, url: [:0]const u8, + headers: Headers, body: ?[]const u8 = null, - header: ?[:0]const u8 = null, cookie: RequestCookie, // arbitrary data that can be associated with this request diff --git a/src/http/Http.zig b/src/http/Http.zig index 52176568..b4596524 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -185,20 +185,20 @@ pub const Connection = struct { try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_POSTFIELDS, body.ptr)); } - pub fn commonHeaders(self: *const Connection) *c.curl_slist { - var header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0"); + // These are headers that may not be send to the users for inteception. + pub fn secretHeaders(self: *const Connection, headers: *Headers) !void { if (self.opts.proxy_bearer_token) |hdr| { - header_list = c.curl_slist_append(header_list, hdr); + try headers.add(hdr); } - return header_list; } pub fn request(self: *const Connection) !u16 { const easy = self.easy; - const header_list = self.commonHeaders(); - defer c.curl_slist_free_all(header_list); - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list)); + const header_list = try Headers.init(); + defer header_list.deinit(); + try self.secretHeaders(&header_list); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); try errorCheck(c.curl_easy_perform(easy)); var http_code: c_long = undefined; @@ -210,6 +210,59 @@ pub const Connection = struct { } }; +pub const Headers = struct { + headers: *c.curl_slist, + + pub fn init() !Headers { + const header_list = c.curl_slist_append(null, "User-Agent: Lightpanda/1.0"); + if (header_list == null) return error.OutOfMemory; + return .{ .headers = header_list }; + } + + pub fn deinit(self: *Headers) void { + c.curl_slist_free_all(self.headers); + } + + pub fn add(self: *Headers, header: [*c]const u8) !void { + const updated_headers = c.curl_slist_append(self.headers, header); + if (updated_headers == null) return error.OutOfMemory; + self.headers = updated_headers; + } + + pub fn asHashMap(self: *const Headers, allocator: Allocator) !std.StringArrayHashMapUnmanaged([]const u8) { + var list: std.StringArrayHashMapUnmanaged([]const u8) = .empty; + try list.ensureTotalCapacity(allocator, self.count()); + + var current: [*c]c.curl_slist = self.headers; + while (current) |node| { + const str = std.mem.span(@as([*:0]const u8, @ptrCast(node.*.data))); + const header = parseHeader(str) orelse return error.InvalidHeader; + list.putAssumeCapacity(header.name, header.value); + current = node.*.next; + } + return list; + } + + fn parseHeader(header_str: []const u8) ?struct { name: []const u8, value: []const u8 } { + const colon_pos = std.mem.indexOf(u8, header_str, ":") orelse return null; + + const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); + const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t"); + + return .{ .name = name, .value = value }; + } + + pub fn count(self: *const Headers) usize { + var current: [*c]c.curl_slist = self.headers; + var num: usize = 0; + while (current) |node| { + num += 1; + current = node.*.next; + } + return num; + } +}; + pub fn errorCheck(code: c.CURLcode) errors.Error!void { if (code == c.CURLE_OK) { return; diff --git a/src/notification.zig b/src/notification.zig index 7caed088..e351e9b3 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -4,6 +4,7 @@ const log = @import("log.zig"); const URL = @import("url.zig").URL; const page = @import("browser/page.zig"); const Http = @import("http/Http.zig"); +const Request = @import("http/Client.zig").Request; const Allocator = std.mem.Allocator; @@ -61,6 +62,7 @@ pub const Notification = struct { page_navigated: List = .{}, http_request_fail: List = .{}, http_request_start: List = .{}, + http_request_intercept: List = .{}, http_request_complete: List = .{}, notification_created: List = .{}, }; @@ -72,6 +74,7 @@ pub const Notification = struct { page_navigated: *const PageNavigated, http_request_fail: *const RequestFail, http_request_start: *const RequestStart, + http_request_intercept: *const RequestIntercept, http_request_complete: *const RequestComplete, notification_created: *Notification, }; @@ -91,11 +94,12 @@ pub const Notification = struct { }; pub const RequestStart = struct { - arena: Allocator, - id: usize, - url: *const std.Uri, - method: Http.Method, - has_body: bool, + request: *Request, + }; + + pub const RequestIntercept = struct { + request: *Request, + wait_for_interception: *bool, }; pub const RequestFail = struct { From 77eee7f087b69854409869bb4457ec6398ab247b Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Tue, 12 Aug 2025 14:40:23 +0200 Subject: [PATCH 32/44] Cookies --- src/browser/ScriptManager.zig | 14 ++++++++++---- src/browser/page.zig | 3 ++- src/browser/xhr/xhr.zig | 3 ++- src/http/Client.zig | 34 ++++++++++++++++------------------ src/http/Http.zig | 1 + 5 files changed, 31 insertions(+), 24 deletions(-) diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 4d6c3e62..3d5b86bd 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -229,12 +229,15 @@ pub fn addFromElement(self: *ScriptManager, element: *parser.Element) !void { errdefer pending_script.deinit(); + var headers = try HttpClient.Headers.init(); + try page.requestCookie(.{}).headersForRequest(self.allocator, remote_url.?, &headers); + try self.client.request(.{ .url = remote_url.?, .ctx = pending_script, .method = .GET, - .headers = try HttpClient.Headers.init(), - .cookie = page.requestCookie(.{}), + .headers = headers, + .cookie_jar = page.cookie_jar, .start_callback = if (log.enabled(.http, .debug)) startCallback else null, .header_done_callback = headerCallback, .data_callback = dataCallback, @@ -293,13 +296,16 @@ pub fn blockingGet(self: *ScriptManager, url: [:0]const u8) !BlockingResult { .buffer_pool = &self.buffer_pool, }; + var headers = try HttpClient.Headers.init(); + try self.page.requestCookie(.{}).headersForRequest(self.allocator, url, &headers); + var client = self.client; try client.blockingRequest(.{ .url = url, .method = .GET, - .headers = try HttpClient.Headers.init(), + .headers = headers, + .cookie_jar = self.page.cookie_jar, .ctx = &blocking, - .cookie = self.page.requestCookie(.{}), .start_callback = if (log.enabled(.http, .debug)) Blocking.startCallback else null, .header_done_callback = Blocking.headerCallback, .data_callback = Blocking.dataCallback, diff --git a/src/browser/page.zig b/src/browser/page.zig index b2c89de2..cfe261f9 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -469,6 +469,7 @@ pub const Page = struct { var headers = try HttpClient.Headers.init(); if (opts.header) |hdr| try headers.add(hdr); + try self.requestCookie(.{ .is_navigation = true }).headersForRequest(self.arena, owned_url, &headers); self.http_client.request(.{ .ctx = self, @@ -476,7 +477,7 @@ pub const Page = struct { .method = opts.method, .headers = headers, .body = opts.body, - .cookie = self.requestCookie(.{ .is_navigation = true }), + .cookie_jar = self.cookie_jar, .header_done_callback = pageHeaderDoneCallback, .data_callback = pageDataCallback, .done_callback = pageDoneCallback, diff --git a/src/browser/xhr/xhr.zig b/src/browser/xhr/xhr.zig index f22796e5..720b813b 100644 --- a/src/browser/xhr/xhr.zig +++ b/src/browser/xhr/xhr.zig @@ -374,6 +374,7 @@ pub const XMLHttpRequest = struct { for (self.headers.items) |hdr| { try headers.add(hdr); } + try page.requestCookie(.{}).headersForRequest(self.arena, self.url.?, &headers); try page.http_client.request(.{ .ctx = self, @@ -381,7 +382,7 @@ pub const XMLHttpRequest = struct { .method = self.method, .headers = headers, .body = self.request_body, - .cookie = page.requestCookie(.{}), + .cookie_jar = page.cookie_jar, .start_callback = httpStartCallback, .header_callback = httpHeaderCallback, .header_done_callback = httpHeaderDoneCallback, diff --git a/src/http/Client.zig b/src/http/Client.zig index 56673d22..41767772 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -22,6 +22,7 @@ const builtin = @import("builtin"); const Http = @import("Http.zig"); pub const Headers = Http.Headers; const Notification = @import("../notification.zig").Notification; +const storage = @import("../browser/storage/storage.zig"); const c = Http.c; @@ -271,20 +272,6 @@ fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { try conn.setBody(b); } - // { // TODO move up to `fn request()` - // const aa = self.arena.allocator(); - // var arr: std.ArrayListUnmanaged(u8) = .{}; - // try req.cookie.forRequest(&uri, arr.writer(aa)); - - // if (arr.items.len > 0) { - // try arr.append(aa, 0); //null terminate - - // // copies the value - // header_list = c.curl_slist_append(header_list, @ptrCast(arr.items.ptr)); - // defer _ = self.arena.reset(.{ .retain_with_limit = 2048 }); - // } - // } - try conn.secretHeaders(&header_list); // Add headers that must be hidden from intercepts try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); } @@ -475,13 +462,24 @@ pub const RequestCookie = struct { origin: *const std.Uri, jar: *@import("../browser/storage/cookie.zig").Jar, - fn forRequest(self: *const RequestCookie, uri: *const std.Uri, writer: anytype) !void { - return self.jar.forRequest(uri, writer, .{ + pub fn headersForRequest(self: *const RequestCookie, temp: Allocator, url: [:0]const u8, headers: *Headers) !void { + const uri = std.Uri.parse(url) catch |err| { + log.warn(.http, "invalid url", .{ .err = err, .url = url }); + return error.InvalidUrl; + }; + + var arr: std.ArrayListUnmanaged(u8) = .{}; + try self.jar.forRequest(&uri, arr.writer(temp), .{ .is_http = self.is_http, .is_navigation = self.is_navigation, .origin_uri = self.origin, .prefix = "Cookie: ", }); + + if (arr.items.len > 0) { + try arr.append(temp, 0); //null terminate + try headers.add(@ptrCast(arr.items.ptr)); + } } }; @@ -491,7 +489,7 @@ pub const Request = struct { url: [:0]const u8, headers: Headers, body: ?[]const u8 = null, - cookie: RequestCookie, + cookie_jar: *storage.CookieJar, // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, @@ -616,7 +614,7 @@ pub const Transfer = struct { if (header.len > SET_COOKIE_LEN) { if (std.ascii.eqlIgnoreCase(header[0..SET_COOKIE_LEN], "set-cookie:")) { const value = std.mem.trimLeft(u8, header[SET_COOKIE_LEN..], " "); - transfer.req.cookie.jar.populateFromResponse(&transfer.uri, value) catch |err| { + transfer.req.cookie_jar.populateFromResponse(&transfer.uri, value) catch |err| { log.err(.http, "set cookie", .{ .err = err, .req = transfer }); }; } diff --git a/src/http/Http.zig b/src/http/Http.zig index b4596524..aa5c59a2 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -224,6 +224,7 @@ pub const Headers = struct { } pub fn add(self: *Headers, header: [*c]const u8) !void { + // Copies the value const updated_headers = c.curl_slist_append(self.headers, header); if (updated_headers == null) return error.OutOfMemory; self.headers = updated_headers; From a49154acf4df3e57b4845abf7386c16522f2a6d3 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Tue, 12 Aug 2025 15:20:48 +0200 Subject: [PATCH 33/44] http_request_fail --- src/cdp/domains/network.zig | 6 +++--- src/http/Client.zig | 23 +++++++++++++++++++---- src/notification.zig | 5 ++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 164cc745..f143ff52 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -203,7 +203,7 @@ fn putAssumeCapacity(headers: *std.ArrayListUnmanaged(std.http.Header), extra: s return true; } -pub fn httpRequestFail(arena: Allocator, bc: anytype, request: *const Notification.RequestFail) !void { +pub fn httpRequestFail(arena: Allocator, bc: anytype, data: *const Notification.RequestFail) !void { // It's possible that the request failed because we aborted when the client // sent Target.closeTarget. In that case, bc.session_id will be cleared // already, and we can skip sending these messages to the client. @@ -215,10 +215,10 @@ pub fn httpRequestFail(arena: Allocator, bc: anytype, request: *const Notificati // We're missing a bunch of fields, but, for now, this seems like enough try bc.cdp.sendEvent("Network.loadingFailed", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.request.id.?}), // Seems to be what chrome answers with. I assume it depends on the type of error? .type = "Ping", - .errorText = request.err, + .errorText = data.err, .canceled = false, }, .{ .session_id = session_id }); } diff --git a/src/http/Client.zig b/src/http/Client.zig index 41767772..c72da2ee 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -151,7 +151,7 @@ pub fn abort(self: *Client) void { log.err(.http, "get private info", .{ .err = err, .source = "abort" }); continue; }; - transfer.req.error_callback(transfer.ctx, error.Abort); + self.requestFailed(&transfer.req, error.Abort); self.endTransfer(transfer); } std.debug.assert(self.active == 0); @@ -221,6 +221,20 @@ pub fn blockingRequest(self: *Client, req: Request) !void { return self.makeRequest(&self.blocking, req); } +fn requestFailed(self: *Client, req: *Request, err: anyerror) void { + if (req._notified_fail) return; + req._notified_fail = true; + + if (self.notification) |notification| { + notification.dispatch(.http_request_fail, &.{ + .request = req, + .err = err, + }); + } + + req.error_callback(req.ctx, err); +} + // Restrictive since it'll only work if there are no inflight requests. In some // cases, the libcurl documentation is clear that changing settings while a // connection is inflight is undefined. It doesn't say anything about CURLOPT_PROXY, @@ -326,7 +340,6 @@ fn perform(self: *Client, timeout_ms: c_int) !void { const transfer = try Transfer.fromEasy(easy); const ctx = transfer.ctx; const done_callback = transfer.req.done_callback; - const error_callback = transfer.req.error_callback; // release it ASAP so that it's available; some done_callbacks // will load more resources. @@ -336,10 +349,10 @@ fn perform(self: *Client, timeout_ms: c_int) !void { done_callback(ctx) catch |err| { // transfer isn't valid at this point, don't use it. log.err(.http, "done_callback", .{ .err = err }); - error_callback(ctx, err); + self.requestFailed(&transfer.req, err); }; } else |err| { - error_callback(ctx, err); + self.requestFailed(&transfer.req, err); } } } @@ -491,6 +504,8 @@ pub const Request = struct { body: ?[]const u8 = null, cookie_jar: *storage.CookieJar, + _notified_fail: bool = false, + // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, diff --git a/src/notification.zig b/src/notification.zig index e351e9b3..6f2e29c9 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -103,9 +103,8 @@ pub const Notification = struct { }; pub const RequestFail = struct { - id: usize, - url: *const std.Uri, - err: []const u8, + request: *Request, + err: anyerror, }; pub const RequestComplete = struct { From ca9e850ac7d72c1e9f7c96c4c46114201c291ba7 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 13 Aug 2025 18:05:00 +0800 Subject: [PATCH 34/44] Create Client.Transfer earlier. On client.request(req) we now immediately wrap the request into a Transfer. This results in less copying of the Request object. It also makes the transfer.uri available, so CDP no longer needs to std.Uri(request.url) anymore. The main advantage is that it's easier to manage resources. There was a use- after free before due to the sensitive nature of the tranfer's lifetime. There were also corner cases where some resources might not be freed. This is hopefully fixed with the lifetime of Transfer being extended. --- src/cdp/domains/fetch.zig | 34 ++++--- src/cdp/domains/network.zig | 13 +-- src/http/Client.zig | 195 +++++++++++++++++++++--------------- src/http/Http.zig | 2 +- src/notification.zig | 8 +- 5 files changed, 144 insertions(+), 108 deletions(-) diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index 708e3d9a..6b02ae58 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -20,8 +20,8 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Notification = @import("../../notification.zig").Notification; const log = @import("../../log.zig"); -const Request = @import("../../http/Client.zig").Request; const Method = @import("../../http/Client.zig").Method; +const Transfer = @import("../../http/Client.zig").Transfer; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { @@ -42,11 +42,11 @@ pub fn processMessage(cmd: anytype) !void { // Stored in CDP pub const InterceptState = struct { const Self = @This(); - waiting: std.AutoArrayHashMap(u64, Request), + waiting: std.AutoArrayHashMap(u64, *Transfer), pub fn init(allocator: Allocator) !InterceptState { return .{ - .waiting = std.AutoArrayHashMap(u64, Request).init(allocator), + .waiting = std.AutoArrayHashMap(u64, *Transfer).init(allocator), }; } @@ -135,10 +135,11 @@ pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notificati // NOTE: we assume whomever created the request created it with a lifetime of the Page. // TODO: What to do when receiving replies for a previous page's requests? - try cdp.intercept_state.waiting.put(intercept.request.id.?, intercept.request.*); + const transfer = intercept.transfer; + try cdp.intercept_state.waiting.put(transfer.id, transfer); // NOTE: .request data preparation is duped from network.zig - const full_request_url = try std.Uri.parse(intercept.request.url); + const full_request_url = transfer.uri; const request_url = try @import("network.zig").urlToString(arena, &full_request_url, .{ .scheme = true, .authentication = true, @@ -149,21 +150,21 @@ pub fn requestPaused(arena: Allocator, bc: anytype, intercept: *const Notificati const request_fragment = try @import("network.zig").urlToString(arena, &full_request_url, .{ .fragment = true, }); - const headers = try intercept.request.headers.asHashMap(arena); + const headers = try transfer.req.headers.asHashMap(arena); // End of duped code try cdp.sendEvent("Fetch.requestPaused", .{ - .requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{intercept.request.id.?}), + .requestId = try std.fmt.allocPrint(arena, "INTERCEPT-{d}", .{transfer.id}), .request = .{ .url = request_url, .urlFragment = request_fragment, - .method = @tagName(intercept.request.method), - .hasPostData = intercept.request.body != null, + .method = @tagName(transfer.req.method), + .hasPostData = transfer.req.body != null, .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, }, .frameId = target_id, .resourceType = ResourceType.Document, // TODO! - .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{intercept.request.id.?}), + .networkId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), }, .{ .session_id = session_id }); // Await either continueRequest, failRequest or fulfillRequest @@ -188,19 +189,20 @@ fn continueRequest(cmd: anytype) !void { if (params.postData != null or params.headers != null or params.interceptResponse) return error.NotYetImplementedParams; const request_id = try idFromRequestId(params.requestId); - var waiting_request = (bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound).value; + const entry = bc.cdp.intercept_state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound; + const transfer = entry.value; // Update the request with the new parameters if (params.url) |url| { // The request url must be modified in a way that's not observable by page. So page.url is not updated. - waiting_request.url = try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url); + try transfer.updateURL(try bc.cdp.browser.page_arena.allocator().dupeZ(u8, url)); } if (params.method) |method| { - waiting_request.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams; + transfer.req.method = std.meta.stringToEnum(Method, method) orelse return error.InvalidParams; } log.info(.cdp, "Request continued by intercept", .{ .id = params.requestId }); - try bc.cdp.browser.http_client.request(waiting_request); + try bc.cdp.browser.http_client.process(transfer); return cmd.sendResult(null, .{}); } @@ -214,7 +216,9 @@ fn failRequest(cmd: anytype) !void { })) orelse return error.InvalidParams; const request_id = try idFromRequestId(params.requestId); - if (state.waiting.fetchSwapRemove(request_id) == null) return error.RequestNotFound; + const entry = state.waiting.fetchSwapRemove(request_id) orelse return error.RequestNotFound; + // entry.value is the transfer + entry.value.abort(); log.info(.cdp, "Request aborted by intercept", .{ .reason = params.errorReason }); return cmd.sendResult(null, .{}); diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index f143ff52..befa824c 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -215,7 +215,7 @@ pub fn httpRequestFail(arena: Allocator, bc: anytype, data: *const Notification. // We're missing a bunch of fields, but, for now, this seems like enough try bc.cdp.sendEvent("Network.loadingFailed", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.request.id.?}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.transfer.id}), // Seems to be what chrome answers with. I assume it depends on the type of error? .type = "Ping", .errorText = data.err, @@ -251,7 +251,8 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification .query = true, }); - const full_request_url = try std.Uri.parse(data.request.url); + const transfer = data.transfer; + const full_request_url = transfer.uri; const request_url = try urlToString(arena, &full_request_url, .{ .scheme = true, .authentication = true, @@ -263,19 +264,19 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification .fragment = true, // TODO since path is false, this likely does not work as intended }); - const headers = try data.request.headers.asHashMap(arena); + const headers = try transfer.req.headers.asHashMap(arena); // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.requestWillBeSent", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{data.request.id.?}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{transfer.id}), .frameId = target_id, .loaderId = bc.loader_id, .documentUrl = document_url, .request = .{ .url = request_url, .urlFragment = request_fragment, - .method = @tagName(data.request.method), - .hasPostData = data.request.body != null, + .method = @tagName(transfer.req.method), + .hasPostData = transfer.req.body != null, .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, }, }, .{ .session_id = session_id }); diff --git a/src/http/Client.zig b/src/http/Client.zig index c72da2ee..f27533f5 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -64,10 +64,10 @@ handles: Handles, next_request_id: u64 = 0, // When handles has no more available easys, requests get queued. -queue: RequestQueue, +queue: TransferQueue, // Memory pool for Queue nodes. -queue_node_pool: std.heap.MemoryPool(RequestQueue.Node), +queue_node_pool: std.heap.MemoryPool(TransferQueue.Node), // The main app allocator allocator: Allocator, @@ -93,13 +93,13 @@ arena: ArenaAllocator, // restoring, this originally-configured value is what it goes to. http_proxy: ?[:0]const u8 = null, -const RequestQueue = std.DoublyLinkedList(Request); +const TransferQueue = std.DoublyLinkedList(*Transfer); pub fn init(allocator: Allocator, ca_blob: ?c.curl_blob, opts: Http.Opts) !*Client { var transfer_pool = std.heap.MemoryPool(Transfer).init(allocator); errdefer transfer_pool.deinit(); - var queue_node_pool = std.heap.MemoryPool(RequestQueue.Node).init(allocator); + var queue_node_pool = std.heap.MemoryPool(TransferQueue.Node).init(allocator); errdefer queue_node_pool.deinit(); const client = try allocator.create(Client); @@ -151,8 +151,7 @@ pub fn abort(self: *Client) void { log.err(.http, "get private info", .{ .err = err, .source = "abort" }); continue; }; - self.requestFailed(&transfer.req, error.Abort); - self.endTransfer(transfer); + transfer.abort(); } std.debug.assert(self.active == 0); @@ -193,46 +192,83 @@ pub fn tick(self: *Client, timeout_ms: usize) !void { } pub fn request(self: *Client, req: Request) !void { - var req_copy = req; // We need it mutable + const transfer = try self.makeTransfer(req); - if (req_copy.id == null) { // If the ID has already been set that means the request was previously intercepted - req_copy.id = self.next_request_id; - self.next_request_id += 1; - if (self.notification) |notification| { - notification.dispatch(.http_request_start, &.{ .request = &req_copy }); + if (self.notification) |notification| { + notification.dispatch(.http_request_start, &.{ .transfer = transfer }); - var wait_for_interception = false; - notification.dispatch(.http_request_intercept, &.{ .request = &req_copy, .wait_for_interception = &wait_for_interception }); - if (wait_for_interception) return; // The user is send an invitation to intercept this request. + var wait_for_interception = false; + notification.dispatch(.http_request_intercept, &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception }); + if (wait_for_interception) { + // The user is send an invitation to intercept this request. + return; } } + return self.process(transfer); +} + +// Above, request will not process if there's an interception request. In such +// cases, the interecptor is expected to call process to continue the transfer +// or transfer.abort() to abort it. +pub fn process(self: *Client, transfer: *Transfer) !void { if (self.handles.getFreeHandle()) |handle| { - return self.makeRequest(handle, req_copy); + return self.makeRequest(handle, transfer); } const node = try self.queue_node_pool.create(); - node.data = req_copy; + node.data = transfer; self.queue.append(node); } // See ScriptManager.blockingGet pub fn blockingRequest(self: *Client, req: Request) !void { - return self.makeRequest(&self.blocking, req); + const transfer = try self.makeTransfer(req); + return self.makeRequest(&self.blocking, transfer); } -fn requestFailed(self: *Client, req: *Request, err: anyerror) void { - if (req._notified_fail) return; - req._notified_fail = true; +fn makeTransfer(self: *Client, req: Request) !*Transfer { + errdefer req.headers.deinit(); + + // we need this for cookies + const uri = std.Uri.parse(req.url) catch |err| { + log.warn(.http, "invalid url", .{ .err = err, .url = req.url }); + return err; + }; + + const transfer = try self.transfer_pool.create(); + errdefer self.transfer_pool.destroy(transfer); + + const id = self.next_request_id + 1; + self.next_request_id = id; + transfer.* = .{ + .id = id, + .uri = uri, + .req = req, + .ctx = req.ctx, + .client = self, + }; + return transfer; +} + +fn requestFailed(self: *Client, transfer: *Transfer, err: anyerror) void { + // this shoudln't happen, we'll crash in debug mode. But in release, we'll + // just noop this state. + std.debug.assert(transfer._notified_fail == false); + if (transfer._notified_fail) { + return; + } + + transfer._notified_fail = true; if (self.notification) |notification| { notification.dispatch(.http_request_fail, &.{ - .request = req, + .transfer = transfer, .err = err, }); } - req.error_callback(req.ctx, err); + transfer.req.error_callback(transfer.ctx, err); } // Restrictive since it'll only work if there are no inflight requests. In some @@ -265,54 +301,40 @@ pub fn restoreOriginalProxy(self: *Client) !void { try errorCheck(c.curl_easy_setopt(self.blocking.conn.easy, c.CURLOPT_PROXY, proxy)); } -fn makeRequest(self: *Client, handle: *Handle, req: Request) !void { +fn makeRequest(self: *Client, handle: *Handle, transfer: *Transfer) !void { const conn = handle.conn; const easy = conn.easy; + const req = &transfer.req; - // we need this for cookies - const uri = std.Uri.parse(req.url) catch |err| { - self.handles.release(handle); - log.warn(.http, "invalid url", .{ .err = err, .url = req.url }); - return; - }; - - var header_list = req.headers; { - errdefer self.handles.release(handle); - try conn.setMethod(req.method); - try conn.setURL(req.url); + transfer._handle = handle; + errdefer transfer.deinit(); + try conn.setURL(req.url); + try conn.setMethod(req.method); if (req.body) |b| { try conn.setBody(b); } + var header_list = req.headers; try conn.secretHeaders(&header_list); // Add headers that must be hidden from intercepts try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); + try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer)); } - { - errdefer self.handles.release(handle); + // Once soon as this is called, our "perform" loop is responsible for + // cleaning things up. That's why the above code is in a block. If anything + // fails BEFORE `curl_multi_add_handle` suceeds, the we still need to do + // cleanup. But if things fail after `curl_multi_add_handle`, we expect + // perfom to pickup the failure and cleanup. + try errorMCheck(c.curl_multi_add_handle(self.multi, easy)); - const transfer = try self.transfer_pool.create(); - transfer.* = .{ - .id = 0, - .uri = uri, - .req = req, - .ctx = req.ctx, - .handle = handle, - ._request_header_list = header_list.headers, + if (req.start_callback) |cb| { + cb(transfer) catch |err| { + try errorMCheck(c.curl_multi_remove_handle(self.multi, easy)); + transfer.deinit(); + return err; }; - errdefer self.transfer_pool.destroy(transfer); - - try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_PRIVATE, transfer)); - - try errorMCheck(c.curl_multi_add_handle(self.multi, easy)); - if (req.start_callback) |cb| { - cb(transfer) catch |err| { - try errorMCheck(c.curl_multi_remove_handle(self.multi, easy)); - return err; - }; - } } self.active += 1; @@ -336,38 +358,36 @@ fn perform(self: *Client, timeout_ms: c_int) !void { std.debug.assert(msg.msg == c.CURLMSG_DONE); const easy = msg.easy_handle.?; - const transfer = try Transfer.fromEasy(easy); - const ctx = transfer.ctx; - const done_callback = transfer.req.done_callback; // release it ASAP so that it's available; some done_callbacks // will load more resources. self.endTransfer(transfer); + defer transfer.deinit(); + if (errorCheck(msg.data.result)) { - done_callback(ctx) catch |err| { + transfer.req.done_callback(transfer.ctx) catch |err| { // transfer isn't valid at this point, don't use it. log.err(.http, "done_callback", .{ .err = err }); - self.requestFailed(&transfer.req, err); + self.requestFailed(transfer, err); }; + // self.requestComplete(transfer); } else |err| { - self.requestFailed(&transfer.req, err); + self.requestFailed(transfer, err); } } } fn endTransfer(self: *Client, transfer: *Transfer) void { - const handle = transfer.handle; - - transfer.deinit(); - self.transfer_pool.destroy(transfer); + const handle = transfer._handle.?; errorMCheck(c.curl_multi_remove_handle(self.multi, handle.conn.easy)) catch |err| { log.fatal(.http, "Failed to remove handle", .{ .err = err }); }; self.handles.release(handle); + transfer._handle = null; self.active -= 1; } @@ -497,50 +517,49 @@ pub const RequestCookie = struct { }; pub const Request = struct { - id: ?u64 = null, method: Method, url: [:0]const u8, headers: Headers, body: ?[]const u8 = null, cookie_jar: *storage.CookieJar, - _notified_fail: bool = false, - // arbitrary data that can be associated with this request ctx: *anyopaque = undefined, - start_callback: ?*const fn (req: *Transfer) anyerror!void = null, - header_callback: ?*const fn (req: *Transfer, header: []const u8) anyerror!void = null, - header_done_callback: *const fn (req: *Transfer) anyerror!void, - data_callback: *const fn (req: *Transfer, data: []const u8) anyerror!void, + start_callback: ?*const fn (transfer: *Transfer) anyerror!void = null, + header_callback: ?*const fn (transfer: *Transfer, header: []const u8) anyerror!void = null, + header_done_callback: *const fn (transfer: *Transfer) anyerror!void, + data_callback: *const fn (transfer: *Transfer, data: []const u8) anyerror!void, done_callback: *const fn (ctx: *anyopaque) anyerror!void, error_callback: *const fn (ctx: *anyopaque, err: anyerror) void, }; pub const Transfer = struct { - id: usize, + id: usize = 0, req: Request, - ctx: *anyopaque, uri: std.Uri, // used for setting/getting the cookie + ctx: *anyopaque, // copied from req.ctx to make it easier for callback handlers + client: *Client, + _notified_fail: bool = false, // We'll store the response header here response_header: ?Header = null, - handle: *Handle, + _handle: ?*Handle = null, _redirecting: bool = false, - // needs to be freed when we're done - _request_header_list: ?*c.curl_slist = null, fn deinit(self: *Transfer) void { - if (self._request_header_list) |list| { - c.curl_slist_free_all(list); + self.req.headers.deinit(); + if (self._handle) |handle| { + self.client.handles.release(handle); } + self.client.transfer_pool.destroy(self); } pub fn format(self: *const Transfer, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { const req = self.req; - return writer.print("[{d}] {s} {s}", .{ self.id, @tagName(req.method), req.url }); + return writer.print("{s} {s}", .{ @tagName(req.method), req.url }); } pub fn setBody(self: *Transfer, body: []const u8) !void { @@ -553,8 +572,20 @@ pub const Transfer = struct { self._request_header_list = c.curl_slist_append(self._request_header_list, value); } + pub fn updateURL(self: *Transfer, url: [:0]const u8) !void { + // for cookies + self.uri = try std.Uri.parse(url); + + // for the request itself + self.req.url = url; + } + pub fn abort(self: *Transfer) void { - self.handle.client.endTransfer(self); + self.client.requestFailed(self, error.Abort); + if (self._handle != null) { + self.client.endTransfer(self); + } + self.deinit(); } fn headerCallback(buffer: [*]const u8, header_count: usize, buf_len: usize, data: *anyopaque) callconv(.c) usize { diff --git a/src/http/Http.zig b/src/http/Http.zig index aa5c59a2..b6a15b9e 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -219,7 +219,7 @@ pub const Headers = struct { return .{ .headers = header_list }; } - pub fn deinit(self: *Headers) void { + pub fn deinit(self: *const Headers) void { c.curl_slist_free_all(self.headers); } diff --git a/src/notification.zig b/src/notification.zig index 6f2e29c9..450d8fb0 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -4,7 +4,7 @@ const log = @import("log.zig"); const URL = @import("url.zig").URL; const page = @import("browser/page.zig"); const Http = @import("http/Http.zig"); -const Request = @import("http/Client.zig").Request; +const Transfer = @import("http/Client.zig").Transfer; const Allocator = std.mem.Allocator; @@ -94,16 +94,16 @@ pub const Notification = struct { }; pub const RequestStart = struct { - request: *Request, + transfer: *Transfer, }; pub const RequestIntercept = struct { - request: *Request, + transfer: *Transfer, wait_for_interception: *bool, }; pub const RequestFail = struct { - request: *Request, + transfer: *Transfer, err: anyerror, }; From 5a3d5f5512076d6858a276901b2c3112fbab61eb Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 13 Aug 2025 18:17:59 +0800 Subject: [PATCH 35/44] improve elapsed display for larger numbers --- src/log.zig | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/log.zig b/src/log.zig index 9ac06d46..f8e9b0fb 100644 --- a/src/log.zig +++ b/src/log.zig @@ -215,7 +215,8 @@ fn logPrettyPrefix(comptime scope: Scope, level: Level, comptime msg: []const u8 if (@mod(padding, 2) == 1) { try writer.writeByte(' '); } - try writer.print(" \x1b[0m[+{d}ms]", .{elapsed()}); + const el = elapsed(); + try writer.print(" \x1b[0m[+{d}{s}]", .{ el.time, el.unit }); try writer.writeByte('\n'); } } @@ -321,7 +322,7 @@ fn timestamp() i64 { } var first_log: i64 = 0; -fn elapsed() i64 { +fn elapsed() struct { time: f64, unit: []const u8 } { const now = timestamp(); last_log_lock.lock(); @@ -331,7 +332,11 @@ fn elapsed() i64 { first_log = now; } - return now - first_log; + const e = now - first_log; + if (e < 10_000) { + return .{ .time = @floatFromInt(e), .unit = "ms" }; + } + return .{ .time = @as(f64, @floatFromInt(e)) / @as(f64, 1000), .unit = "s" }; } const testing = @import("testing.zig"); From 9bd8b2fc430ed3442c6cd95ec3a25ad7fafb548a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 13 Aug 2025 19:39:49 +0800 Subject: [PATCH 36/44] fix wpt runner --- src/main_wpt.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 2e78ae5e..7089a0d5 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -170,7 +170,7 @@ fn run( var try_catch: Env.TryCatch = undefined; try_catch.init(runner.page.main_context); defer try_catch.deinit(); - try runner.page.loop.run(std.time.ns_per_ms * 200); + runner.page.wait(std.time.ns_per_ms * 200); if (try_catch.hasCaught()) { err_out.* = (try try_catch.err(arena)) orelse "unknwon error"; From 3c8065fdee53912557a8fa1fa0a2bfa6700fef9f Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 13 Aug 2025 20:12:39 +0800 Subject: [PATCH 37/44] fix fmt --- src/browser/dom/shadow_root.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/browser/dom/shadow_root.zig b/src/browser/dom/shadow_root.zig index 7d8977df..aaa4a0a2 100644 --- a/src/browser/dom/shadow_root.zig +++ b/src/browser/dom/shadow_root.zig @@ -62,7 +62,7 @@ const testing = @import("../../testing.zig"); test "Browser.DOM.ShadowRoot" { defer testing.reset(); - var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = + var runner = try testing.jsRunner(testing.tracking_allocator, .{ .html = \\
nope
}); defer runner.deinit(); @@ -94,8 +94,8 @@ test "Browser.DOM.ShadowRoot" { try runner.testCases(&.{ .{ "sr2.getElementById('conflict')", "null" }, .{ "const n1 = document.createElement('div')", null }, - .{ "n1.id = 'conflict'", null}, - .{ "sr2.append(n1)", null}, + .{ "n1.id = 'conflict'", null }, + .{ "sr2.append(n1)", null }, .{ "sr2.getElementById('conflict') == n1", "true" }, }, .{}); From f6c68e45805fd2a252d9ff48b3658a9e4fe53a2a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Wed, 13 Aug 2025 20:16:14 +0800 Subject: [PATCH 38/44] fix release build (constness via telemetry, not seen in debug) --- src/http/Http.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/http/Http.zig b/src/http/Http.zig index b6a15b9e..b83f3477 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -195,7 +195,7 @@ pub const Connection = struct { pub fn request(self: *const Connection) !u16 { const easy = self.easy; - const header_list = try Headers.init(); + var header_list = try Headers.init(); defer header_list.deinit(); try self.secretHeaders(&header_list); try errorCheck(c.curl_easy_setopt(easy, c.CURLOPT_HTTPHEADER, header_list.headers)); From c0106a238bbffea51c4b78e8cb922ed9a5675028 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Wed, 13 Aug 2025 14:29:23 +0200 Subject: [PATCH 39/44] http_headers_done_receiving --- src/cdp/cdp.zig | 23 +++++++++++--- src/cdp/domains/network.zig | 62 ++++++++++++++++++++++++++++++------- src/http/Client.zig | 17 ++++++++++ src/http/Http.zig | 4 +-- src/notification.zig | 22 ++++++++----- 5 files changed, 101 insertions(+), 27 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 696448c2..1c0690f7 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -29,6 +29,7 @@ const Page = @import("../browser/page.zig").Page; const Inspector = @import("../browser/env.zig").Env.Inspector; const Incrementing = @import("../id.zig").Incrementing; const Notification = @import("../notification.zig").Notification; +const NetworkState = @import("domains/network.zig").NetworkState; const InterceptState = @import("domains/fetch.zig").InterceptState; const polyfill = @import("../browser/polyfill/polyfill.zig"); @@ -76,6 +77,7 @@ pub fn CDPT(comptime TypeProvider: type) type { // Extra headers to add to all requests. TBD under which conditions this should be reset. extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty, + network_state: NetworkState, intercept_state: InterceptState, const Self = @This(); @@ -92,6 +94,7 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), + .network_state = try NetworkState.init(allocator), .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -101,6 +104,7 @@ pub fn CDPT(comptime TypeProvider: type) type { bc.deinit(); } self.intercept_state.deinit(); // TBD Should this live in BC? + self.network_state.deinit(); self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -447,13 +451,15 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn networkEnable(self: *Self) !void { try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail); try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart); - try self.cdp.browser.notification.register(.http_request_complete, self, onHttpRequestComplete); + try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived); + try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving); } pub fn networkDisable(self: *Self) void { self.cdp.browser.notification.unregister(.http_request_fail, self); self.cdp.browser.notification.unregister(.http_request_start, self); - self.cdp.browser.notification.unregister(.http_request_complete, self); + self.cdp.browser.notification.unregister(.http_header_received, self); + self.cdp.browser.notification.unregister(.http_headers_done_receiving, self); } pub fn fetchEnable(self: *Self) !void { @@ -466,7 +472,8 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void { const self: *Self = @alignCast(@ptrCast(ctx)); - return @import("domains/page.zig").pageRemove(self); + try @import("domains/page.zig").pageRemove(self); + try @import("domains/network.zig").pageRemove(self); } pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void { @@ -497,16 +504,22 @@ pub fn BrowserContext(comptime CDP_T: type) type { try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); } + pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void { + const self: *Self = @alignCast(@ptrCast(ctx)); + defer self.resetNotificationArena(); + try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header); + } + pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data); } - pub fn onHttpRequestComplete(ctx: *anyopaque, data: *const Notification.RequestComplete) !void { + pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - return @import("domains/network.zig").httpRequestComplete(self.notification_arena, self, data); + return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data); } fn resetNotificationArena(self: *Self) void { diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index befa824c..fe5ce21e 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -51,6 +51,49 @@ pub fn processMessage(cmd: anytype) !void { } } +const Response = struct { + status: u16, + headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty, // These may not be complete yet, but we only tell the client Network.responseReceived when all the headers are in + // Later should store body as well to support getResponseBody which should only work once Network.loadingFinished is sent + // but the body itself would be loaded with each chunks as Network.dataReceiveds are coming in. +}; + +// Stored in CDP +pub const NetworkState = struct { + const Self = @This(); + received: std.AutoArrayHashMap(u64, Response), + arena: std.heap.ArenaAllocator, + + pub fn init(allocator: Allocator) !NetworkState { + return .{ + .received = std.AutoArrayHashMap(u64, Response).init(allocator), + .arena = std.heap.ArenaAllocator.init(allocator), + }; + } + + pub fn deinit(self: *Self) void { + self.received.deinit(); + self.arena.deinit(); + } + + pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void { + const kv = try self.received.getOrPut(request_id); + if (!kv.found_existing) kv.value_ptr.* = .{ .status = status }; + + const a = self.arena.allocator(); + const name = try a.dupe(u8, header.name); + const value = try a.dupe(u8, header.value); + try kv.value_ptr.headers.put(a, name, value); + } +}; + +pub fn pageRemove(bc: anytype) !void { + // The main page is going to be removed + const state = &bc.cdp.network_state; + state.received.clearRetainingCapacity(); // May need to be in pageRemoved + _ = state.arena.reset(.{ .retain_with_limit = 1024 }); +} + fn enable(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; try bc.networkEnable(); @@ -282,7 +325,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification }, .{ .session_id = session_id }); } -pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notification.RequestComplete) !void { +pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -293,7 +336,7 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi const session_id = bc.session_id orelse unreachable; const target_id = bc.target_id orelse unreachable; - const url = try urlToString(arena, request.url, .{ + const url = try urlToString(arena, &request.transfer.uri, .{ .scheme = true, .authentication = true, .authority = true, @@ -301,22 +344,17 @@ pub fn httpRequestComplete(arena: Allocator, bc: anytype, request: *const Notifi .query = true, }); - // @newhttp - const headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - // try headers.ensureTotalCapacity(arena, request.headers.len); - // for (request.headers) |header| { - // headers.putAssumeCapacity(header.name, header.value); - // } + const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound; // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ - .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.id}), + .requestId = try std.fmt.allocPrint(arena, "REQ-{d}", .{request.transfer.id}), .loaderId = bc.loader_id, .response = .{ .url = url, - .status = request.status, - .statusText = @as(std.http.Status, @enumFromInt(request.status)).phrase() orelse "Unknown", - .headers = std.json.ArrayHashMap([]const u8){ .map = headers }, + .status = response.status, + .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown", + .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers }, }, .frameId = target_id, }, .{ .session_id = session_id }); diff --git a/src/http/Client.zig b/src/http/Client.zig index f27533f5..58e99112 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -247,6 +247,7 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer { .req = req, .ctx = req.ctx, .client = self, + .notification = &self.notification, }; return transfer; } @@ -549,6 +550,8 @@ pub const Transfer = struct { _redirecting: bool = false, + notification: *?*Notification, // Points to the Client's notification. TBD if a Browser can remove the notification before all Transfers are gone. + fn deinit(self: *Transfer) void { self.req.headers.deinit(); if (self._handle) |handle| { @@ -673,12 +676,26 @@ pub const Transfer = struct { // returning < buf_len terminates the request return 0; }; + if (transfer.notification.*) |notification| { // TBD before or after callback? + notification.dispatch(.http_headers_done_receiving, &.{ + .transfer = transfer, + }); + } } else { if (transfer.req.header_callback) |cb| { cb(transfer, header) catch |err| { log.err(.http, "header_callback", .{ .err = err, .req = transfer }); return 0; }; + if (transfer.notification.*) |notification| { // TBD before or after callback? + if (Http.Headers.parseHeader(header)) |hdr_name_value| { + notification.dispatch(.http_header_received, &.{ + .request_id = transfer.id, + .status = hdr.status, + .header = hdr_name_value, + }); + } else log.err(.http, "invalid header", .{ .line = header }); + } } } return buf_len; diff --git a/src/http/Http.zig b/src/http/Http.zig index b83f3477..506a0f13 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -244,8 +244,8 @@ pub const Headers = struct { return list; } - fn parseHeader(header_str: []const u8) ?struct { name: []const u8, value: []const u8 } { - const colon_pos = std.mem.indexOf(u8, header_str, ":") orelse return null; + pub fn parseHeader(header_str: []const u8) ?std.http.Header { + const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; const name = std.mem.trim(u8, header_str[0..colon_pos], " \t"); const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t"); diff --git a/src/notification.zig b/src/notification.zig index 450d8fb0..0e8860de 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -63,7 +63,8 @@ pub const Notification = struct { http_request_fail: List = .{}, http_request_start: List = .{}, http_request_intercept: List = .{}, - http_request_complete: List = .{}, + http_header_received: List = .{}, + http_headers_done_receiving: List = .{}, notification_created: List = .{}, }; @@ -75,7 +76,8 @@ pub const Notification = struct { http_request_fail: *const RequestFail, http_request_start: *const RequestStart, http_request_intercept: *const RequestIntercept, - http_request_complete: *const RequestComplete, + http_header_received: *const ResponseHeader, + http_headers_done_receiving: *const ResponseHeadersDone, notification_created: *Notification, }; const EventType = std.meta.FieldEnum(Events); @@ -102,17 +104,21 @@ pub const Notification = struct { wait_for_interception: *bool, }; + pub const ResponseHeader = struct { + request_id: u64, + status: u16, + header: std.http.Header, + }; + + pub const ResponseHeadersDone = struct { + transfer: *Transfer, + }; + pub const RequestFail = struct { transfer: *Transfer, err: anyerror, }; - pub const RequestComplete = struct { - id: usize, - url: *const std.Uri, - status: u16, - }; - pub fn init(allocator: Allocator, parent: ?*Notification) !*Notification { // This is put on the heap because we want to raise a .notification_created // event, so that, something like Telemetry, can receive the From 7d05712f405394cda7052b8dad291490181a5d74 Mon Sep 17 00:00:00 2001 From: sjorsdonkers <72333389+sjorsdonkers@users.noreply.github.com> Date: Wed, 13 Aug 2025 14:54:59 +0200 Subject: [PATCH 40/44] setExtraHTTPHeaders --- src/cdp/cdp.zig | 2 +- src/cdp/domains/network.zig | 25 +++++-------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index 1c0690f7..a01ddce7 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -75,7 +75,7 @@ pub fn CDPT(comptime TypeProvider: type) type { notification_arena: std.heap.ArenaAllocator, // Extra headers to add to all requests. TBD under which conditions this should be reset. - extra_headers: std.ArrayListUnmanaged(std.http.Header) = .empty, + extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty, network_state: NetworkState, intercept_state: InterceptState, diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index fe5ce21e..548c323a 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -121,7 +121,8 @@ fn setExtraHTTPHeaders(cmd: anytype) !void { try extra_headers.ensureTotalCapacity(arena, params.headers.map.count()); var it = params.headers.map.iterator(); while (it.next()) |header| { - extra_headers.appendAssumeCapacity(.{ .name = try arena.dupe(u8, header.key_ptr.*), .value = try arena.dupe(u8, header.value_ptr.*) }); + const header_string = try std.fmt.allocPrintZ(arena, "{s}: {s}", .{ header.key_ptr.*, header.value_ptr.* }); + extra_headers.appendAssumeCapacity(header_string); } return cmd.sendResult(null, .{}); @@ -233,19 +234,6 @@ fn getCookies(cmd: anytype) !void { try cmd.sendResult(.{ .cookies = writer }, .{}); } -// Upsert a header into the headers array. -// returns true if the header was added, false if it was updated -fn putAssumeCapacity(headers: *std.ArrayListUnmanaged(std.http.Header), extra: std.http.Header) bool { - for (headers.items) |*header| { - if (std.mem.eql(u8, header.name, extra.name)) { - header.value = extra.value; - return false; - } - } - headers.appendAssumeCapacity(extra); - return true; -} - pub fn httpRequestFail(arena: Allocator, bc: anytype, data: *const Notification.RequestFail) !void { // It's possible that the request failed because we aborted when the client // sent Target.closeTarget. In that case, bc.session_id will be cleared @@ -279,12 +267,9 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification const page = bc.session.currentPage() orelse unreachable; // Modify request with extra CDP headers - // @newhttp - // try request.headers.ensureTotalCapacity(request.arena, request.headers.items.len + cdp.extra_headers.items.len); - // for (cdp.extra_headers.items) |extra| { - // const new = putAssumeCapacity(request.headers, extra); - // if (!new) log.debug(.cdp, "request header overwritten", .{ .name = extra.name }); - // } + for (cdp.extra_headers.items) |extra| { + try data.transfer.req.headers.add(extra); + } const document_url = try urlToString(arena, &page.url.uri, .{ .scheme = true, From 8d2d4ffdd2a3ffda14817addffd54da13b486720 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Wed, 13 Aug 2025 19:44:06 +0200 Subject: [PATCH 41/44] fix integer overflow for sleeping delay --- src/browser/page.zig | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/browser/page.zig b/src/browser/page.zig index cfe261f9..a02a447b 100644 --- a/src/browser/page.zig +++ b/src/browser/page.zig @@ -313,7 +313,13 @@ pub const Page = struct { return; } _ = try scheduler.runLowPriority(); - std.time.sleep(std.time.ns_per_ms * ms); + + // We must use a u64 here b/c ms is a u32 and the + // conversion to ns can generate an integer + // overflow. + const _ms: u64 = @intCast(ms); + + std.time.sleep(std.time.ns_per_ms * _ms); break :SW; } From 5100e06f381de3f5e56e272f9d4ddbb8ad5275f6 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 14 Aug 2025 14:51:02 +0800 Subject: [PATCH 42/44] fix header done callback --- src/cdp/domains/network.zig | 20 ++++++++++++-------- src/http/Client.zig | 27 +++++++++++++-------------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 548c323a..aa88f6f8 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -53,32 +53,36 @@ pub fn processMessage(cmd: anytype) !void { const Response = struct { status: u16, - headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty, // These may not be complete yet, but we only tell the client Network.responseReceived when all the headers are in - // Later should store body as well to support getResponseBody which should only work once Network.loadingFinished is sent - // but the body itself would be loaded with each chunks as Network.dataReceiveds are coming in. + headers: std.StringArrayHashMapUnmanaged([]const u8) = .empty, + // These may not be complete yet, but we only tell the client + // Network.responseReceived when all the headers are in. + // Later should store body as well to support getResponseBody which should + // only work once Network.loadingFinished is sent but the body itself would + // be loaded with each chunks as Network.dataReceiveds are coming in. }; // Stored in CDP pub const NetworkState = struct { - const Self = @This(); - received: std.AutoArrayHashMap(u64, Response), arena: std.heap.ArenaAllocator, + received: std.AutoArrayHashMap(u64, Response), pub fn init(allocator: Allocator) !NetworkState { return .{ - .received = std.AutoArrayHashMap(u64, Response).init(allocator), .arena = std.heap.ArenaAllocator.init(allocator), + .received = std.AutoArrayHashMap(u64, Response).init(allocator), }; } - pub fn deinit(self: *Self) void { + pub fn deinit(self: *NetworkState) void { self.received.deinit(); self.arena.deinit(); } pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void { const kv = try self.received.getOrPut(request_id); - if (!kv.found_existing) kv.value_ptr.* = .{ .status = status }; + if (!kv.found_existing) { + kv.value_ptr.* = .{ .status = status }; + } const a = self.arena.allocator(); const name = try a.dupe(u8, header.name); diff --git a/src/http/Client.zig b/src/http/Client.zig index 58e99112..e66ba9ec 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -247,13 +247,12 @@ fn makeTransfer(self: *Client, req: Request) !*Transfer { .req = req, .ctx = req.ctx, .client = self, - .notification = &self.notification, }; return transfer; } fn requestFailed(self: *Client, transfer: *Transfer, err: anyerror) void { - // this shoudln't happen, we'll crash in debug mode. But in release, we'll + // this shouldn't happen, we'll crash in debug mode. But in release, we'll // just noop this state. std.debug.assert(transfer._notified_fail == false); if (transfer._notified_fail) { @@ -550,8 +549,6 @@ pub const Transfer = struct { _redirecting: bool = false, - notification: *?*Notification, // Points to the Client's notification. TBD if a Browser can remove the notification before all Transfers are gone. - fn deinit(self: *Transfer) void { self.req.headers.deinit(); if (self._handle) |handle| { @@ -676,7 +673,8 @@ pub const Transfer = struct { // returning < buf_len terminates the request return 0; }; - if (transfer.notification.*) |notification| { // TBD before or after callback? + + if (transfer.client.notification) |notification| { notification.dispatch(.http_headers_done_receiving, &.{ .transfer = transfer, }); @@ -687,15 +685,16 @@ pub const Transfer = struct { log.err(.http, "header_callback", .{ .err = err, .req = transfer }); return 0; }; - if (transfer.notification.*) |notification| { // TBD before or after callback? - if (Http.Headers.parseHeader(header)) |hdr_name_value| { - notification.dispatch(.http_header_received, &.{ - .request_id = transfer.id, - .status = hdr.status, - .header = hdr_name_value, - }); - } else log.err(.http, "invalid header", .{ .line = header }); - } + } + + if (transfer.client.notification) |notification| { + if (Http.Headers.parseHeader(header)) |hdr_name_value| { + notification.dispatch(.http_header_received, &.{ + .request_id = transfer.id, + .status = hdr.status, + .header = hdr_name_value, + }); + } else log.err(.http, "invalid header", .{ .line = header }); } } return buf_len; From 96b10f4b8517cb4675dce5bc666e996942ad2d3a Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Thu, 14 Aug 2025 15:50:56 +0800 Subject: [PATCH 43/44] Optimize Network.responseReceived Add a header iterator to the transfer. This removes the need for NetworkState, duping header name/values, and the http_header_received event. --- src/cdp/cdp.zig | 20 +++-------- src/cdp/domains/network.zig | 68 ++++++++++++++----------------------- src/http/Client.zig | 58 +++++++++++++++++++++++++------ src/http/Http.zig | 3 +- src/notification.zig | 12 ++----- 5 files changed, 81 insertions(+), 80 deletions(-) diff --git a/src/cdp/cdp.zig b/src/cdp/cdp.zig index a01ddce7..2435f7fd 100644 --- a/src/cdp/cdp.zig +++ b/src/cdp/cdp.zig @@ -77,7 +77,6 @@ pub fn CDPT(comptime TypeProvider: type) type { // Extra headers to add to all requests. TBD under which conditions this should be reset. extra_headers: std.ArrayListUnmanaged([*c]const u8) = .empty, - network_state: NetworkState, intercept_state: InterceptState, const Self = @This(); @@ -94,7 +93,6 @@ pub fn CDPT(comptime TypeProvider: type) type { .browser_context = null, .message_arena = std.heap.ArenaAllocator.init(allocator), .notification_arena = std.heap.ArenaAllocator.init(allocator), - .network_state = try NetworkState.init(allocator), .intercept_state = try InterceptState.init(allocator), // TBD or browser session arena? }; } @@ -104,7 +102,6 @@ pub fn CDPT(comptime TypeProvider: type) type { bc.deinit(); } self.intercept_state.deinit(); // TBD Should this live in BC? - self.network_state.deinit(); self.browser.deinit(); self.message_arena.deinit(); self.notification_arena.deinit(); @@ -451,15 +448,13 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn networkEnable(self: *Self) !void { try self.cdp.browser.notification.register(.http_request_fail, self, onHttpRequestFail); try self.cdp.browser.notification.register(.http_request_start, self, onHttpRequestStart); - try self.cdp.browser.notification.register(.http_header_received, self, onHttpHeaderReceived); - try self.cdp.browser.notification.register(.http_headers_done_receiving, self, onHttpHeadersDoneReceiving); + try self.cdp.browser.notification.register(.http_headers_done, self, onHttpHeadersDone); } pub fn networkDisable(self: *Self) void { self.cdp.browser.notification.unregister(.http_request_fail, self); self.cdp.browser.notification.unregister(.http_request_start, self); - self.cdp.browser.notification.unregister(.http_header_received, self); - self.cdp.browser.notification.unregister(.http_headers_done_receiving, self); + self.cdp.browser.notification.unregister(.http_headers_done, self); } pub fn fetchEnable(self: *Self) !void { @@ -473,7 +468,6 @@ pub fn BrowserContext(comptime CDP_T: type) type { pub fn onPageRemove(ctx: *anyopaque, _: Notification.PageRemove) !void { const self: *Self = @alignCast(@ptrCast(ctx)); try @import("domains/page.zig").pageRemove(self); - try @import("domains/network.zig").pageRemove(self); } pub fn onPageCreated(ctx: *anyopaque, page: *Page) !void { @@ -504,22 +498,16 @@ pub fn BrowserContext(comptime CDP_T: type) type { try @import("domains/fetch.zig").requestPaused(self.notification_arena, self, data); } - pub fn onHttpHeaderReceived(ctx: *anyopaque, data: *const Notification.ResponseHeader) !void { - const self: *Self = @alignCast(@ptrCast(ctx)); - defer self.resetNotificationArena(); - try self.cdp.network_state.putOrAppendReceivedHeader(data.request_id, data.status, data.header); - } - pub fn onHttpRequestFail(ctx: *anyopaque, data: *const Notification.RequestFail) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); return @import("domains/network.zig").httpRequestFail(self.notification_arena, self, data); } - pub fn onHttpHeadersDoneReceiving(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { + pub fn onHttpHeadersDone(ctx: *anyopaque, data: *const Notification.ResponseHeadersDone) !void { const self: *Self = @alignCast(@ptrCast(ctx)); defer self.resetNotificationArena(); - return @import("domains/network.zig").httpHeadersDoneReceiving(self.notification_arena, self, data); + return @import("domains/network.zig").httpHeadersDone(self.notification_arena, self, data); } fn resetNotificationArena(self: *Self) void { diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index aa88f6f8..d3f30be2 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -22,6 +22,7 @@ const Allocator = std.mem.Allocator; const Notification = @import("../../notification.zig").Notification; const log = @import("../../log.zig"); const CdpStorage = @import("storage.zig"); +const Transfer = @import("../../http/Client.zig").Transfer; pub fn processMessage(cmd: anytype) !void { const action = std.meta.stringToEnum(enum { @@ -61,43 +62,6 @@ const Response = struct { // be loaded with each chunks as Network.dataReceiveds are coming in. }; -// Stored in CDP -pub const NetworkState = struct { - arena: std.heap.ArenaAllocator, - received: std.AutoArrayHashMap(u64, Response), - - pub fn init(allocator: Allocator) !NetworkState { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .received = std.AutoArrayHashMap(u64, Response).init(allocator), - }; - } - - pub fn deinit(self: *NetworkState) void { - self.received.deinit(); - self.arena.deinit(); - } - - pub fn putOrAppendReceivedHeader(self: *NetworkState, request_id: u64, status: u16, header: std.http.Header) !void { - const kv = try self.received.getOrPut(request_id); - if (!kv.found_existing) { - kv.value_ptr.* = .{ .status = status }; - } - - const a = self.arena.allocator(); - const name = try a.dupe(u8, header.name); - const value = try a.dupe(u8, header.value); - try kv.value_ptr.headers.put(a, name, value); - } -}; - -pub fn pageRemove(bc: anytype) !void { - // The main page is going to be removed - const state = &bc.cdp.network_state; - state.received.clearRetainingCapacity(); // May need to be in pageRemoved - _ = state.arena.reset(.{ .retain_with_limit = 1024 }); -} - fn enable(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; try bc.networkEnable(); @@ -314,7 +278,7 @@ pub fn httpRequestStart(arena: Allocator, bc: anytype, data: *const Notification }, .{ .session_id = session_id }); } -pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { +pub fn httpHeadersDone(arena: Allocator, bc: anytype, request: *const Notification.ResponseHeadersDone) !void { // Isn't possible to do a network request within a Browser (which our // notification is tied to), without a page. std.debug.assert(bc.session.page != null); @@ -333,7 +297,7 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N .query = true, }); - const response = bc.cdp.network_state.received.get(request.transfer.id) orelse return error.ResponseNotFound; + const status = request.transfer.response_header.?.status; // We're missing a bunch of fields, but, for now, this seems like enough try cdp.sendEvent("Network.responseReceived", .{ @@ -341,9 +305,9 @@ pub fn httpHeadersDoneReceiving(arena: Allocator, bc: anytype, request: *const N .loaderId = bc.loader_id, .response = .{ .url = url, - .status = response.status, - .statusText = @as(std.http.Status, @enumFromInt(response.status)).phrase() orelse "Unknown", - .headers = std.json.ArrayHashMap([]const u8){ .map = response.headers }, + .status = status, + .statusText = @as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown", + .headers = ResponseHeaderWriter.init(request.transfer), }, .frameId = target_id, }, .{ .session_id = session_id }); @@ -355,6 +319,26 @@ pub fn urlToString(arena: Allocator, url: *const std.Uri, opts: std.Uri.WriteToS return buf.items; } +const ResponseHeaderWriter = struct { + transfer: *Transfer, + + fn init(transfer: *Transfer) ResponseHeaderWriter { + return .{ + .transfer = transfer, + }; + } + + pub fn jsonStringify(self: *const ResponseHeaderWriter, writer: anytype) !void { + try writer.beginObject(); + var it = self.transfer.responseHeaderIterator(); + while (it.next()) |hdr| { + try writer.objectField(hdr.name); + try writer.write(hdr.value); + } + try writer.endObject(); + } +}; + const testing = @import("../testing.zig"); test "cdp.network setExtraHTTPHeaders" { var ctx = testing.context(); diff --git a/src/http/Client.zig b/src/http/Client.zig index e66ba9ec..1e426f6f 100644 --- a/src/http/Client.zig +++ b/src/http/Client.zig @@ -668,6 +668,12 @@ pub const Transfer = struct { } if (buf_len == 2) { + if (getResponseHeader(easy, "content-type")) |value| { + const len = @min(value.len, hdr._content_type.len); + hdr._content_type_len = len; + @memcpy(hdr._content_type[0..len], value[0..len]); + } + transfer.req.header_done_callback(transfer) catch |err| { log.err(.http, "header_done_callback", .{ .err = err, .req = transfer }); // returning < buf_len terminates the request @@ -675,7 +681,7 @@ pub const Transfer = struct { }; if (transfer.client.notification) |notification| { - notification.dispatch(.http_headers_done_receiving, &.{ + notification.dispatch(.http_headers_done, &.{ .transfer = transfer, }); } @@ -686,16 +692,6 @@ pub const Transfer = struct { return 0; }; } - - if (transfer.client.notification) |notification| { - if (Http.Headers.parseHeader(header)) |hdr_name_value| { - notification.dispatch(.http_header_received, &.{ - .request_id = transfer.id, - .status = hdr.status, - .header = hdr_name_value, - }); - } else log.err(.http, "invalid header", .{ .line = header }); - } } return buf_len; } @@ -721,6 +717,12 @@ pub const Transfer = struct { return chunk_len; } + // we assume that the caller is smart and only calling this after being + // told that the header was ready. + pub fn responseHeaderIterator(self: *Transfer) HeaderIterator { + return .{ .easy = self._handle.?.conn.easy }; + } + // pub because Page.printWaitAnalysis uses it pub fn fromEasy(easy: *c.CURL) !*Transfer { var private: *anyopaque = undefined; @@ -742,3 +744,37 @@ pub const Header = struct { return self._content_type[0..self._content_type_len]; } }; + +const HeaderIterator = struct { + easy: *c.CURL, + prev: ?*c.curl_header = null, + + pub fn next(self: *HeaderIterator) ?struct { name: []const u8, value: []const u8 } { + const h = c.curl_easy_nextheader(self.easy, c.CURLH_HEADER, -1, self.prev) orelse return null; + self.prev = h; + + const header = h.*; + return .{ + .name = std.mem.span(header.name), + .value = std.mem.span(header.value), + }; + } +}; + +fn getResponseHeader(easy: *c.CURL, name: [:0]const u8) ?[]const u8 { + var hdr: [*c]c.curl_header = null; + const result = c.curl_easy_header(easy, name, 0, c.CURLH_HEADER, -1, &hdr); + if (result == c.CURLE_OK) { + return std.mem.span(hdr.*.value); + } + + if (result == c.CURLE_FAILED_INIT) { + // seems to be what it returns if the header isn't found + return null; + } + log.err(.http, "get response header", .{ + .name = name, + .err = @import("errors.zig").fromCode(result), + }); + return null; +} diff --git a/src/http/Http.zig b/src/http/Http.zig index 506a0f13..3195a24c 100644 --- a/src/http/Http.zig +++ b/src/http/Http.zig @@ -21,8 +21,9 @@ const std = @import("std"); pub const c = @cImport({ @cInclude("curl/curl.h"); }); -const errors = @import("errors.zig"); + const Client = @import("Client.zig"); +const errors = @import("errors.zig"); const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; diff --git a/src/notification.zig b/src/notification.zig index 0e8860de..c499c584 100644 --- a/src/notification.zig +++ b/src/notification.zig @@ -63,8 +63,7 @@ pub const Notification = struct { http_request_fail: List = .{}, http_request_start: List = .{}, http_request_intercept: List = .{}, - http_header_received: List = .{}, - http_headers_done_receiving: List = .{}, + http_headers_done: List = .{}, notification_created: List = .{}, }; @@ -76,8 +75,7 @@ pub const Notification = struct { http_request_fail: *const RequestFail, http_request_start: *const RequestStart, http_request_intercept: *const RequestIntercept, - http_header_received: *const ResponseHeader, - http_headers_done_receiving: *const ResponseHeadersDone, + http_headers_done: *const ResponseHeadersDone, notification_created: *Notification, }; const EventType = std.meta.FieldEnum(Events); @@ -104,12 +102,6 @@ pub const Notification = struct { wait_for_interception: *bool, }; - pub const ResponseHeader = struct { - request_id: u64, - status: u16, - header: std.http.Header, - }; - pub const ResponseHeadersDone = struct { transfer: *Transfer, }; From 1e095fede5f3066cfd011283fa4faae7d63eca01 Mon Sep 17 00:00:00 2001 From: Pierre Tachoire Date: Thu, 14 Aug 2025 11:27:31 +0200 Subject: [PATCH 44/44] zig fmt build.zig --- build.zig | 336 ++++++++++++++++++++++++++---------------------------- 1 file changed, 162 insertions(+), 174 deletions(-) diff --git a/build.zig b/build.zig index 4921c6f3..dc2e8bc4 100644 --- a/build.zig +++ b/build.zig @@ -163,7 +163,6 @@ fn common(b: *Build, opts: *Build.Step.Options, step: *Build.Step.Compile) !void v8_mod.addOptions("default_exports", v8_opts); mod.addImport("v8", v8_mod); - const release_dir = if (mod.optimize.? == .Debug) "debug" else "release"; const os = switch (target.result.os.tag) { .linux => "linux", @@ -452,30 +451,27 @@ fn buildZlib(b: *Build, m: *Build.Module) !void { const root = "vendor/zlib/"; zlib.installHeader(b.path(root ++ "zlib.h"), "zlib.h"); zlib.installHeader(b.path(root ++ "zconf.h"), "zconf.h"); - zlib.addCSourceFiles(.{ - .flags = &.{ - "-DHAVE_SYS_TYPES_H", - "-DHAVE_STDINT_H", - "-DHAVE_STDDEF_H", - }, - .files = &.{ - root ++ "adler32.c", - root ++ "compress.c", - root ++ "crc32.c", - root ++ "deflate.c", - root ++ "gzclose.c", - root ++ "gzlib.c", - root ++ "gzread.c", - root ++ "gzwrite.c", - root ++ "inflate.c", - root ++ "infback.c", - root ++ "inftrees.c", - root ++ "inffast.c", - root ++ "trees.c", - root ++ "uncompr.c", - root ++ "zutil.c", - } - }); + zlib.addCSourceFiles(.{ .flags = &.{ + "-DHAVE_SYS_TYPES_H", + "-DHAVE_STDINT_H", + "-DHAVE_STDDEF_H", + }, .files = &.{ + root ++ "adler32.c", + root ++ "compress.c", + root ++ "crc32.c", + root ++ "deflate.c", + root ++ "gzclose.c", + root ++ "gzlib.c", + root ++ "gzread.c", + root ++ "gzwrite.c", + root ++ "inflate.c", + root ++ "infback.c", + root ++ "inftrees.c", + root ++ "inffast.c", + root ++ "trees.c", + root ++ "uncompr.c", + root ++ "zutil.c", + } }); } fn buildMbedtls(b: *Build, m: *Build.Module) !void { @@ -488,118 +484,114 @@ fn buildMbedtls(b: *Build, m: *Build.Module) !void { mbedtls.addIncludePath(b.path(root ++ "include")); mbedtls.addIncludePath(b.path(root ++ "library")); - mbedtls.addCSourceFiles(.{ - .flags = &.{ - }, - .files = &.{ - root ++ "library/aes.c", - root ++ "library/aesni.c", - root ++ "library/aesce.c", - root ++ "library/aria.c", - root ++ "library/asn1parse.c", - root ++ "library/asn1write.c", - root ++ "library/base64.c", - root ++ "library/bignum.c", - root ++ "library/bignum_core.c", - root ++ "library/bignum_mod.c", - root ++ "library/bignum_mod_raw.c", - root ++ "library/camellia.c", - root ++ "library/ccm.c", - root ++ "library/chacha20.c", - root ++ "library/chachapoly.c", - root ++ "library/cipher.c", - root ++ "library/cipher_wrap.c", - root ++ "library/constant_time.c", - root ++ "library/cmac.c", - root ++ "library/ctr_drbg.c", - root ++ "library/des.c", - root ++ "library/dhm.c", - root ++ "library/ecdh.c", - root ++ "library/ecdsa.c", - root ++ "library/ecjpake.c", - root ++ "library/ecp.c", - root ++ "library/ecp_curves.c", - root ++ "library/entropy.c", - root ++ "library/entropy_poll.c", - root ++ "library/error.c", - root ++ "library/gcm.c", - root ++ "library/hkdf.c", - root ++ "library/hmac_drbg.c", - root ++ "library/lmots.c", - root ++ "library/lms.c", - root ++ "library/md.c", - root ++ "library/md5.c", - root ++ "library/memory_buffer_alloc.c", - root ++ "library/nist_kw.c", - root ++ "library/oid.c", - root ++ "library/padlock.c", - root ++ "library/pem.c", - root ++ "library/pk.c", - root ++ "library/pk_ecc.c", - root ++ "library/pk_wrap.c", - root ++ "library/pkcs12.c", - root ++ "library/pkcs5.c", - root ++ "library/pkparse.c", - root ++ "library/pkwrite.c", - root ++ "library/platform.c", - root ++ "library/platform_util.c", - root ++ "library/poly1305.c", - root ++ "library/psa_crypto.c", - root ++ "library/psa_crypto_aead.c", - root ++ "library/psa_crypto_cipher.c", - root ++ "library/psa_crypto_client.c", - root ++ "library/psa_crypto_ffdh.c", - root ++ "library/psa_crypto_driver_wrappers_no_static.c", - root ++ "library/psa_crypto_ecp.c", - root ++ "library/psa_crypto_hash.c", - root ++ "library/psa_crypto_mac.c", - root ++ "library/psa_crypto_pake.c", - root ++ "library/psa_crypto_rsa.c", - root ++ "library/psa_crypto_se.c", - root ++ "library/psa_crypto_slot_management.c", - root ++ "library/psa_crypto_storage.c", - root ++ "library/psa_its_file.c", - root ++ "library/psa_util.c", - root ++ "library/ripemd160.c", - root ++ "library/rsa.c", - root ++ "library/rsa_alt_helpers.c", - root ++ "library/sha1.c", - root ++ "library/sha3.c", - root ++ "library/sha256.c", - root ++ "library/sha512.c", - root ++ "library/threading.c", - root ++ "library/timing.c", - root ++ "library/version.c", - root ++ "library/version_features.c", - root ++ "library/pkcs7.c", - root ++ "library/x509.c", - root ++ "library/x509_create.c", - root ++ "library/x509_crl.c", - root ++ "library/x509_crt.c", - root ++ "library/x509_csr.c", - root ++ "library/x509write.c", - root ++ "library/x509write_crt.c", - root ++ "library/x509write_csr.c", - root ++ "library/debug.c", - root ++ "library/mps_reader.c", - root ++ "library/mps_trace.c", - root ++ "library/net_sockets.c", - root ++ "library/ssl_cache.c", - root ++ "library/ssl_ciphersuites.c", - root ++ "library/ssl_client.c", - root ++ "library/ssl_cookie.c", - root ++ "library/ssl_debug_helpers_generated.c", - root ++ "library/ssl_msg.c", - root ++ "library/ssl_ticket.c", - root ++ "library/ssl_tls.c", - root ++ "library/ssl_tls12_client.c", - root ++ "library/ssl_tls12_server.c", - root ++ "library/ssl_tls13_keys.c", - root ++ "library/ssl_tls13_server.c", - root ++ "library/ssl_tls13_client.c", - root ++ "library/ssl_tls13_generic.c", - } - }); + mbedtls.addCSourceFiles(.{ .flags = &.{}, .files = &.{ + root ++ "library/aes.c", + root ++ "library/aesni.c", + root ++ "library/aesce.c", + root ++ "library/aria.c", + root ++ "library/asn1parse.c", + root ++ "library/asn1write.c", + root ++ "library/base64.c", + root ++ "library/bignum.c", + root ++ "library/bignum_core.c", + root ++ "library/bignum_mod.c", + root ++ "library/bignum_mod_raw.c", + root ++ "library/camellia.c", + root ++ "library/ccm.c", + root ++ "library/chacha20.c", + root ++ "library/chachapoly.c", + root ++ "library/cipher.c", + root ++ "library/cipher_wrap.c", + root ++ "library/constant_time.c", + root ++ "library/cmac.c", + root ++ "library/ctr_drbg.c", + root ++ "library/des.c", + root ++ "library/dhm.c", + root ++ "library/ecdh.c", + root ++ "library/ecdsa.c", + root ++ "library/ecjpake.c", + root ++ "library/ecp.c", + root ++ "library/ecp_curves.c", + root ++ "library/entropy.c", + root ++ "library/entropy_poll.c", + root ++ "library/error.c", + root ++ "library/gcm.c", + root ++ "library/hkdf.c", + root ++ "library/hmac_drbg.c", + root ++ "library/lmots.c", + root ++ "library/lms.c", + root ++ "library/md.c", + root ++ "library/md5.c", + root ++ "library/memory_buffer_alloc.c", + root ++ "library/nist_kw.c", + root ++ "library/oid.c", + root ++ "library/padlock.c", + root ++ "library/pem.c", + root ++ "library/pk.c", + root ++ "library/pk_ecc.c", + root ++ "library/pk_wrap.c", + root ++ "library/pkcs12.c", + root ++ "library/pkcs5.c", + root ++ "library/pkparse.c", + root ++ "library/pkwrite.c", + root ++ "library/platform.c", + root ++ "library/platform_util.c", + root ++ "library/poly1305.c", + root ++ "library/psa_crypto.c", + root ++ "library/psa_crypto_aead.c", + root ++ "library/psa_crypto_cipher.c", + root ++ "library/psa_crypto_client.c", + root ++ "library/psa_crypto_ffdh.c", + root ++ "library/psa_crypto_driver_wrappers_no_static.c", + root ++ "library/psa_crypto_ecp.c", + root ++ "library/psa_crypto_hash.c", + root ++ "library/psa_crypto_mac.c", + root ++ "library/psa_crypto_pake.c", + root ++ "library/psa_crypto_rsa.c", + root ++ "library/psa_crypto_se.c", + root ++ "library/psa_crypto_slot_management.c", + root ++ "library/psa_crypto_storage.c", + root ++ "library/psa_its_file.c", + root ++ "library/psa_util.c", + root ++ "library/ripemd160.c", + root ++ "library/rsa.c", + root ++ "library/rsa_alt_helpers.c", + root ++ "library/sha1.c", + root ++ "library/sha3.c", + root ++ "library/sha256.c", + root ++ "library/sha512.c", + root ++ "library/threading.c", + root ++ "library/timing.c", + root ++ "library/version.c", + root ++ "library/version_features.c", + root ++ "library/pkcs7.c", + root ++ "library/x509.c", + root ++ "library/x509_create.c", + root ++ "library/x509_crl.c", + root ++ "library/x509_crt.c", + root ++ "library/x509_csr.c", + root ++ "library/x509write.c", + root ++ "library/x509write_crt.c", + root ++ "library/x509write_csr.c", + root ++ "library/debug.c", + root ++ "library/mps_reader.c", + root ++ "library/mps_trace.c", + root ++ "library/net_sockets.c", + root ++ "library/ssl_cache.c", + root ++ "library/ssl_ciphersuites.c", + root ++ "library/ssl_client.c", + root ++ "library/ssl_cookie.c", + root ++ "library/ssl_debug_helpers_generated.c", + root ++ "library/ssl_msg.c", + root ++ "library/ssl_ticket.c", + root ++ "library/ssl_tls.c", + root ++ "library/ssl_tls12_client.c", + root ++ "library/ssl_tls12_server.c", + root ++ "library/ssl_tls13_keys.c", + root ++ "library/ssl_tls13_server.c", + root ++ "library/ssl_tls13_client.c", + root ++ "library/ssl_tls13_generic.c", + } }); } fn buildNghttp2(b: *Build, m: *Build.Module) !void { @@ -611,41 +603,38 @@ fn buildNghttp2(b: *Build, m: *Build.Module) !void { const root = "vendor/nghttp2/"; nghttp2.addIncludePath(b.path(root ++ "lib")); nghttp2.addIncludePath(b.path(root ++ "lib/includes")); - nghttp2.addCSourceFiles(.{ - .flags = &.{ - "-DNGHTTP2_STATICLIB", - "-DHAVE_NETINET_IN", - "-DHAVE_TIME_H", - }, - .files = &.{ - root ++ "lib/sfparse.c", - root ++ "lib/nghttp2_alpn.c", - root ++ "lib/nghttp2_buf.c", - root ++ "lib/nghttp2_callbacks.c", - root ++ "lib/nghttp2_debug.c", - root ++ "lib/nghttp2_extpri.c", - root ++ "lib/nghttp2_frame.c", - root ++ "lib/nghttp2_hd.c", - root ++ "lib/nghttp2_hd_huffman.c", - root ++ "lib/nghttp2_hd_huffman_data.c", - root ++ "lib/nghttp2_helper.c", - root ++ "lib/nghttp2_http.c", - root ++ "lib/nghttp2_map.c", - root ++ "lib/nghttp2_mem.c", - root ++ "lib/nghttp2_option.c", - root ++ "lib/nghttp2_outbound_item.c", - root ++ "lib/nghttp2_pq.c", - root ++ "lib/nghttp2_priority_spec.c", - root ++ "lib/nghttp2_queue.c", - root ++ "lib/nghttp2_rcbuf.c", - root ++ "lib/nghttp2_session.c", - root ++ "lib/nghttp2_stream.c", - root ++ "lib/nghttp2_submit.c", - root ++ "lib/nghttp2_version.c", - root ++ "lib/nghttp2_ratelim.c", - root ++ "lib/nghttp2_time.c", - } - }); + nghttp2.addCSourceFiles(.{ .flags = &.{ + "-DNGHTTP2_STATICLIB", + "-DHAVE_NETINET_IN", + "-DHAVE_TIME_H", + }, .files = &.{ + root ++ "lib/sfparse.c", + root ++ "lib/nghttp2_alpn.c", + root ++ "lib/nghttp2_buf.c", + root ++ "lib/nghttp2_callbacks.c", + root ++ "lib/nghttp2_debug.c", + root ++ "lib/nghttp2_extpri.c", + root ++ "lib/nghttp2_frame.c", + root ++ "lib/nghttp2_hd.c", + root ++ "lib/nghttp2_hd_huffman.c", + root ++ "lib/nghttp2_hd_huffman_data.c", + root ++ "lib/nghttp2_helper.c", + root ++ "lib/nghttp2_http.c", + root ++ "lib/nghttp2_map.c", + root ++ "lib/nghttp2_mem.c", + root ++ "lib/nghttp2_option.c", + root ++ "lib/nghttp2_outbound_item.c", + root ++ "lib/nghttp2_pq.c", + root ++ "lib/nghttp2_priority_spec.c", + root ++ "lib/nghttp2_queue.c", + root ++ "lib/nghttp2_rcbuf.c", + root ++ "lib/nghttp2_session.c", + root ++ "lib/nghttp2_stream.c", + root ++ "lib/nghttp2_submit.c", + root ++ "lib/nghttp2_version.c", + root ++ "lib/nghttp2_ratelim.c", + root ++ "lib/nghttp2_time.c", + } }); } fn buildCurl(b: *Build, m: *Build.Module) !void { @@ -659,8 +648,7 @@ fn buildCurl(b: *Build, m: *Build.Module) !void { curl.addIncludePath(b.path(root ++ "lib")); curl.addIncludePath(b.path(root ++ "include")); curl.addCSourceFiles(.{ - .flags = &.{ - }, + .flags = &.{}, .files = &.{ root ++ "lib/altsvc.c", root ++ "lib/amigaos.c",