diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..44e7ac1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,16 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- `local_address` parameter to bind outgoing TCP connections to a specific local IP for outbound interface/IP selection (#65) + +### Fixed +- `Do()` dropped `ServerName`, `TLS13AutoRetry`, and `DisableGrease` request fields when constructing the underlying request (#65) +- `TLS13AutoRetry` proactive upgrade corrupted JA3 `supported_groups`; the original JA3 ordering is now preserved across the retry (#65) +- `dispatchSSEAsync` could enter an infinite loop on stream cancel/EOF (#65) diff --git a/cycletls/_config.py b/cycletls/_config.py index 6a3e2e2..76f3b74 100644 --- a/cycletls/_config.py +++ b/cycletls/_config.py @@ -19,6 +19,7 @@ "default_disable_grease": "disable_grease", "default_user_agent": "user_agent", "default_proxy": "proxy", + "default_local_address": "local_address", "default_timeout": "timeout", "default_enable_connection_reuse": "enable_connection_reuse", "default_insecure_skip_verify": "insecure_skip_verify", @@ -47,6 +48,7 @@ def _validate_config(name: str, value: Any) -> None: "default_disable_grease": lambda v: isinstance(v, bool), "default_order_headers_as_provided": lambda v: isinstance(v, bool), "default_proxy": lambda v: v is None or isinstance(v, str), + "default_local_address": lambda v: v is None or isinstance(v, str), "default_ja3": lambda v: v is None or isinstance(v, str), "default_ja4r": lambda v: v is None or isinstance(v, str), "default_user_agent": lambda v: v is None or isinstance(v, str), diff --git a/cycletls/api.py b/cycletls/api.py index 9e09696..790f610 100644 --- a/cycletls/api.py +++ b/cycletls/api.py @@ -516,7 +516,17 @@ def request( If a string, looks up the profile in FingerprintRegistry. Applies the profile's ja3, user_agent, header_order, etc. auth: (username, password) tuple for HTTP Basic authentication - **kwargs: Additional CycleTLS options (headers, cookies, proxy, etc.) + **kwargs: Additional CycleTLS options. Notable options include: + - headers: dict of extra HTTP headers + - cookies: list of cookie dicts + - ja3: JA3 TLS fingerprint string + - user_agent: User-Agent header value + - proxy: proxy URL (e.g. "http://user:pass@host:port") + - local_address: local IP address to bind the outgoing TCP + connection to (useful on multi-homed hosts) + - timeout: request timeout in seconds + - insecure_skip_verify: skip TLS certificate verification + - enable_connection_reuse: reuse persistent connections (default True) Returns: Response: Response object with status, headers, body, cookies, etc. diff --git a/cycletls/schema.py b/cycletls/schema.py index 4b423f8..ecce314 100644 --- a/cycletls/schema.py +++ b/cycletls/schema.py @@ -157,6 +157,7 @@ class Request: # Connection options user_agent: str = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:87.0) Gecko/20100101 Firefox/87.0" proxy: str = "" + local_address: Optional[str] = None # Bind outgoing TCP connections to this local IP address cookies: Optional[List[Cookie]] = None timeout: Union[int, float] = 6 # Timeout in seconds (floats are rounded up to nearest integer) disable_redirect: bool = False @@ -226,6 +227,8 @@ def to_dict(self) -> dict: result["quicFingerprint"] = self.quic_fingerprint if self.server_name is not None: result["serverName"] = self.server_name + if self.local_address is not None: + result["localAddress"] = self.local_address if self.protocol is not None: result["protocol"] = self.protocol.value if self.cookies is not None: diff --git a/golang/client.go b/golang/client.go index edd98a8..218ec8a 100644 --- a/golang/client.go +++ b/golang/client.go @@ -6,6 +6,7 @@ import ( fhttp "github.com/Danny-Dasilva/fhttp" "hash/crc32" "hash/fnv" + "net" "sync" "time" @@ -94,6 +95,16 @@ type Browser struct { ForceHTTP1 bool ForceHTTP3 bool + // LocalAddress, when non-empty, is the local IP the kernel binds the + // outbound TCP socket to (via net.Dialer.LocalAddr). When a proxy is in + // the path, the bind applies to the client->proxy hop only — the proxy + // opens its own socket to the destination, so the destination server + // never observes this IP. Treat this as routing/interface control, NOT + // as an anonymizer against the proxy itself: the proxy host always sees + // LocalAddress (and any on-path observer between client and proxy does + // too). + LocalAddress string + // DisableKeepAlives, when true, disables HTTP keep-alives at the // inner http.Transport layer for every request that uses this Browser. // Set this when the caller passes enable_connection_reuse=false from @@ -207,7 +218,7 @@ func NewTransportWithProxy(ja3 string, useragent string, proxy proxy.ContextDial } // generateClientKey creates a unique key for client pooling based on browser configuration -func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxyURL string) string { +func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxyURL string, localAddress string) string { // Create cookie signature for the key cookieStr := "" for _, cookie := range browser.Cookies { @@ -215,7 +226,7 @@ func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxy } // Create a hash of the configuration that affects connection behavior - configStr := fmt.Sprintf("ja3:%s|ja4r:%s|http2:%s|quic:%s|ua:%s|sni:%s|proxy:%s|timeout:%d|redirect:%t|skipverify:%t|forcehttp1:%t|forcehttp3:%t%s", + configStr := fmt.Sprintf("ja3:%s|ja4r:%s|http2:%s|quic:%s|ua:%s|sni:%s|proxy:%s|local:%s|timeout:%d|redirect:%t|skipverify:%t|forcehttp1:%t|forcehttp3:%t%s", browser.JA3, browser.JA4r, browser.HTTP2Fingerprint, @@ -223,6 +234,7 @@ func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxy browser.UserAgent, browser.ServerName, proxyURL, + localAddress, timeout, disableRedirect, browser.InsecureSkipVerify, @@ -259,12 +271,12 @@ func generateClientKey(browser Browser, timeout int, disableRedirect bool, proxy // http.Transport would still pool idle conns per address — see the badssl // triage report at .claude/cache/agents/source-tracer/output.md for the // failure mode this prevents. -func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userAgent string, enableConnectionReuse bool, proxyURL ...string) (fhttp.Client, error) { +func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userAgent string, enableConnectionReuse bool, localAddress string, proxyURL ...string) (fhttp.Client, error) { // If connection reuse is disabled, always create a new client and tell // the inner http.Transport to disable keep-alives. if !enableConnectionReuse { browser.DisableKeepAlives = true - return createNewClient(browser, timeout, disableRedirect, userAgent, proxyURL...) + return createNewClient(browser, timeout, disableRedirect, userAgent, localAddress, proxyURL...) } proxyStr := "" @@ -272,7 +284,7 @@ func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userA proxyStr = proxyURL[0] } - clientKey := generateClientKey(browser, timeout, disableRedirect, proxyStr) + clientKey := generateClientKey(browser, timeout, disableRedirect, proxyStr, localAddress) // Get the appropriate shard for this client key shard := globalPool.getShard(clientKey) @@ -299,7 +311,7 @@ func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userA } // Create new client - client, err := createNewClient(browser, timeout, disableRedirect, userAgent, proxyURL...) + client, err := createNewClient(browser, timeout, disableRedirect, userAgent, localAddress, proxyURL...) if err != nil { return fhttp.Client{}, err } @@ -316,17 +328,26 @@ func getOrCreateClient(browser Browser, timeout int, disableRedirect bool, userA } // createNewClient creates a new HTTP client (internal function) -func createNewClient(browser Browser, timeout int, disableRedirect bool, userAgent string, proxyURL ...string) (fhttp.Client, error) { +func createNewClient(browser Browser, timeout int, disableRedirect bool, userAgent string, localAddress string, proxyURL ...string) (fhttp.Client, error) { + // Stamp localAddress onto the Browser so downstream consumers (the + // roundTripper, in particular the HTTP/3 UDP listener) can honor it. + browser.LocalAddress = localAddress var dialer proxy.ContextDialer if len(proxyURL) > 0 && len(proxyURL[0]) > 0 { var err error - dialer, err = newConnectDialer(proxyURL[0], userAgent) + dialer, err = newConnectDialer(proxyURL[0], userAgent, localAddress) if err != nil { return fhttp.Client{ Timeout: time.Duration(timeout) * time.Second, CheckRedirect: disabledRedirect, }, err } + } else if localAddress != "" { + ip := net.ParseIP(localAddress) + if ip == nil { + return fhttp.Client{}, fmt.Errorf("invalid local_address %q: not a valid IP address", localAddress) + } + dialer = &net.Dialer{LocalAddr: &net.TCPAddr{IP: ip}} } else { dialer = proxy.Direct } @@ -373,12 +394,12 @@ func clearAllConnections() { // newClient creates a new http client (backward compatibility - defaults to no connection reuse) func newClient(browser Browser, timeout int, disableRedirect bool, UserAgent string, proxyURL ...string) (fhttp.Client, error) { // Backward compatibility: default to no connection reuse for existing code - return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, false, proxyURL...) + return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, false, "", proxyURL...) } // newClientWithReuse creates a new http client with configurable connection reuse -func newClientWithReuse(browser Browser, timeout int, disableRedirect bool, UserAgent string, enableConnectionReuse bool, proxyURL ...string) (fhttp.Client, error) { - return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, enableConnectionReuse, proxyURL...) +func newClientWithReuse(browser Browser, timeout int, disableRedirect bool, UserAgent string, enableConnectionReuse bool, localAddress string, proxyURL ...string) (fhttp.Client, error) { + return getOrCreateClient(browser, timeout, disableRedirect, UserAgent, enableConnectionReuse, localAddress, proxyURL...) } // WebSocketConnect establishes a WebSocket connection @@ -439,7 +460,7 @@ func (browser Browser) WebSocketConnect(ctx context.Context, urlStr string) (*we // SSEConnect establishes an SSE connection func (browser Browser) SSEConnect(ctx context.Context, urlStr string) (*SSEResponse, error) { // Create HTTP client with connection reuse enabled - httpClient, err := newClientWithReuse(browser, 30, false, browser.UserAgent, true) + httpClient, err := newClientWithReuse(browser, 30, false, browser.UserAgent, true, "") if err != nil { return nil, err } diff --git a/golang/connect.go b/golang/connect.go index 3d16112..1af95c6 100644 --- a/golang/connect.go +++ b/golang/connect.go @@ -6,6 +6,7 @@ import ( "context" "crypto/tls" "encoding/base64" + "errors" "fmt" "io" @@ -13,6 +14,7 @@ import ( "net/url" "strconv" "sync" + "time" http "github.com/Danny-Dasilva/fhttp" http2 "github.com/Danny-Dasilva/fhttp/http2" @@ -32,6 +34,120 @@ func (d *SocksDialer) Dial(network, addr string) (net.Conn, error) { return d.socksDial(network, addr) } +// socks4ContextDialer is a minimal SOCKS4 client that uses a configurable +// underlying TCP dialer (so net.Dialer.LocalAddr / context cancellation are +// honored on the client->proxy connection). h12.io/socks's DialSocksProxy +// always uses net.DialTimeout internally with no LocalAddr support, so we +// implement the SOCKS4 handshake here when localAddress binding is required. +// +// Wire format (RFC 1928 predecessor): VN=4, CD=1 (CONNECT), DSTPORT (BE u16), +// DSTIP (4 bytes), USERID (empty + null terminator). Server replies 8 bytes; +// resp[1]==0x5A means request granted. +type socks4ContextDialer struct { + proxyAddr string // host:port of the SOCKS4 proxy + dialer *net.Dialer // TCP dialer; carries LocalAddr when set +} + +func (d *socks4ContextDialer) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + conn, err := d.dialer.DialContext(ctx, "tcp", d.proxyAddr) + if err != nil { + return nil, err + } + // On any handshake failure, close the conn we just opened. + defer func() { + if err != nil { + _ = conn.Close() + } + }() + + // Resolve target host to IPv4 (SOCKS4 has no hostname field; SOCKS4A + // would, but the previous h12.io/socks SOCKS4 path also resolves locally). + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + port, err := strconv.Atoi(portStr) + if err != nil { + return nil, fmt.Errorf("invalid port %q: %w", portStr, err) + } + if port < 0 || port > 0xFFFF { + return nil, fmt.Errorf("port out of range: %d", port) + } + + var ip4 net.IP + if parsed := net.ParseIP(host); parsed != nil { + ip4 = parsed.To4() + } + if ip4 == nil { + ips, lerr := net.DefaultResolver.LookupIPAddr(ctx, host) + if lerr != nil { + return nil, lerr + } + for _, ia := range ips { + if v4 := ia.IP.To4(); v4 != nil { + ip4 = v4 + break + } + } + if ip4 == nil { + return nil, fmt.Errorf("no IPv4 address found for %q", host) + } + } + + req := []byte{ + 0x04, // VN + 0x01, // CD = CONNECT + byte(port >> 8), byte(port), // DSTPORT (big-endian) + ip4[0], ip4[1], ip4[2], ip4[3], // DSTIP + 0x00, // USERID = empty + null terminator + } + + if dl, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(dl) + } + + if _, err = conn.Write(req); err != nil { + return nil, err + } + + resp := make([]byte, 8) + if _, err = io.ReadFull(conn, resp); err != nil { + return nil, err + } + if resp[0] != 0x00 { + err = fmt.Errorf("socks4: malformed reply, first byte %#x not zero", resp[0]) + return nil, err + } + switch resp[1] { + case 0x5A: + // granted + case 0x5B: + err = errors.New("socks4: connection request rejected or failed") + return nil, err + case 0x5C: + err = errors.New("socks4: rejected because SOCKS server cannot connect to identd on the client") + return nil, err + case 0x5D: + err = errors.New("socks4: rejected because client and identd report different user-ids") + return nil, err + default: + err = fmt.Errorf("socks4: unknown reply code %#x", resp[1]) + return nil, err + } + + // Clear the deadline before returning. + if cerr := conn.SetDeadline(time.Time{}); cerr != nil { + err = cerr + return nil, err + } + + return conn, nil +} + +func (d *socks4ContextDialer) Dial(network, addr string) (net.Conn, error) { + return d.DialContext(context.Background(), network, addr) +} + // connectDialer allows to configure one-time use HTTP CONNECT client type connectDialer struct { ProxyURL url.URL @@ -43,6 +159,14 @@ type connectDialer struct { // MUST return connection with completed Handshake, and NegotiatedProtocol DialTLS func(network string, address string) (net.Conn, string, error) + // tlsDialer holds the underlying net.Dialer used when this connectDialer + // must establish a TLS connection to an https:// proxy. It carries + // LocalAddr (when set) so the client->proxy TCP socket binds the chosen + // local IP. tls.Dial does not consult any net.Dialer, which is why we + // need a separate field rather than reusing Dialer (which is a + // proxy.ContextDialer interface — possibly a SOCKS wrapper). + tlsDialer *net.Dialer + EnableH2ConnReuse bool cacheH2Mu sync.Mutex cachedH2ClientConn *http2.ClientConn @@ -52,7 +176,12 @@ type connectDialer struct { // newConnectDialer creates a dialer to issue CONNECT requests and tunnel traffic via HTTP/S proxy. // proxyUrlStr must provide Scheme and Host, may provide credentials and port. // Example: https://username:password@golang.org:443 -func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer, error) { +// localAddress optionally binds the outgoing TCP connection to the given local IP. +// The bind applies to the client->proxy hop only (the proxy opens its own +// socket to the destination), so the destination server never observes +// localAddress when a proxy is in the path. Only the proxy and any on-path +// observer between client and proxy see this IP. +func newConnectDialer(proxyURLStr string, UserAgent string, localAddress string) (proxy.ContextDialer, error) { proxyURL, err := url.Parse(proxyURLStr) if err != nil { return nil, err @@ -69,6 +198,20 @@ func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer EnableH2ConnReuse: true, } + // baseDialer is the underlying TCP dialer for the client->proxy hop. + // When localAddress is set it carries LocalAddr so the kernel binds the + // outbound socket to the chosen IP. We construct it BEFORE the scheme + // switch so SOCKS branches can plumb it through their forward / proxy + // dial hooks (the original code put this block AFTER the SOCKS early + // returns, silently dropping localAddress for SOCKS proxies). + baseDialer := &net.Dialer{} + if localAddress != "" { + if ip := net.ParseIP(localAddress); ip != nil { + baseDialer.LocalAddr = &net.TCPAddr{IP: ip} + } + } + client.tlsDialer = baseDialer + switch proxyURL.Scheme { case "http": if proxyURL.Port() == "" { @@ -87,10 +230,12 @@ func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer auth = &proxy.Auth{User: username, Password: password} } } - var forward proxy.Dialer - if proxyURL.Scheme == "socks5h" { - forward = proxy.Direct - } + // Use baseDialer (LocalAddr-aware when set) as the forward dialer so + // proxy.SOCKS5 invokes it for the TCP connect to the SOCKS5 proxy. + // For socks5h the original code used proxy.Direct; baseDialer is a + // strict superset (zero-value net.Dialer behaves like Direct when + // localAddress is empty). + var forward proxy.Dialer = baseDialer dialSocksProxy, err := proxy.SOCKS5("tcp", proxyURL.Host, auth, forward) if err != nil { return nil, fmt.Errorf("Error creating SOCKS5 proxy, reason %s", err) @@ -103,9 +248,20 @@ func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer client.DefaultHeader.Set("User-Agent", UserAgent) return client, nil case "socks4": - var dialer *SocksDialer - dialer = &SocksDialer{socks.DialSocksProxy(socks.SOCKS4, proxyURL.Host)} - client.Dialer = dialer + // h12.io/socks does not expose a forward-dialer hook, so when + // localAddress is set we use a custom SOCKS4 client that dials the + // proxy via baseDialer (with LocalAddr) and performs the SOCKS4 + // handshake manually. Otherwise we keep the original h12.io/socks + // path so the default behavior is unchanged. + if localAddress != "" { + client.Dialer = &socks4ContextDialer{ + proxyAddr: proxyURL.Host, + dialer: baseDialer, + } + } else { + dialer := &SocksDialer{socks.DialSocksProxy(socks.SOCKS4, proxyURL.Host)} + client.Dialer = dialer + } client.DefaultHeader.Set("User-Agent", UserAgent) return client, nil case "": @@ -114,7 +270,7 @@ func newConnectDialer(proxyURLStr string, UserAgent string) (proxy.ContextDialer return nil, errors.New("scheme " + proxyURL.Scheme + " is not supported") } - client.Dialer = &net.Dialer{} + client.Dialer = baseDialer if proxyURL.User != nil { if proxyURL.User.Username() != "" { @@ -249,13 +405,23 @@ func (c *connectDialer) DialContext(ctx context.Context, network, address string ServerName: c.ProxyURL.Hostname(), InsecureSkipVerify: true, } - tlsConn, err := tls.Dial(network, c.ProxyURL.Host, &tlsConf) - if err != nil { - return nil, err + // Dial the underlying TCP via the LocalAddr-aware net.Dialer + // (c.tlsDialer) so the client->proxy socket honors localAddress, + // then wrap in tls.Client and complete the handshake. The + // previous tls.Dial(...) path bypassed any custom Dialer and + // silently dropped localAddress for HTTPS proxies. + netDialer := c.tlsDialer + if netDialer == nil { + netDialer = &net.Dialer{} } - err = tlsConn.Handshake() - if err != nil { - return nil, err + plainConn, derr := netDialer.DialContext(ctx, network, c.ProxyURL.Host) + if derr != nil { + return nil, derr + } + tlsConn := tls.Client(plainConn, &tlsConf) + if herr := tlsConn.HandshakeContext(ctx); herr != nil { + _ = plainConn.Close() + return nil, herr } negotiatedProtocol = tlsConn.ConnectionState().NegotiatedProtocol rawConn = tlsConn diff --git a/golang/http3.go b/golang/http3.go index 9da0ce3..01ba0b3 100644 --- a/golang/http3.go +++ b/golang/http3.go @@ -409,8 +409,19 @@ func (rt *roundTripper) http3Dial(ctx context.Context, remoteAddr, port string, return nil, fmt.Errorf("HTTP/3 proxy support not yet implemented") } - // Direct UDP connection - conn, err := net.ListenPacket("udp", "") + // Direct UDP connection. + // + // When rt.LocalAddress is set, bind the UDP socket to that local IP so + // HTTP/3 honors the same local-address semantics as the TCP-based dial + // paths. Empty laddr (the default) lets the kernel choose, matching the + // previous behavior for the unset case. + listenAddr := "" + if rt.LocalAddress != "" { + if ip := net.ParseIP(rt.LocalAddress); ip != nil { + listenAddr = net.JoinHostPort(ip.String(), "0") + } + } + conn, err := net.ListenPacket("udp", listenAddr) if err != nil { return nil, fmt.Errorf("failed to create UDP packet connection: %w", err) } diff --git a/golang/index.go b/golang/index.go index 732df5f..b0783c6 100644 --- a/golang/index.go +++ b/golang/index.go @@ -121,7 +121,18 @@ type Options struct { UserAgent string `json:"userAgent" msgpack:"userAgent"` // Connection options - Proxy string `json:"proxy" msgpack:"proxy"` + Proxy string `json:"proxy" msgpack:"proxy"` + // LocalAddress, when non-empty, binds outgoing TCP/UDP sockets to the + // given local IP via SO_BINDTODEVICE-style net.Dialer.LocalAddr. + // + // IMPORTANT: when a Proxy is also set, the bind applies to the + // client->proxy hop only. The proxy opens its own socket to the + // destination, so the destination server never sees this IP. The proxy + // host (and any on-path observer between the client and the proxy) + // always observes LocalAddress. Use this for routing/interface + // control (multi-homed hosts, residential IP rotation through your own + // egress proxy, etc.), NOT as an anonymizer against the proxy itself. + LocalAddress string `json:"localAddress" msgpack:"localAddress"` ServerName string `json:"serverName" msgpack:"serverName"` // Custom TLS SNI override Cookies []Cookie `json:"cookies" msgpack:"cookies"` Timeout int `json:"timeout" msgpack:"timeout"` @@ -182,23 +193,23 @@ var debugLogger = log.New(os.Stdout, "DEBUG: ", log.Ldate|log.Ltime|log.Lshortfi // WebSocket connection management type WebSocketConnection struct { - Conn *websocket.Conn - RequestID string - URL string - ReadyState int // 0=CONNECTING, 1=OPEN, 2=CLOSING, 3=CLOSED - mu sync.RWMutex - commandChan chan WebSocketCommand - closeChan chan struct{} - chanWrite *safeChannelWriter - protocol string // Negotiated subprotocol - extensions string // Negotiated extensions + Conn *websocket.Conn + RequestID string + URL string + ReadyState int // 0=CONNECTING, 1=OPEN, 2=CLOSING, 3=CLOSED + mu sync.RWMutex + commandChan chan WebSocketCommand + closeChan chan struct{} + chanWrite *safeChannelWriter + protocol string // Negotiated subprotocol + extensions string // Negotiated extensions } type WebSocketCommand struct { - Type string // "send", "close", "ping", "pong" - Data []byte - IsBinary bool - CloseCode int + Type string // "send", "close", "ping", "pong" + Data []byte + IsBinary bool + CloseCode int CloseReason string } @@ -260,6 +271,7 @@ func processRequest(request cycleTLSRequest) (result fullRequest) { request.Options.DisableRedirect, request.Options.UserAgent, enableConnectionReuse, + request.Options.LocalAddress, request.Options.Proxy, ) if err != nil { @@ -412,6 +424,7 @@ func dispatchHTTP3Request(request cycleTLSRequest) (result fullRequest) { request.Options.DisableRedirect, request.Options.UserAgent, enableConnectionReuse, + request.Options.LocalAddress, request.Options.Proxy, ) if err != nil { @@ -500,6 +513,7 @@ func dispatchSSERequest(request cycleTLSRequest) (result fullRequest) { request.Options.DisableRedirect, request.Options.UserAgent, enableConnectionReuse, + request.Options.LocalAddress, request.Options.Proxy, ) if err != nil { @@ -856,9 +870,9 @@ func dispatcherAsync(res fullRequest, chanWrite *safeChannelWriter) { b.WriteString(message) if !chanWrite.write(b.Bytes()) { - log.Printf("Failed to write to channel: channel closed") - return - } + log.Printf("Failed to write to channel: channel closed") + return + } break loop } @@ -882,9 +896,9 @@ func dispatcherAsync(res fullRequest, chanWrite *safeChannelWriter) { b.Write(chunkBuffer[:n]) if !chanWrite.write(b.Bytes()) { - log.Printf("Failed to write to channel: channel closed") - return - } + log.Printf("Failed to write to channel: channel closed") + return + } } // EOF reached, exit the loop break loop @@ -912,9 +926,9 @@ func dispatcherAsync(res fullRequest, chanWrite *safeChannelWriter) { b.Write(chunkBuffer[:n]) if !chanWrite.write(b.Bytes()) { - log.Printf("Failed to write to channel: channel closed") - return - } + log.Printf("Failed to write to channel: channel closed") + return + } } } } @@ -1030,21 +1044,22 @@ func dispatchSSEAsync(res fullRequest, chanWrite *safeChannelWriter) { } // Read SSE events +sseLoop: for { select { case <-res.req.Context().Done(): debugLogger.Printf("SSE request %s was canceled", res.options.RequestID) - break + break sseLoop default: event, err := sseResp.NextEvent() if err != nil { if err == io.EOF { // Normal end of stream - break + break sseLoop } debugLogger.Printf("SSE read error: %s", err.Error()) - break + break sseLoop } if event == nil { @@ -1083,9 +1098,9 @@ func dispatchSSEAsync(res fullRequest, chanWrite *safeChannelWriter) { b.Write(eventBytes) if !chanWrite.write(b.Bytes()) { - log.Printf("Failed to write to channel: channel closed") - return - } + log.Printf("Failed to write to channel: channel closed") + return + } } } @@ -1138,15 +1153,15 @@ func dispatchWebSocketAsync(res fullRequest, chanWrite *safeChannelWriter) { // Create WebSocket connection object wsConn := &WebSocketConnection{ - Conn: conn, - RequestID: res.options.RequestID, - URL: res.options.Options.URL, - ReadyState: 1, // OPEN + Conn: conn, + RequestID: res.options.RequestID, + URL: res.options.Options.URL, + ReadyState: 1, // OPEN commandChan: make(chan WebSocketCommand, 100), - closeChan: make(chan struct{}), - chanWrite: chanWrite, - protocol: negotiatedProtocol, - extensions: negotiatedExtensions, + closeChan: make(chan struct{}), + chanWrite: chanWrite, + protocol: negotiatedProtocol, + extensions: negotiatedExtensions, } // Register the WebSocket connection @@ -1703,6 +1718,9 @@ func (client CycleTLS) Do(URL string, options Options, Method string) (Response, UserAgent: options.UserAgent, Cookies: options.Cookies, InsecureSkipVerify: options.InsecureSkipVerify, + ServerName: options.ServerName, + TLS13AutoRetry: options.TLS13AutoRetry, + DisableGrease: options.DisableGrease, ForceHTTP1: options.ForceHTTP1, ForceHTTP3: options.ForceHTTP3, HeaderOrder: options.HeaderOrder, @@ -1725,6 +1743,7 @@ func (client CycleTLS) Do(URL string, options Options, Method string) (Response, options.DisableRedirect, options.UserAgent, enableConnectionReuse, + options.LocalAddress, options.Proxy, ) if err != nil { diff --git a/golang/roundtripper.go b/golang/roundtripper.go index 2d14489..7c3a684 100644 --- a/golang/roundtripper.go +++ b/golang/roundtripper.go @@ -25,8 +25,8 @@ type roundTripper struct { sync.Mutex // Per-address mutexes for preventing concurrent transport creation - addressMutexes map[string]*sync.Mutex - addressMutexLock sync.Mutex + addressMutexes map[string]*sync.Mutex + addressMutexLock sync.Mutex // TLS fingerprinting options JA3 string @@ -48,6 +48,11 @@ type roundTripper struct { ForceHTTP1 bool ForceHTTP3 bool + // LocalAddress is the local IP to bind outbound sockets to. Used by + // http3Dial for the UDP ListenPacket address; for TCP dials it is + // applied via the net.Dialer in client.go / connect.go. + LocalAddress string + // DisableKeepAlives, when true, sets DisableKeepAlives on every // inner http.Transport that this roundTripper constructs. Wired // from Browser.DisableKeepAlives, which getOrCreateClient sets when @@ -179,12 +184,12 @@ func (rt *roundTripper) getTransport(req *http.Request, addr string) error { case "http": // Allow connection reuse with optimized connection pooling rt.cachedTransports[addr] = &http.Transport{ - DialContext: rt.dialer.DialContext, - MaxIdleConns: 100, - MaxConnsPerHost: 100, - MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close - IdleConnTimeout: 60 * time.Second, - DisableKeepAlives: rt.DisableKeepAlives, + DialContext: rt.dialer.DialContext, + MaxIdleConns: 100, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close + IdleConnTimeout: 60 * time.Second, + DisableKeepAlives: rt.DisableKeepAlives, } return nil case "https": @@ -356,12 +361,12 @@ func (rt *roundTripper) dialTLS(ctx context.Context, network, addr string) (net. default: // HTTP/1.x transport - optimized for connection reuse rt.cachedTransports[addr] = &http.Transport{ - DialTLSContext: rt.dialTLS, - MaxIdleConns: 100, - MaxConnsPerHost: 100, - MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close - IdleConnTimeout: 60 * time.Second, - DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False + DialTLSContext: rt.dialTLS, + MaxIdleConns: 100, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close + IdleConnTimeout: 60 * time.Second, + DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False } } @@ -458,12 +463,12 @@ func (rt *roundTripper) retryWithTLS13CompatibleCurves(ctx context.Context, netw default: // HTTP/1.x transport - optimized for connection reuse rt.cachedTransports[addr] = &http.Transport{ - DialTLSContext: rt.dialTLS, - MaxIdleConns: 100, - MaxConnsPerHost: 100, - MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close - IdleConnTimeout: 60 * time.Second, - DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False + DialTLSContext: rt.dialTLS, + MaxIdleConns: 100, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close + IdleConnTimeout: 60 * time.Second, + DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False } } @@ -537,12 +542,12 @@ func (rt *roundTripper) retryWithOriginalTLS12JA3(ctx context.Context, network, default: // HTTP/1.x transport - optimized for connection reuse rt.cachedTransports[addr] = &http.Transport{ - DialTLSContext: rt.dialTLS, - MaxIdleConns: 100, - MaxConnsPerHost: 100, - MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close - IdleConnTimeout: 60 * time.Second, - DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False + DialTLSContext: rt.dialTLS, + MaxIdleConns: 100, + MaxConnsPerHost: 100, + MaxIdleConnsPerHost: 100, // Go default is 2, which causes 98% of connections to close + IdleConnTimeout: 60 * time.Second, + DisableKeepAlives: rt.DisableKeepAlives, // Bound to enable_connection_reuse=False } } @@ -568,15 +573,15 @@ func (rt *roundTripper) getDialTLSAddr(req *http.Request) string { func (rt *roundTripper) getAddressMutex(addr string) *sync.Mutex { rt.addressMutexLock.Lock() defer rt.addressMutexLock.Unlock() - + if rt.addressMutexes == nil { rt.addressMutexes = make(map[string]*sync.Mutex) } - + if mu, exists := rt.addressMutexes[addr]; exists { return mu } - + mu := &sync.Mutex{} rt.addressMutexes[addr] = mu return mu @@ -643,6 +648,7 @@ func newRoundTripper(browser Browser, dialer ...proxy.ContextDialer) http.RoundT InsecureSkipVerify: browser.InsecureSkipVerify, ForceHTTP1: browser.ForceHTTP1, ForceHTTP3: browser.ForceHTTP3, + LocalAddress: browser.LocalAddress, DisableKeepAlives: browser.DisableKeepAlives, // TLS 1.3 specific options diff --git a/golang/utils.go b/golang/utils.go index 858295b..b979d3b 100644 --- a/golang/utils.go +++ b/golang/utils.go @@ -281,27 +281,48 @@ func StringToTLS13CompatibleSpec(ja3 string, userAgent string, forceHTTP1 bool) return StringToSpec(tls13CompatibleJA3, userAgent, forceHTTP1) } -// convertJA3ForTLS13 converts a JA3 string to use TLS 1.3 compatible curves +// convertJA3ForTLS13 converts a JA3 string to use TLS 1.3 compatible groups. +// It upgrades the TLS version token from 1.2 (771) to 1.3 (772) and filters the +// supported_groups field to only include groups that are valid per RFC 8446 §4.2.7, +// preserving the original ordering and all compatible groups (P-256, P-384, P-521, +// X25519, X448, FFDHE groups, post-quantum hybrids). func convertJA3ForTLS13(ja3 string) string { tokens := strings.Split(ja3, ",") if len(tokens) != 5 { return ja3 // Return original if malformed } - // Replace TLS version (position 0) with TLS 1.3 (772) if it's TLS 1.2 (771) + // Upgrade TLS version from 1.2 (771) to 1.3 (772) if tokens[0] == "771" { - tokens[0] = "772" // Upgrade TLS 1.2 to TLS 1.3 + tokens[0] = "772" } - // Replace curves (position 3) with TLS 1.3 compatible ones: X25519 (29) and secp256r1 (23) - tokens[3] = "29-23" // X25519 and secp256r1 + // Filter the curves/groups field to only TLS 1.3 compatible NamedGroups, + // preserving the original order and all compatible entries. + var compatible []string + for _, c := range strings.Split(tokens[3], "-") { + if c == "" { + continue + } + cid, err := strconv.ParseUint(c, 10, 16) + if err == nil && isTLS13CompatibleCurve(uint16(cid)) { + compatible = append(compatible, c) + } + } + if len(compatible) > 0 { + tokens[3] = strings.Join(compatible, "-") + } else { + tokens[3] = "29-23" // fallback: X25519 + secp256r1 + } return strings.Join(tokens, ",") } // isTLS13CompatibleCurve checks if a curve ID is compatible with TLS 1.3 func isTLS13CompatibleCurve(curveID uint16) bool { - // TLS 1.3 compatible curves based on RFC 8446 and common implementations + // TLS 1.3 compatible NamedGroups per RFC 8446 §4.2.7 and common implementations. + // Includes both ECDHE groups and FFDHE groups (RFC 7919) which are valid in + // the supported_groups extension even though TLS 1.3 uses only ECDHE for key exchange. switch curveID { case 23: // secp256r1 (P-256) return true @@ -314,6 +335,18 @@ func isTLS13CompatibleCurve(curveID uint16) bool { case 30: // X448 return true + // FFDHE groups (RFC 7919) — Firefox and other browsers advertise these + case 256: // ffdhe2048 + return true + case 257: // ffdhe3072 + return true + case 258: // ffdhe4096 + return true + case 259: // ffdhe6144 + return true + case 260: // ffdhe8192 + return true + // Post-quantum hybrid curves (emerging standard) case 4587: // 0x11EB - SecP256r1MLKEM768 (P-256 + MLKEM768) return true diff --git a/tests/test_local_address.py b/tests/test_local_address.py new file mode 100644 index 0000000..4184b8d --- /dev/null +++ b/tests/test_local_address.py @@ -0,0 +1,106 @@ +""" +Tests for the local_address parameter. + +local_address binds the outgoing TCP connection to a specific local IP, +useful on multi-homed hosts. Unit tests cover schema serialisation and +invalid-IP validation; the live test makes a real request with the +loopback address (127.0.0.1) binding to verify end-to-end wiring. +""" + +import socket +import pytest +from cycletls import CycleTLS +from cycletls.schema import Request +from cycletls.exceptions import ConnectionError as CycleTLSConnectionError + + +# --------------------------------------------------------------------------- +# Unit tests — no network required +# --------------------------------------------------------------------------- + +class TestLocalAddressSchema: + """local_address is correctly serialised into the request payload.""" + + def test_local_address_included_when_set(self): + req = Request(method="GET", url="https://example.com", local_address="192.168.1.10") + payload = req.to_dict() + assert payload["localAddress"] == "192.168.1.10" + + def test_local_address_omitted_when_none(self): + req = Request(method="GET", url="https://example.com") + payload = req.to_dict() + assert "localAddress" not in payload + + def test_local_address_default_is_none(self): + req = Request(method="GET", url="https://example.com") + assert req.local_address is None + + +class TestLocalAddressValidation: + """The Go layer rejects invalid IP addresses.""" + + def test_invalid_local_address_raises(self): + """A non-IP string must raise an error, not silently succeed.""" + with CycleTLS() as client: + with pytest.raises(Exception) as exc_info: + client.get( + "https://httpbin.org/get", + local_address="not-an-ip-address", + enable_connection_reuse=False, + timeout=10, + ) + assert "invalid" in str(exc_info.value).lower() or \ + "local_address" in str(exc_info.value).lower() or \ + "not a valid" in str(exc_info.value).lower() + + +# --------------------------------------------------------------------------- +# Live test — requires network access +# --------------------------------------------------------------------------- + +@pytest.mark.live +class TestLocalAddressLive: + """Verify that local_address is wired through to the dialer.""" + + def test_local_address_loopback(self): + """ + Binding to 127.0.0.1 and connecting to an httpbin endpoint works + as long as the OS allows source-IP 127.0.0.1 for outbound traffic + (Linux does by default; the connection may fail on strict systems, + in which case we skip rather than fail). + """ + with CycleTLS() as client: + try: + response = client.get( + "https://httpbin.org/get", + local_address="127.0.0.1", + enable_connection_reuse=False, + timeout=15, + ) + assert response.status_code == 200 + except (CycleTLSConnectionError, Exception) as exc: + msg = str(exc).lower() + # Some OS/network configurations refuse source-IP 127.0.0.1 + # for non-loopback destinations — skip instead of failing. + if any(w in msg for w in ("invalid", "cannot assign", "bind", "network unreachable")): + pytest.skip(f"OS rejected loopback source binding: {exc}") + raise + + def test_local_address_default_outbound_ip(self): + """Binding to the machine's own outbound IP works for a normal request.""" + try: + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + s.close() + except Exception: + pytest.skip("Could not determine local outbound IP") + + with CycleTLS() as client: + response = client.get( + "https://httpbin.org/get", + local_address=local_ip, + enable_connection_reuse=False, + timeout=15, + ) + assert response.status_code == 200