起因

抓取某個HTTPS網站的時候
開啟charles代理能夠抓取成功,關閉被風控
通過檢測,懷疑可能是tls的時候有區別

嘗試

golang的http中,Transport.TLSClientConfig是可以自定義設置的
但起初通過隨意設置并不能繞過風控

困難

1. 使用golang的http客戶端,修改DialTLSContext函數的方式是可以實繞過風控,但使用proxy的時候,代碼會使用pconn.addTLS(ctx, cm.tlsHost(), trace) 重新以普通方式進行握手,導致JA3修改失敗
2. 因為golang強關聯,第三方庫并不能完美的集成到現有代碼中,都需要重構代碼
3. 某些網站會對于新建鏈接進行ClientSession檢測,因此需要 KeepAlive+ClientSessionCache,這樣通過復用連接減少風控概率

最終實現

1. 只需要拿到合法的參數,并且配置到TLSClientConfig里即可
2. 使用github.com/refraction-networking/utls中的UTLSIdToSpec拿到CipherSuites并傳入

package main
import ("bytes""crypto/tls"tlsx "github.com/refraction-networking/utls""net/http"
)
func main() {c, _ := tlsx.UTLSIdToSpec(tlsx.HelloRandomized)a := &http.Client{Transport: &http.Transport{DisableKeepAlives: false,Proxy: proxy,TLSClientConfig: &tls.Config{InsecureSkipVerify: true,MinVersion:         c.TLSVersMin,MaxVersion:         c.TLSVersMax,CipherSuites:       c.CipherSuites,ClientSessionCache: tls.NewLRUClientSessionCache(32),},},}aw, bw := a.Get("https://tls.browserleaks.com/json")defer aw.Body.Close()var buf bytes.Bufferaw.Write(&buf)println(string(buf.String()), bw)
}

參考文章

1. https://github.com/baixudong007/gospider
2. https://juejin.cn/post/7073264626506399751 用Go構建你專屬的JA3指紋
3. https://blog.csdn.net/qq523176585/article/details/127116542 好庫推薦|強烈推薦，支持Ja3指紋修改的golang請求庫
4. https://github.com/wangluozhe/requests
5. https://github.com/refraction-networking/utls
6. http://www.ctfiot.com/64337.html 如何繞過 JA3 指紋校驗？
7. https://www.coder.work/article/7192419 http - 發送請求時如何使用uTLS連接？
8. https://segmentfault.com/a/1190000041699815/en Build your own JA3 fingerprint with Go
9. https://zhuanlan.zhihu.com/p/601474166 curl_cffi: 支持原生模擬瀏覽器 TLS/JA3 指紋的 Python 庫
10. https://tools.scrapfly.io/api/fp/ja3

歷史編寫的代碼

這些代碼都不太好用

package mainimport ("context""fmt"tls "github.com/refraction-networking/utls"xtls "github.com/refraction-networking/utls""net""net/http"
)func GetTransport(helloID *xtls.ClientHelloID) *http.Transport {if helloID == nil {helloID = &xtls.HelloChrome_83}transport := http.DefaultTransport.(*http.Transport).Clone()transport.DialTLSContext = func(ctx context.Context, network, addr string) (_ net.Conn, err error) {dialer := net.Dialer{}con, err := dialer.DialContext(ctx, network, addr)if err != nil {return nil, err}// 根據地址獲取host信息host, _, err := net.SplitHostPort(addr)if err != nil {return nil, err}c := transport.TLSClientConfig// 并且不驗證host信息xtlsConf := &xtls.Config{ServerName: host,//Renegotiation:      xtls.RenegotiateNever,ClientSessionCache: xtls.NewLRUClientSessionCache(32),NextProtos:         []string{"h2", "http/1.1"},Rand: c.Rand,Time: c.Time,VerifyPeerCertificate: c.VerifyPeerCertificate,RootCAs: c.RootCAs,ClientCAs:                c.ClientCAs,InsecureSkipVerify:       c.InsecureSkipVerify,CipherSuites:             c.CipherSuites,PreferServerCipherSuites: c.PreferServerCipherSuites,SessionTicketsDisabled:   c.SessionTicketsDisabled,SessionTicketKey:         c.SessionTicketKey,MinVersion: c.MinVersion,MaxVersion: c.MaxVersion,DynamicRecordSizingDisabled: c.DynamicRecordSizingDisabled,KeyLogWriter: c.KeyLogWriter,}// 構建tls.UConnxtlsConn := xtls.UClient(con, xtlsConf, *helloID)// 握手err = xtlsConn.HandshakeContext(ctx)if err != nil {return nil, err}fmt.Println("當前請求使用協議：", xtlsConn.HandshakeState.ServerHello.AlpnProtocol)return xtlsConn, err}//transport.DialContext = transport.DialTLSContexttransport.DisableKeepAlives = truereturn transport
}
func CreateHTTPClient() *http.Transport {return &http.Transport{DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) {//initialize the tcp connectiontcpConn, err := (&net.Dialer{}).DialContext(ctx, network, addr)if err != nil {return nil, err}host, _, err := net.SplitHostPort(addr)if err != nil {return nil, err}//initialize the conifg for tlsconfig := tls.Config{ServerName:         host, //set the server name with the provided addrClientSessionCache: xtls.NewLRUClientSessionCache(0),}//initialize a tls connection with the underlying tcop connection and configtlsConn := tls.UClient(tcpConn, &config, tls.HelloRandomized)//start the tls handshake between serverserr = tlsConn.Handshake()if err != nil {return nil, fmt.Errorf("uTlsConn.Handshake() error: %w", err)}return tlsConn, nil},ForceAttemptHTTP2: false,}}