github.com/gospider007/requests@v0.0.0-20240506025355-c73d46169a23/requests.go (about) 1 package requests 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "io" 8 "net" 9 "strings" 10 "time" 11 12 "net/url" 13 "os" 14 15 "net/http" 16 17 "github.com/gospider007/gtls" 18 "github.com/gospider007/ja3" 19 "github.com/gospider007/re" 20 "github.com/gospider007/tools" 21 "github.com/gospider007/websocket" 22 ) 23 24 type contextKey string 25 26 const gospiderContextKey contextKey = "GospiderContextKey" 27 28 var errFatal = errors.New("ErrFatal") 29 30 type reqCtxData struct { 31 isWs bool 32 forceHttp1 bool 33 maxRedirect int 34 proxy *url.URL 35 disProxy bool 36 disAlive bool 37 orderHeaders []string 38 responseHeaderTimeout time.Duration 39 tlsHandshakeTimeout time.Duration 40 41 requestCallBack func(context.Context, *http.Request, *http.Response) error 42 43 h2Ja3Spec ja3.H2Ja3Spec 44 ja3Spec ja3.Ja3Spec 45 46 dialTimeout time.Duration 47 keepAlive time.Duration 48 localAddr *net.TCPAddr //network card ip 49 addrType gtls.AddrType //first ip type 50 dns *net.UDPAddr 51 isNewConn bool 52 } 53 54 func NewReqCtxData(ctx context.Context, option *RequestOption) (*reqCtxData, error) { 55 //init ctxData 56 ctxData := new(reqCtxData) 57 ctxData.ja3Spec = option.Ja3Spec 58 ctxData.h2Ja3Spec = option.H2Ja3Spec 59 ctxData.forceHttp1 = option.ForceHttp1 60 ctxData.disAlive = option.DisAlive 61 ctxData.maxRedirect = option.MaxRedirect 62 ctxData.requestCallBack = option.RequestCallBack 63 ctxData.responseHeaderTimeout = option.ResponseHeaderTimeout 64 ctxData.addrType = option.AddrType 65 ctxData.dialTimeout = option.DialTimeout 66 ctxData.keepAlive = option.KeepAlive 67 ctxData.localAddr = option.LocalAddr 68 ctxData.dns = option.Dns 69 ctxData.disProxy = option.DisProxy 70 ctxData.tlsHandshakeTimeout = option.TlsHandshakeTimeout 71 //init scheme 72 if option.Url != nil { 73 if option.Url.Scheme == "ws" { 74 ctxData.isWs = true 75 option.Url.Scheme = "http" 76 } else if option.Url.Scheme == "wss" { 77 ctxData.isWs = true 78 option.Url.Scheme = "https" 79 } 80 } 81 //init tls timeout 82 if option.TlsHandshakeTimeout == 0 { 83 ctxData.tlsHandshakeTimeout = time.Second * 15 84 } 85 //init proxy 86 if option.Proxy != "" { 87 tempProxy, err := gtls.VerifyProxy(option.Proxy) 88 if err != nil { 89 return nil, tools.WrapError(errFatal, errors.New("tempRequest init proxy error"), err) 90 } 91 ctxData.proxy = tempProxy 92 } 93 return ctxData, nil 94 } 95 func CreateReqCtx(ctx context.Context, ctxData *reqCtxData) context.Context { 96 return context.WithValue(ctx, gospiderContextKey, ctxData) 97 } 98 func GetReqCtxData(ctx context.Context) *reqCtxData { 99 return ctx.Value(gospiderContextKey).(*reqCtxData) 100 } 101 102 // sends a GET request and returns the response. 103 func Get(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 104 return defaultClient.Request(ctx, http.MethodGet, href, options...) 105 } 106 107 // sends a Head request and returns the response. 108 func Head(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 109 return defaultClient.Request(ctx, http.MethodHead, href, options...) 110 } 111 112 // sends a Post request and returns the response. 113 func Post(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 114 return defaultClient.Request(ctx, http.MethodPost, href, options...) 115 } 116 117 // sends a Put request and returns the response. 118 func Put(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 119 return defaultClient.Request(ctx, http.MethodPut, href, options...) 120 } 121 122 // sends a Patch request and returns the response. 123 func Patch(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 124 return defaultClient.Request(ctx, http.MethodPatch, href, options...) 125 } 126 127 // sends a Delete request and returns the response. 128 func Delete(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 129 return defaultClient.Request(ctx, http.MethodDelete, href, options...) 130 } 131 132 // sends a Connect request and returns the response. 133 func Connect(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 134 return defaultClient.Request(ctx, http.MethodConnect, href, options...) 135 } 136 137 // sends a Options request and returns the response. 138 func Options(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 139 return defaultClient.Request(ctx, http.MethodOptions, href, options...) 140 } 141 142 // sends a Trace request and returns the response. 143 func Trace(ctx context.Context, href string, options ...RequestOption) (resp *Response, err error) { 144 return defaultClient.Request(ctx, http.MethodTrace, href, options...) 145 } 146 147 // Define a function named Request that takes in four parameters: 148 func Request(ctx context.Context, method string, href string, options ...RequestOption) (resp *Response, err error) { 149 return defaultClient.Request(ctx, method, href, options...) 150 } 151 152 // sends a Get request and returns the response. 153 func (obj *Client) Get(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 154 return obj.Request(ctx, http.MethodGet, href, options...) 155 } 156 157 // sends a Head request and returns the response. 158 func (obj *Client) Head(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 159 return obj.Request(ctx, http.MethodHead, href, options...) 160 } 161 162 // sends a Post request and returns the response. 163 func (obj *Client) Post(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 164 return obj.Request(ctx, http.MethodPost, href, options...) 165 } 166 167 // sends a Put request and returns the response. 168 func (obj *Client) Put(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 169 return obj.Request(ctx, http.MethodPut, href, options...) 170 } 171 172 // sends a Patch request and returns the response. 173 func (obj *Client) Patch(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 174 return obj.Request(ctx, http.MethodPatch, href, options...) 175 } 176 177 // sends a Delete request and returns the response. 178 func (obj *Client) Delete(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 179 return obj.Request(ctx, http.MethodDelete, href, options...) 180 } 181 182 // sends a Connect request and returns the response. 183 func (obj *Client) Connect(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 184 return obj.Request(ctx, http.MethodConnect, href, options...) 185 } 186 187 // sends a Options request and returns the response. 188 func (obj *Client) Options(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 189 return obj.Request(ctx, http.MethodOptions, href, options...) 190 } 191 192 // sends a Trace request and returns the response. 193 func (obj *Client) Trace(ctx context.Context, href string, options ...RequestOption) (*Response, error) { 194 return obj.Request(ctx, http.MethodTrace, href, options...) 195 } 196 197 // Define a function named Request that takes in four parameters: 198 func (obj *Client) Request(ctx context.Context, method string, href string, options ...RequestOption) (response *Response, err error) { 199 if obj.closed { 200 return nil, errors.New("client is closed") 201 } 202 if ctx == nil { 203 ctx = obj.ctx 204 } 205 var rawOption RequestOption 206 if len(options) > 0 { 207 rawOption = options[0] 208 } 209 optionBak := obj.newRequestOption(rawOption) 210 if optionBak.Method == "" { 211 optionBak.Method = method 212 } 213 uhref := optionBak.Url 214 if uhref == nil { 215 if uhref, err = url.Parse(href); err != nil { 216 err = tools.WrapError(err, "url parse error") 217 return 218 } 219 } 220 for maxRetries := 0; maxRetries <= optionBak.MaxRetries; maxRetries++ { 221 option := optionBak 222 option.Url = cloneUrl(uhref) 223 option.client = obj 224 response, err = obj.request(ctx, &option) 225 if err == nil || errors.Is(err, errFatal) || option.once { 226 return 227 } 228 } 229 return 230 } 231 func (obj *Client) request(ctx context.Context, option *RequestOption) (response *Response, err error) { 232 response = new(Response) 233 defer func() { 234 //read body 235 if err == nil && !response.IsStream() { 236 err = response.ReadBody() 237 } 238 //result callback 239 if err == nil && option.ResultCallBack != nil { 240 err = option.ResultCallBack(ctx, option, response) 241 } 242 243 if err != nil { //err callback, must close body 244 response.CloseBody() 245 if option.ErrCallBack != nil { 246 if err2 := option.ErrCallBack(ctx, option, response, err); err2 != nil { 247 err = tools.WrapError(errFatal, err2) 248 } 249 } 250 } else if !response.IsStream() { //is not is stream must close body 251 response.CloseBody() 252 } 253 }() 254 if option.OptionCallBack != nil { 255 if err = option.OptionCallBack(ctx, option); err != nil { 256 return 257 } 258 } 259 response.bar = option.Bar 260 response.disUnzip = option.DisUnZip 261 response.disDecode = option.DisDecode 262 response.stream = option.Stream 263 264 //init headers and orderheaders,befor init ctxData 265 headers, orderHeaders, err := option.initHeaders() 266 if err != nil { 267 return response, tools.WrapError(err, errors.New("tempRequest init headers error"), err) 268 } 269 //设置 h2 请求头顺序 270 if orderHeaders != nil { 271 if !option.H2Ja3Spec.IsSet() { 272 option.H2Ja3Spec = ja3.DefaultH2Ja3Spec() 273 option.H2Ja3Spec.OrderHeaders = orderHeaders 274 } else if option.H2Ja3Spec.OrderHeaders == nil { 275 option.H2Ja3Spec.OrderHeaders = orderHeaders 276 } 277 } 278 //init ctxData 279 ctxData, err := NewReqCtxData(ctx, option) 280 if err != nil { 281 return response, tools.WrapError(err, " reqCtxData init error") 282 } 283 if headers == nil { 284 headers = defaultHeaders() 285 } 286 //设置 h1 请求头顺序 287 if orderHeaders != nil { 288 ctxData.orderHeaders = orderHeaders 289 } else { 290 ctxData.orderHeaders = ja3.DefaultOrderHeaders() 291 } 292 //init ctx,cnl 293 if option.Timeout > 0 { //超时 294 response.ctx, response.cnl = context.WithTimeout(CreateReqCtx(ctx, ctxData), option.Timeout) 295 } else { 296 response.ctx, response.cnl = context.WithCancel(CreateReqCtx(ctx, ctxData)) 297 } 298 //init url 299 href, err := option.initParams() 300 if err != nil { 301 err = tools.WrapError(err, "url init error") 302 return 303 } 304 if href.User != nil { 305 headers.Set("Authorization", "Basic "+tools.Base64Encode(href.User.String())) 306 } 307 //init body 308 body, err := option.initBody(response.ctx) 309 if err != nil { 310 return response, tools.WrapError(err, errors.New("tempRequest init body error"), err) 311 } 312 //create request 313 reqs, err := NewRequestWithContext(response.ctx, option.Method, href, body) 314 if err != nil { 315 return response, tools.WrapError(errFatal, errors.New("tempRequest 构造request失败"), err) 316 } 317 reqs.Header = headers 318 //add Referer 319 if reqs.Header.Get("Referer") == "" { 320 if option.Referer != "" { 321 reqs.Header.Set("Referer", option.Referer) 322 } else if reqs.URL.Scheme != "" && reqs.URL.Host != "" { 323 reqs.Header.Set("Referer", fmt.Sprintf("%s://%s", reqs.URL.Scheme, reqs.URL.Host)) 324 } 325 } 326 327 //set ContentType 328 if option.ContentType != "" && reqs.Header.Get("Content-Type") == "" { 329 reqs.Header.Set("Content-Type", option.ContentType) 330 } 331 332 //init ws 333 if ctxData.isWs { 334 websocket.SetClientHeadersWithOption(reqs.Header, option.WsOption) 335 } 336 337 if reqs.URL.Scheme == "file" { 338 response.filePath = re.Sub(`^/+`, "", reqs.URL.Path) 339 response.content, err = os.ReadFile(response.filePath) 340 if err != nil { 341 err = tools.WrapError(errFatal, errors.New("read filePath data error"), err) 342 } 343 return 344 } 345 //add host 346 if option.Host != "" { 347 reqs.Host = option.Host 348 } else if reqs.Header.Get("Host") != "" { 349 reqs.Host = reqs.Header.Get("Host") 350 } else { 351 reqs.Host = reqs.URL.Host 352 } 353 354 //init cookies 355 cookies, err := option.initCookies() 356 if err != nil { 357 return response, tools.WrapError(err, errors.New("tempRequest init cookies error"), err) 358 } 359 if cookies != nil { 360 addCookie(reqs, cookies) 361 } 362 //send req 363 response.response, err = obj.do(reqs, option) 364 response.isNewConn = ctxData.isNewConn 365 if err != nil { 366 err = tools.WrapError(err, "client do error") 367 return 368 } 369 if response.response == nil { 370 err = errors.New("response is nil") 371 return 372 } 373 if response.response.Body != nil { 374 response.rawConn = response.response.Body.(*readWriteCloser) 375 } 376 if !response.disUnzip { 377 response.disUnzip = response.response.Uncompressed 378 } 379 if response.response.StatusCode == 101 { 380 response.webSocket = websocket.NewClientConn(response.rawConn.Conn(), websocket.GetResponseHeaderOption(response.response.Header)) 381 } else if strings.Contains(response.response.Header.Get("Content-Type"), "text/event-stream") { 382 response.sse = newSse(response.response.Body) 383 } else if !response.disUnzip { 384 var unCompressionBody io.ReadCloser 385 unCompressionBody, err = tools.CompressionDecode(response.response.Body, response.ContentEncoding()) 386 if err != nil { 387 if err != io.ErrUnexpectedEOF && err != io.EOF { 388 return 389 } 390 } 391 if unCompressionBody != nil { 392 response.response.Body = unCompressionBody 393 } 394 } 395 return 396 }