github.com/projectdiscovery/nuclei/v2@v2.9.15/pkg/protocols/offlinehttp/request.go (about)

     1  package offlinehttp
     2  
     3  import (
     4  	"io"
     5  	"net/http/httputil"
     6  	"os"
     7  
     8  	"github.com/pkg/errors"
     9  	"github.com/remeh/sizedwaitgroup"
    10  
    11  	"github.com/projectdiscovery/gologger"
    12  	"github.com/projectdiscovery/nuclei/v2/pkg/output"
    13  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
    14  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
    15  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/helpers/eventcreator"
    16  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/tostring"
    17  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/utils"
    18  	templateTypes "github.com/projectdiscovery/nuclei/v2/pkg/templates/types"
    19  )
    20  
    21  var _ protocols.Request = &Request{}
    22  
    23  const maxSize = 5 * 1024 * 1024
    24  
    25  // Type returns the type of the protocol request
    26  func (request *Request) Type() templateTypes.ProtocolType {
    27  	return templateTypes.OfflineHTTPProtocol
    28  }
    29  
    30  // ExecuteWithResults executes the protocol requests and returns results instead of writing them.
    31  func (request *Request) ExecuteWithResults(input *contextargs.Context, metadata /*TODO review unused parameter*/, previous output.InternalEvent, callback protocols.OutputEventCallback) error {
    32  	wg := sizedwaitgroup.New(request.options.Options.BulkSize)
    33  
    34  	err := request.getInputPaths(input.MetaInput.Input, func(data string) {
    35  		wg.Add()
    36  
    37  		go func(data string) {
    38  			defer wg.Done()
    39  
    40  			file, err := os.Open(data)
    41  			if err != nil {
    42  				gologger.Error().Msgf("Could not open file path %s: %s\n", data, err)
    43  				return
    44  			}
    45  			defer file.Close()
    46  
    47  			stat, err := file.Stat()
    48  			if err != nil {
    49  				gologger.Error().Msgf("Could not stat file path %s: %s\n", data, err)
    50  				return
    51  			}
    52  			if stat.Size() >= int64(maxSize) {
    53  				gologger.Verbose().Msgf("Could not process path %s: exceeded max size\n", data)
    54  				return
    55  			}
    56  
    57  			buffer, err := io.ReadAll(file)
    58  			if err != nil {
    59  				gologger.Error().Msgf("Could not read file path %s: %s\n", data, err)
    60  				return
    61  			}
    62  			dataStr := tostring.UnsafeToString(buffer)
    63  
    64  			resp, err := readResponseFromString(dataStr)
    65  			if err != nil {
    66  				gologger.Error().Msgf("Could not read raw response %s: %s\n", data, err)
    67  				return
    68  			}
    69  
    70  			if request.options.Options.Debug || request.options.Options.DebugRequests {
    71  				gologger.Info().Msgf("[%s] Dumped offline-http request for %s", request.options.TemplateID, data)
    72  				gologger.Print().Msgf("%s", dataStr)
    73  			}
    74  			gologger.Verbose().Msgf("[%s] Sent OFFLINE-HTTP request to %s", request.options.TemplateID, data)
    75  
    76  			dumpedResponse, err := httputil.DumpResponse(resp, true)
    77  			if err != nil {
    78  				gologger.Error().Msgf("Could not dump raw http response %s: %s\n", data, err)
    79  				return
    80  			}
    81  
    82  			body, err := io.ReadAll(resp.Body)
    83  			if err != nil {
    84  				gologger.Error().Msgf("Could not read raw http response body %s: %s\n", data, err)
    85  				return
    86  			}
    87  
    88  			outputEvent := request.responseToDSLMap(resp, data, data, data, tostring.UnsafeToString(dumpedResponse), tostring.UnsafeToString(body), utils.HeadersToString(resp.Header), 0, nil)
    89  			outputEvent["ip"] = ""
    90  			for k, v := range previous {
    91  				outputEvent[k] = v
    92  			}
    93  
    94  			event := eventcreator.CreateEvent(request, outputEvent, request.options.Options.Debug || request.options.Options.DebugResponse)
    95  			callback(event)
    96  		}(data)
    97  	})
    98  	wg.Wait()
    99  	if err != nil {
   100  		request.options.Output.Request(request.options.TemplatePath, input.MetaInput.Input, "file", err)
   101  		request.options.Progress.IncrementFailedRequestsBy(1)
   102  		return errors.Wrap(err, "could not send file request")
   103  	}
   104  	request.options.Progress.IncrementRequests()
   105  	return nil
   106  }