github.com/projectdiscovery/nuclei/v2@v2.9.15/internal/runner/enumerate.go (about)

     1  package runner
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"crypto/sha1"
     7  	"encoding/base64"
     8  	"encoding/hex"
     9  	"io"
    10  	_ "net/http/pprof"
    11  	"os"
    12  	"path/filepath"
    13  	"strings"
    14  	"sync/atomic"
    15  	"time"
    16  
    17  	"github.com/klauspost/compress/zlib"
    18  	"github.com/pkg/errors"
    19  	"github.com/projectdiscovery/gologger"
    20  	"github.com/projectdiscovery/nuclei/v2/internal/runner/nucleicloud"
    21  	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/loader"
    22  	"github.com/projectdiscovery/nuclei/v2/pkg/core"
    23  	"github.com/projectdiscovery/nuclei/v2/pkg/output"
    24  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols"
    25  	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/common/contextargs"
    26  	"github.com/projectdiscovery/nuclei/v2/pkg/types"
    27  )
    28  
    29  // runStandardEnumeration runs standard enumeration
    30  func (r *Runner) runStandardEnumeration(executerOpts protocols.ExecutorOptions, store *loader.Store, engine *core.Engine) (*atomic.Bool, error) {
    31  	if r.options.AutomaticScan {
    32  		return r.executeSmartWorkflowInput(executerOpts, store, engine)
    33  	}
    34  	return r.executeTemplatesInput(store, engine)
    35  }
    36  
    37  // runCloudEnumeration runs cloud based enumeration
    38  func (r *Runner) runCloudEnumeration(store *loader.Store, cloudTemplates, cloudTargets []string, nostore bool, limit int) (*atomic.Bool, error) {
    39  	count := &atomic.Int64{}
    40  	now := time.Now()
    41  	defer func() {
    42  		gologger.Info().Msgf("Scan execution took %s and found %d results", time.Since(now), count.Load())
    43  	}()
    44  	results := &atomic.Bool{}
    45  
    46  	// TODO: Add payload file and workflow support for private templates
    47  	catalogChecksums := nucleicloud.ReadCatalogChecksum()
    48  
    49  	targets := make([]string, 0, r.hmapInputProvider.Count())
    50  	r.hmapInputProvider.Scan(func(value *contextargs.MetaInput) bool {
    51  		targets = append(targets, value.Input)
    52  		return true
    53  	})
    54  	templates := make([]string, 0, len(store.Templates()))
    55  	privateTemplates := make(map[string]string)
    56  
    57  	for _, template := range store.Templates() {
    58  		data, _ := os.ReadFile(template.Path)
    59  		h := sha1.New()
    60  		_, _ = io.Copy(h, bytes.NewReader(data))
    61  		newhash := hex.EncodeToString(h.Sum(nil))
    62  
    63  		templateRelativePath := getTemplateRelativePath(template.Path)
    64  		if hash, ok := catalogChecksums[templateRelativePath]; ok || newhash == hash {
    65  			templates = append(templates, templateRelativePath)
    66  		} else {
    67  			privateTemplates[filepath.Base(template.Path)] = gzipBase64EncodeData(data)
    68  		}
    69  	}
    70  
    71  	taskID, err := r.cloudClient.AddScan(&nucleicloud.AddScanRequest{
    72  		RawTargets:       targets,
    73  		PublicTemplates:  templates,
    74  		CloudTargets:     cloudTargets,
    75  		CloudTemplates:   cloudTemplates,
    76  		PrivateTemplates: privateTemplates,
    77  		IsTemporary:      nostore,
    78  		Filtering:        getCloudFilteringFromOptions(r.options),
    79  	})
    80  	if err != nil {
    81  		return results, err
    82  	}
    83  	gologger.Info().Msgf("Created task with ID: %d", taskID)
    84  	if nostore {
    85  		gologger.Info().Msgf("Cloud scan storage: disabled")
    86  	}
    87  	time.Sleep(3 * time.Second)
    88  
    89  	scanResponse, err := r.cloudClient.GetScan(taskID)
    90  	if err != nil {
    91  		return results, errors.Wrap(err, "could not get scan status")
    92  	}
    93  	ctx, cancel := context.WithCancel(context.Background())
    94  	defer cancel()
    95  
    96  	// Start progress logging for the created scan
    97  	if r.progress != nil {
    98  		ticker := time.NewTicker(time.Duration(r.options.StatsInterval) * time.Second)
    99  		r.progress.Init(r.hmapInputProvider.Count(), int(scanResponse.Templates), int64(scanResponse.Total))
   100  		go func() {
   101  			defer ticker.Stop()
   102  			for {
   103  				select {
   104  				case <-ctx.Done():
   105  					return
   106  				case <-ticker.C:
   107  					if scanResponse, err := r.cloudClient.GetScan(taskID); err == nil {
   108  						r.progress.SetRequests(uint64(scanResponse.Current))
   109  					}
   110  				}
   111  			}
   112  		}()
   113  	}
   114  
   115  	err = r.cloudClient.GetResults(taskID, true, limit, func(re *output.ResultEvent) {
   116  		r.progress.IncrementMatched()
   117  		results.CompareAndSwap(false, true)
   118  		_ = count.Add(1)
   119  
   120  		if outputErr := r.output.Write(re); outputErr != nil {
   121  			gologger.Warning().Msgf("Could not write output: %s", err)
   122  		}
   123  		if r.issuesClient != nil {
   124  			if err := r.issuesClient.CreateIssue(re); err != nil {
   125  				gologger.Warning().Msgf("Could not create issue on tracker: %s", err)
   126  			}
   127  		}
   128  	})
   129  	return results, err
   130  }
   131  
   132  func getTemplateRelativePath(templatePath string) string {
   133  	splitted := strings.SplitN(templatePath, "nuclei-templates", 2)
   134  	if len(splitted) < 2 {
   135  		return ""
   136  	}
   137  	return strings.TrimPrefix(splitted[1], "/")
   138  }
   139  
   140  func gzipBase64EncodeData(data []byte) string {
   141  	var buf bytes.Buffer
   142  	writer, _ := zlib.NewWriterLevel(&buf, zlib.BestCompression)
   143  	_, _ = writer.Write(data)
   144  	_ = writer.Close()
   145  	encoded := base64.StdEncoding.EncodeToString(buf.Bytes())
   146  	return encoded
   147  }
   148  
   149  func getCloudFilteringFromOptions(options *types.Options) *nucleicloud.AddScanRequestConfiguration {
   150  	return &nucleicloud.AddScanRequestConfiguration{
   151  		Authors:           options.Authors,
   152  		Tags:              options.Tags,
   153  		ExcludeTags:       options.ExcludeTags,
   154  		IncludeTags:       options.IncludeTags,
   155  		IncludeIds:        options.IncludeIds,
   156  		ExcludeIds:        options.ExcludeIds,
   157  		IncludeTemplates:  options.IncludeTemplates,
   158  		ExcludedTemplates: options.ExcludedTemplates,
   159  		ExcludeMatchers:   options.ExcludeMatchers,
   160  		Severities:        options.Severities,
   161  		ExcludeSeverities: options.ExcludeSeverities,
   162  		Protocols:         options.Protocols,
   163  		ExcludeProtocols:  options.ExcludeProtocols,
   164  		IncludeConditions: options.IncludeConditions,
   165  	}
   166  }