github.com/projectdiscovery/nuclei/v2@v2.9.15/internal/installer/util.go (about)

     1  package installer
     2  
     3  import (
     4  	"bufio"
     5  	"bytes"
     6  	"fmt"
     7  	"io"
     8  	"io/fs"
     9  	"net/http"
    10  	"os"
    11  	"path/filepath"
    12  	"sort"
    13  
    14  	"github.com/Masterminds/semver/v3"
    15  	"github.com/projectdiscovery/gologger"
    16  	"github.com/projectdiscovery/nuclei/v2/pkg/catalog/config"
    17  	errorutil "github.com/projectdiscovery/utils/errors"
    18  )
    19  
    20  // GetNewTemplatesInVersions returns templates path of all newly added templates
    21  // in these versions
    22  func GetNewTemplatesInVersions(versions ...string) []string {
    23  	allTemplates := []string{}
    24  	for _, v := range versions {
    25  		if v == config.DefaultConfig.TemplateVersion {
    26  			allTemplates = append(allTemplates, config.DefaultConfig.GetNewAdditions()...)
    27  		}
    28  		_, err := semver.NewVersion(v)
    29  		if err != nil {
    30  			gologger.Error().Msgf("%v is not a valid semver version. skipping", v)
    31  			continue
    32  		}
    33  		if config.IsOutdatedVersion(v, "v8.8.4") {
    34  			// .new-additions was added in v8.8.4 any version before that is not supported
    35  			gologger.Error().Msgf(".new-additions support was added in v8.8.4 older versions are not supported")
    36  			continue
    37  		}
    38  
    39  		arr, err := getNewAdditionsFileFromGitHub(v)
    40  		if err != nil {
    41  			gologger.Error().Msgf("failed to fetch new additions for %v got: %v", v, err)
    42  			continue
    43  		}
    44  		allTemplates = append(allTemplates, arr...)
    45  	}
    46  	return allTemplates
    47  }
    48  
    49  func getNewAdditionsFileFromGitHub(version string) ([]string, error) {
    50  	resp, err := retryableHttpClient.Get(fmt.Sprintf("https://raw.githubusercontent.com/projectdiscovery/nuclei-templates/%s/.new-additions", version))
    51  	if err != nil {
    52  		return nil, err
    53  	}
    54  	if resp.StatusCode != http.StatusOK {
    55  		return nil, errorutil.New("version not found")
    56  	}
    57  	data, err := io.ReadAll(resp.Body)
    58  	if err != nil {
    59  		return nil, err
    60  	}
    61  	templatesList := []string{}
    62  	scanner := bufio.NewScanner(bytes.NewReader(data))
    63  	for scanner.Scan() {
    64  		text := scanner.Text()
    65  		if text == "" {
    66  			continue
    67  		}
    68  		if config.IsTemplate(text) {
    69  			templatesList = append(templatesList, text)
    70  		}
    71  	}
    72  	return templatesList, nil
    73  }
    74  
    75  func PurgeEmptyDirectories(dir string) {
    76  	alldirs := []string{}
    77  	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
    78  		if d.IsDir() {
    79  			alldirs = append(alldirs, path)
    80  		}
    81  		return nil
    82  	})
    83  	// sort in ascending order
    84  	sort.Strings(alldirs)
    85  	// reverse the order
    86  	sort.Sort(sort.Reverse(sort.StringSlice(alldirs)))
    87  
    88  	for _, d := range alldirs {
    89  		if isEmptyDir(d) {
    90  			_ = os.RemoveAll(d)
    91  		}
    92  	}
    93  }
    94  
    95  func isEmptyDir(dir string) bool {
    96  	hasFiles := false
    97  	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
    98  		if !d.IsDir() {
    99  			hasFiles = true
   100  			return io.EOF
   101  		}
   102  		return nil
   103  	})
   104  	return !hasFiles
   105  }
   106  
   107  // getUtmSource returns utm_source from environment variable
   108  func getUtmSource() string {
   109  	value := ""
   110  	switch {
   111  	case os.Getenv("GH_ACTION") != "":
   112  		value = "ghci"
   113  	case os.Getenv("TRAVIS") != "":
   114  		value = "travis"
   115  	case os.Getenv("CIRCLECI") != "":
   116  		value = "circleci"
   117  	case os.Getenv("CI") != "":
   118  		value = "gitlabci" // this also includes bitbucket
   119  	case os.Getenv("GITHUB_ACTIONS") != "":
   120  		value = "ghci"
   121  	case os.Getenv("AWS_EXECUTION_ENV") != "":
   122  		value = os.Getenv("AWS_EXECUTION_ENV")
   123  	case os.Getenv("JENKINS_URL") != "":
   124  		value = "jenkins"
   125  	case os.Getenv("FUNCTION_TARGET") != "":
   126  		value = "gcf"
   127  	default:
   128  		value = "unknown"
   129  	}
   130  	return value
   131  }