go.mondoo.com/cnquery@v0.0.0-20231005093811-59568235f6ea/explorer/scan/fetcher.go (about)

     1  // Copyright (c) Mondoo, Inc.
     2  // SPDX-License-Identifier: BUSL-1.1
     3  
     4  package scan
     5  
     6  import (
     7  	"context"
     8  	"errors"
     9  	"io"
    10  	"net/http"
    11  
    12  	"go.mondoo.com/cnquery"
    13  	"go.mondoo.com/cnquery/explorer"
    14  	"go.mondoo.com/cnquery/llx"
    15  	"go.mondoo.com/cnquery/utils/multierr"
    16  )
    17  
    18  type fetcher struct {
    19  	cache map[string]*explorer.Bundle
    20  }
    21  
    22  func newFetcher() *fetcher {
    23  	return &fetcher{
    24  		cache: map[string]*explorer.Bundle{},
    25  	}
    26  }
    27  
    28  func (f *fetcher) fetchBundles(ctx context.Context, schema llx.Schema, urls ...string) (*explorer.Bundle, error) {
    29  	var res *explorer.Bundle = &explorer.Bundle{}
    30  
    31  	for i := range urls {
    32  		url := urls[i]
    33  		if cur, ok := f.cache[url]; ok {
    34  			res.AddBundle(cur)
    35  			continue
    36  		}
    37  
    38  		cur, err := f.fetchBundle(url)
    39  		if err != nil {
    40  			return nil, err
    41  		}
    42  
    43  		// need to generate MRNs for everything
    44  		if _, err := cur.Compile(ctx, schema); err != nil {
    45  			return nil, multierr.Wrap(err, "failed to compile fetched bundle")
    46  		}
    47  
    48  		if err = res.AddBundle(cur); err != nil {
    49  			return nil, multierr.Wrap(err, "failed to add fetched bundle")
    50  		}
    51  	}
    52  
    53  	return res, nil
    54  }
    55  
    56  func (f *fetcher) fetchBundle(url string) (*explorer.Bundle, error) {
    57  	client := http.Client{
    58  		CheckRedirect: func(r *http.Request, via []*http.Request) error {
    59  			r.URL.Opaque = r.URL.Path
    60  			return nil
    61  		},
    62  	}
    63  
    64  	req, err := http.NewRequest("GET", url, nil)
    65  	if err != nil {
    66  		return nil, multierr.Wrap(err, "failed to set up request to fetch bundle")
    67  	}
    68  
    69  	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; cnquery/"+cnquery.Version+"; +http://www.mondoo.com)")
    70  
    71  	resp, err := client.Do(req)
    72  	if err != nil {
    73  		return nil, err
    74  	}
    75  	defer resp.Body.Close()
    76  
    77  	if resp.StatusCode != 200 {
    78  		return nil, errors.New("failed to fetch policy bundle from " + url + ": " + resp.Status)
    79  	}
    80  
    81  	raw, err := io.ReadAll(resp.Body)
    82  	if err != nil {
    83  		return nil, err
    84  	}
    85  
    86  	return explorer.BundleFromYAML(raw)
    87  }