github.com/replit/upm@v0.0.0-20240423230255-9ce4fc3ea24c/internal/backends/python/gen_pypi_map/dist_reader.go (about)

     1  package main
     2  
     3  import (
     4  	"fmt"
     5  	"net/http"
     6  	"path"
     7  	"sort"
     8  )
     9  
    10  // These modules are always ignored
    11  var ignoredModules = map[string]bool{
    12  	"test":          true,
    13  	"tests":         true,
    14  	"testing":       true,
    15  	"doc":           true,
    16  	"docs":          true,
    17  	"documentation": true,
    18  	"_private":      true,
    19  	"setup":         true,
    20  }
    21  
    22  func GetModules(metadata PackageData) ([]string, error) {
    23  	fmt.Println("GetModules")
    24  	latest := metadata.Releases[metadata.Info.Version]
    25  	if len(latest) == 0 {
    26  		return nil, PypiError{NoDistributions, metadata.Info.Version, nil}
    27  	}
    28  
    29  	// Sort the releases by priority we want to parse
    30  	distPriorities := map[string]int{
    31  		"bdist_wheel": 1,
    32  		"sdist":       2,
    33  	}
    34  
    35  	sort.Slice(latest, func(a, b int) bool {
    36  		return distPriorities[latest[a].PackageType] < distPriorities[latest[b].PackageType]
    37  	})
    38  
    39  	pkg := latest[0]
    40  
    41  	// Download the distribution
    42  	resp, err := http.Get(pkg.URL)
    43  	if err != nil {
    44  		return nil, err
    45  	}
    46  	defer resp.Body.Close()
    47  
    48  	// Get the right archive reader
    49  	var reader ArchiveReader
    50  	if path.Ext(pkg.Filename) == ".zip" ||
    51  		path.Ext(pkg.Filename) == ".whl" ||
    52  		path.Ext(pkg.Filename) == ".egg" {
    53  		reader, err = MakeZipReader(resp.Body, pkg.Size)
    54  		if err != nil {
    55  			return nil, err
    56  		}
    57  	} else if path.Ext(pkg.Filename) == ".gz" {
    58  		reader, err = MakeTarballReader(resp.Body)
    59  		if err != nil {
    60  			return nil, err
    61  		}
    62  	} else {
    63  		return nil, PypiError{UnknownArchive, path.Ext(pkg.Filename), nil}
    64  	}
    65  	defer reader.Close()
    66  
    67  	var modules []string
    68  	switch pkg.PackageType {
    69  	case "bdist_wheel":
    70  		modules, err = ExtractBdist(reader)
    71  	case "bdist_egg":
    72  		modules, err = ExtractBdist(reader)
    73  	case "bdist_dumb":
    74  		modules, err = ExtractBdist(reader)
    75  	case "sdist":
    76  		modules, err = ExtractSdist(reader)
    77  	default:
    78  		return nil, PypiError{UnknownDist, pkg.PackageType, nil}
    79  	}
    80  
    81  	if err != nil {
    82  		return nil, err
    83  	}
    84  
    85  	// Filter out modules that aren't useful
    86  	ret := make([]string, 0, len(modules))
    87  	for _, m := range modules {
    88  		// Only add top level modules, sub modules aren't useful for guessing
    89  		if path.Dir(m) != "." {
    90  			continue
    91  		}
    92  
    93  		// Skip modules that are common in many packages
    94  		if _, hit := ignoredModules[path.Base(m)]; hit {
    95  			continue
    96  		}
    97  
    98  		ret = append(ret, m)
    99  	}
   100  
   101  	return ret, nil
   102  }