github.com/NVIDIA/aistore@v1.3.23-0.20240517131212-7df6609be51d/cmn/cos/url.go (about)

     1  // Package cos provides common low-level types and utilities for all aistore projects
     2  /*
     3   * Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
     4   */
     5  package cos
     6  
     7  import (
     8  	"net/http"
     9  	"net/url"
    10  	"regexp"
    11  	"strings"
    12  
    13  	"github.com/NVIDIA/aistore/cmn/debug"
    14  )
    15  
    16  const (
    17  	gsStorageURL    = "storage.googleapis.com"
    18  	gsAPIURL        = "www.googleapis.com"
    19  	gsAPIPathPrefix = "/storage/v1"
    20  
    21  	s3UrlRegex = `(s3-|s3\.)?(.*)\.amazonaws\.com`
    22  
    23  	azBlobURL = ".blob.core.windows.net"
    24  )
    25  
    26  func IsHTTPS(url string) bool { return strings.HasPrefix(url, "https://") }
    27  func IsHTTP(url string) bool  { return strings.HasPrefix(url, "http://") }
    28  
    29  func ParseURL(s string) (u *url.URL, valid bool) {
    30  	if s == "" {
    31  		return
    32  	}
    33  	var err error
    34  	u, err = url.Parse(s)
    35  	valid = err == nil && u.Scheme != "" && u.Host != ""
    36  	return
    37  }
    38  
    39  func IsGoogleStorageURL(u *url.URL) bool {
    40  	return u.Host == gsStorageURL
    41  }
    42  
    43  func IsGoogleAPIURL(u *url.URL) bool {
    44  	return u.Host == gsAPIURL && strings.Contains(u.Path, gsAPIPathPrefix)
    45  }
    46  
    47  func IsS3URL(link string) bool {
    48  	re := regexp.MustCompile(s3UrlRegex)
    49  	return re.MatchString(link)
    50  }
    51  
    52  func IsAzureURL(u *url.URL) bool {
    53  	return strings.Contains(u.Host, azBlobURL)
    54  }
    55  
    56  // WARNING: `ReparseQuery` might affect non-tensorflow clients using S3-compatible API
    57  // with AIStore. To be used with caution.
    58  func ReparseQuery(r *http.Request) {
    59  	if !strings.ContainsRune(r.URL.Path, '?') {
    60  		return
    61  	}
    62  	q := r.URL.Query()
    63  	tmpURL, err := url.Parse(r.URL.Path)
    64  	debug.AssertNoErr(err)
    65  	for k, v := range tmpURL.Query() {
    66  		q.Add(k, strings.Join(v, ","))
    67  	}
    68  	r.URL.Path = tmpURL.Path
    69  	r.URL.RawQuery = q.Encode()
    70  }
    71  
    72  // JoinWords uses forward slash to join any number of words into a single path.
    73  // Returned path is prefixed with a slash.
    74  func JoinWords(w string, words ...string) (path string) {
    75  	path = w
    76  	if path[0] != '/' {
    77  		path = "/" + path
    78  	}
    79  	for _, s := range words {
    80  		path += "/" + s
    81  	}
    82  	return
    83  }
    84  
    85  // JoinPath joins two path elements that may (or may not) be prefixed/suffixed with a slash.
    86  func JoinPath(url, path string) string {
    87  	suffix := IsLastB(url, '/')
    88  	prefix := path[0] == '/'
    89  	if suffix && prefix {
    90  		return url + path[1:]
    91  	}
    92  	if !suffix && !prefix {
    93  		return url + "/" + path
    94  	}
    95  	return url + path
    96  }