github.com/pelicanplatform/pelican@v1.0.5/director/advertise.go (about)

     1  /***************************************************************
     2   *
     3   * Copyright (C) 2023, Pelican Project, Morgridge Institute for Research
     4   *
     5   * Licensed under the Apache License, Version 2.0 (the "License"); you
     6   * may not use this file except in compliance with the License.  You may
     7   * obtain a copy of the License at
     8   *
     9   *    http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   *
    17   ***************************************************************/
    18  
    19  package director
    20  
    21  import (
    22  	"net/url"
    23  	"strings"
    24  	"time"
    25  
    26  	"github.com/pkg/errors"
    27  	log "github.com/sirupsen/logrus"
    28  
    29  	"github.com/pelicanplatform/pelican/param"
    30  	"github.com/pelicanplatform/pelican/utils"
    31  )
    32  
    33  func parseServerAd(server utils.Server, serverType ServerType) ServerAd {
    34  	serverAd := ServerAd{}
    35  	serverAd.Type = serverType
    36  	serverAd.Name = server.Resource
    37  
    38  	// url.Parse requires that the scheme be present before the hostname,
    39  	// but endpoints do not have a scheme. As such, we need to add one for the.
    40  	// correct parsing. Luckily, we don't use this anywhere else (it's just to
    41  	// make the url.Parse function behave as expected)
    42  	if !strings.HasPrefix(server.AuthEndpoint, "http") { // just in case there's already an http(s) tacked in front
    43  		server.AuthEndpoint = "https://" + server.AuthEndpoint
    44  	}
    45  	if !strings.HasPrefix(server.Endpoint, "http") { // just in case there's already an http(s) tacked in front
    46  		server.Endpoint = "http://" + server.Endpoint
    47  	}
    48  	serverAuthUrl, err := url.Parse(server.AuthEndpoint)
    49  	if err != nil {
    50  		log.Warningf("Namespace JSON returned server %s with invalid authenticated URL %s",
    51  			server.Resource, server.AuthEndpoint)
    52  	}
    53  	serverAd.AuthURL = *serverAuthUrl
    54  
    55  	serverUrl, err := url.Parse(server.Endpoint)
    56  	if err != nil {
    57  		log.Warningf("Namespace JSON returned server %s with invalid unauthenticated URL %s",
    58  			server.Resource, server.Endpoint)
    59  	}
    60  	serverAd.URL = *serverUrl
    61  
    62  	// We will leave serverAd.WebURL as empty when fetched from topology
    63  
    64  	return serverAd
    65  }
    66  
    67  // Populate internal cache with origin/cache ads
    68  func AdvertiseOSDF() error {
    69  	namespaces, err := utils.GetTopologyJSON()
    70  	if err != nil {
    71  		return errors.Wrapf(err, "Failed to get topology JSON")
    72  	}
    73  
    74  	cacheAdMap := make(map[ServerAd][]NamespaceAd)
    75  	originAdMap := make(map[ServerAd][]NamespaceAd)
    76  	for _, ns := range namespaces.Namespaces {
    77  		nsAd := NamespaceAd{}
    78  		nsAd.RequireToken = ns.UseTokenOnRead
    79  		nsAd.Path = ns.Path
    80  		nsAd.DirlistHost = ns.DirlistHost
    81  		issuerURL, err := url.Parse(ns.CredentialGeneration.Issuer)
    82  		if err != nil {
    83  			log.Warningf("Invalid URL %v when parsing topology response: %v\n", ns.CredentialGeneration.Issuer, err)
    84  			continue
    85  		}
    86  		nsAd.Issuer = *issuerURL
    87  		nsAd.MaxScopeDepth = uint(ns.CredentialGeneration.MaxScopeDepth)
    88  		nsAd.Strategy = StrategyType(ns.CredentialGeneration.Strategy)
    89  		nsAd.BasePath = ns.CredentialGeneration.BasePath
    90  		nsAd.VaultServer = ns.CredentialGeneration.VaultServer
    91  
    92  		// We assume each namespace may have multiple origins, although most likely will not
    93  		// Some namespaces show up in topology but don't have an origin (perhaps because
    94  		// they're listed as inactive by topology). These namespaces will all be mapped to the
    95  		// same useless origin ad, resulting in a 404 for queries to those namespaces
    96  		for _, origin := range ns.Origins {
    97  			originAd := parseServerAd(origin, OriginType)
    98  			originAdMap[originAd] = append(originAdMap[originAd], nsAd)
    99  		}
   100  
   101  		for _, cache := range ns.Caches {
   102  			cacheAd := parseServerAd(cache, CacheType)
   103  			cacheAdMap[cacheAd] = append(cacheAdMap[cacheAd], nsAd)
   104  		}
   105  	}
   106  
   107  	for originAd, namespacesSlice := range originAdMap {
   108  		RecordAd(originAd, &namespacesSlice)
   109  	}
   110  
   111  	for cacheAd, namespacesSlice := range cacheAdMap {
   112  		RecordAd(cacheAd, &namespacesSlice)
   113  	}
   114  
   115  	return nil
   116  }
   117  
   118  func PeriodicCacheReload() {
   119  	for {
   120  		// The ad cache times out every 15 minutes, so update it every
   121  		// 10. If a key isn't updated, it will survive for 5 minutes
   122  		// and then disappear
   123  		time.Sleep(time.Minute * param.Federation_TopologyReloadInterval.GetDuration())
   124  		err := AdvertiseOSDF()
   125  		if err != nil {
   126  			log.Warningf("Failed to re-advertise: %s. Will try again later",
   127  				err)
   128  		}
   129  	}
   130  }