github.com/pelicanplatform/pelican@v1.0.5/director/advertise.go (about) 1 /*************************************************************** 2 * 3 * Copyright (C) 2023, Pelican Project, Morgridge Institute for Research 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); you 6 * may not use this file except in compliance with the License. You may 7 * obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 * 17 ***************************************************************/ 18 19 package director 20 21 import ( 22 "net/url" 23 "strings" 24 "time" 25 26 "github.com/pkg/errors" 27 log "github.com/sirupsen/logrus" 28 29 "github.com/pelicanplatform/pelican/param" 30 "github.com/pelicanplatform/pelican/utils" 31 ) 32 33 func parseServerAd(server utils.Server, serverType ServerType) ServerAd { 34 serverAd := ServerAd{} 35 serverAd.Type = serverType 36 serverAd.Name = server.Resource 37 38 // url.Parse requires that the scheme be present before the hostname, 39 // but endpoints do not have a scheme. As such, we need to add one for the. 40 // correct parsing. Luckily, we don't use this anywhere else (it's just to 41 // make the url.Parse function behave as expected) 42 if !strings.HasPrefix(server.AuthEndpoint, "http") { // just in case there's already an http(s) tacked in front 43 server.AuthEndpoint = "https://" + server.AuthEndpoint 44 } 45 if !strings.HasPrefix(server.Endpoint, "http") { // just in case there's already an http(s) tacked in front 46 server.Endpoint = "http://" + server.Endpoint 47 } 48 serverAuthUrl, err := url.Parse(server.AuthEndpoint) 49 if err != nil { 50 log.Warningf("Namespace JSON returned server %s with invalid authenticated URL %s", 51 server.Resource, server.AuthEndpoint) 52 } 53 serverAd.AuthURL = *serverAuthUrl 54 55 serverUrl, err := url.Parse(server.Endpoint) 56 if err != nil { 57 log.Warningf("Namespace JSON returned server %s with invalid unauthenticated URL %s", 58 server.Resource, server.Endpoint) 59 } 60 serverAd.URL = *serverUrl 61 62 // We will leave serverAd.WebURL as empty when fetched from topology 63 64 return serverAd 65 } 66 67 // Populate internal cache with origin/cache ads 68 func AdvertiseOSDF() error { 69 namespaces, err := utils.GetTopologyJSON() 70 if err != nil { 71 return errors.Wrapf(err, "Failed to get topology JSON") 72 } 73 74 cacheAdMap := make(map[ServerAd][]NamespaceAd) 75 originAdMap := make(map[ServerAd][]NamespaceAd) 76 for _, ns := range namespaces.Namespaces { 77 nsAd := NamespaceAd{} 78 nsAd.RequireToken = ns.UseTokenOnRead 79 nsAd.Path = ns.Path 80 nsAd.DirlistHost = ns.DirlistHost 81 issuerURL, err := url.Parse(ns.CredentialGeneration.Issuer) 82 if err != nil { 83 log.Warningf("Invalid URL %v when parsing topology response: %v\n", ns.CredentialGeneration.Issuer, err) 84 continue 85 } 86 nsAd.Issuer = *issuerURL 87 nsAd.MaxScopeDepth = uint(ns.CredentialGeneration.MaxScopeDepth) 88 nsAd.Strategy = StrategyType(ns.CredentialGeneration.Strategy) 89 nsAd.BasePath = ns.CredentialGeneration.BasePath 90 nsAd.VaultServer = ns.CredentialGeneration.VaultServer 91 92 // We assume each namespace may have multiple origins, although most likely will not 93 // Some namespaces show up in topology but don't have an origin (perhaps because 94 // they're listed as inactive by topology). These namespaces will all be mapped to the 95 // same useless origin ad, resulting in a 404 for queries to those namespaces 96 for _, origin := range ns.Origins { 97 originAd := parseServerAd(origin, OriginType) 98 originAdMap[originAd] = append(originAdMap[originAd], nsAd) 99 } 100 101 for _, cache := range ns.Caches { 102 cacheAd := parseServerAd(cache, CacheType) 103 cacheAdMap[cacheAd] = append(cacheAdMap[cacheAd], nsAd) 104 } 105 } 106 107 for originAd, namespacesSlice := range originAdMap { 108 RecordAd(originAd, &namespacesSlice) 109 } 110 111 for cacheAd, namespacesSlice := range cacheAdMap { 112 RecordAd(cacheAd, &namespacesSlice) 113 } 114 115 return nil 116 } 117 118 func PeriodicCacheReload() { 119 for { 120 // The ad cache times out every 15 minutes, so update it every 121 // 10. If a key isn't updated, it will survive for 5 minutes 122 // and then disappear 123 time.Sleep(time.Minute * param.Federation_TopologyReloadInterval.GetDuration()) 124 err := AdvertiseOSDF() 125 if err != nil { 126 log.Warningf("Failed to re-advertise: %s. Will try again later", 127 err) 128 } 129 } 130 }