github.com/yandex-cloud/geesefs@v0.40.9/internal/cfg/conf_azure.go (about) 1 // Copyright 2019 Databricks 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package cfg 16 17 import ( 18 "context" 19 "fmt" 20 "net/http" 21 "net/url" 22 "os" 23 "strings" 24 "time" 25 26 "github.com/Azure/go-autorest/autorest" 27 "github.com/Azure/go-autorest/autorest/adal" 28 "github.com/Azure/go-autorest/autorest/azure" 29 "github.com/Azure/go-autorest/autorest/azure/auth" 30 "github.com/Azure/go-autorest/autorest/azure/cli" 31 32 "github.com/Azure/azure-pipeline-go/pipeline" 33 azblob "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-04-01/storage" 34 azblob2 "github.com/Azure/azure-storage-blob-go/azblob" 35 "github.com/mitchellh/go-homedir" 36 ini "gopkg.in/ini.v1" 37 ) 38 39 type SASTokenProvider func() (string, error) 40 41 type AZBlobConfig struct { 42 Endpoint string 43 AccountName string 44 AccountKey string 45 SasToken SASTokenProvider 46 TokenRenewBuffer time.Duration 47 48 Container string 49 Prefix string 50 } 51 52 func (config *AZBlobConfig) Init() { 53 config.TokenRenewBuffer = 15 * time.Minute 54 } 55 56 type returnRequestPolicy struct { 57 } 58 59 func (p returnRequestPolicy) Do(ctx context.Context, r pipeline.Request) (pipeline.Response, error) { 60 resp := pipeline.NewHTTPResponse(&http.Response{}) 61 resp.Response().Request = r.Request 62 return resp, nil 63 } 64 65 // hijack the SharedKeyCredentials signing code from azure-storage-blob-go 66 // https://github.com/Azure/go-autorest/issues/456 67 func (config *AZBlobConfig) WithAuthorization() autorest.PrepareDecorator { 68 return func(p autorest.Preparer) autorest.Preparer { 69 return autorest.PreparerFunc(func(r *http.Request) (*http.Request, error) { 70 cred, err := azblob2.NewSharedKeyCredential(config.AccountName, config.AccountKey) 71 if err != nil { 72 return nil, err 73 } 74 75 resp, err := cred.New(returnRequestPolicy{}, nil).Do(context.TODO(), 76 pipeline.Request{r}) 77 if err != nil { 78 return nil, err 79 } 80 return resp.Response().Request, nil 81 }) 82 } 83 } 84 85 type ADLv1Config struct { 86 Endpoint string 87 Authorizer autorest.Authorizer 88 } 89 90 func (config *ADLv1Config) Init() { 91 } 92 93 type ADLv2Config struct { 94 Endpoint string 95 Authorizer autorest.Authorizer 96 } 97 98 type AzureAuthorizerConfig struct { 99 Log *LogHandle 100 TenantId string 101 } 102 103 var azbLog = GetLogger("azblob") 104 var adls1Log = GetLogger("adlv1") 105 106 func sptTest(spt *adal.ServicePrincipalToken) (autorest.Authorizer, error) { 107 err := spt.EnsureFresh() 108 if err != nil { 109 return nil, err 110 } 111 112 return autorest.NewBearerAuthorizer(spt), nil 113 } 114 115 func tokenToAuthorizer(t *cli.Token) (autorest.Authorizer, error) { 116 u, err := url.Parse(t.Authority) 117 if err != nil { 118 return nil, err 119 } 120 121 tenantId := u.Path 122 u.Path = "" 123 124 oauth, err := adal.NewOAuthConfig(u.String(), tenantId) 125 if err != nil { 126 return nil, err 127 } 128 129 aToken, err := t.ToADALToken() 130 if err != nil { 131 return nil, err 132 } 133 134 spt, err := adal.NewServicePrincipalTokenFromManualToken(*oauth, t.ClientID, t.Resource, 135 aToken) 136 if err != nil { 137 return nil, err 138 } 139 140 return sptTest(spt) 141 } 142 143 func msiToAuthorizer(mc auth.MSIConfig) (autorest.Authorizer, error) { 144 // copied from azure/auth/auth.go so we can test this Authorizer 145 msiEndpoint, err := adal.GetMSIVMEndpoint() 146 if err != nil { 147 return nil, err 148 } 149 150 var spt *adal.ServicePrincipalToken 151 if mc.ClientID == "" { 152 spt, err = adal.NewServicePrincipalTokenFromMSI(msiEndpoint, mc.Resource) 153 } else { 154 spt, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, mc.Resource, mc.ClientID) 155 } 156 if err != nil { 157 return nil, err 158 } 159 160 return sptTest(spt) 161 } 162 163 func (c AzureAuthorizerConfig) Authorizer() (autorest.Authorizer, error) { 164 if c.TenantId == "" { 165 defaultSubscription, err := azureDefaultSubscription() 166 if err != nil { 167 return nil, err 168 } 169 c.TenantId = defaultSubscription.TenantID 170 } 171 172 env, err := auth.GetSettingsFromEnvironment() 173 if err != nil { 174 return nil, err 175 } 176 177 if cred, err := env.GetClientCredentials(); err == nil { 178 if authorizer, err := cred.Authorizer(); err == nil { 179 return authorizer, err 180 } 181 } 182 183 if settings, err := auth.GetSettingsFromFile(); err == nil { 184 if authorizer, err := settings.ClientCredentialsAuthorizerWithResource( 185 auth.Resource); err == nil { 186 return authorizer, err 187 } 188 } 189 190 if env.Values[auth.Resource] == "" { 191 env.Values[auth.Resource] = env.Environment.ResourceManagerEndpoint 192 } 193 if env.Values[auth.ActiveDirectoryEndpoint] == "" { 194 env.Values[auth.ActiveDirectoryEndpoint] = env.Environment.ActiveDirectoryEndpoint 195 } 196 adEndpoint := strings.Trim(env.Values[auth.ActiveDirectoryEndpoint], "/") + 197 "/" + c.TenantId 198 c.Log.Debugf("looking for access token for %v", adEndpoint) 199 200 accessTokensPath, err := cli.AccessTokensPath() 201 if err == nil { 202 accessTokens, err := cli.LoadTokens(accessTokensPath) 203 if err == nil { 204 for _, t := range accessTokens { 205 if t.Authority == adEndpoint { 206 c.Log.Debugf("found token for %v %v", t.Resource, t.Authority) 207 var authorizer autorest.Authorizer 208 authorizer, err = tokenToAuthorizer(&t) 209 if err == nil { 210 return authorizer, nil 211 } 212 } 213 } 214 } 215 if err != nil { 216 return nil, err 217 } 218 } 219 220 c.Log.Debug("falling back to MSI") 221 return msiToAuthorizer(env.GetMSI()) 222 } 223 224 func azureDefaultSubscription() (*cli.Subscription, error) { 225 profilePath, err := cli.ProfilePath() 226 if err != nil { 227 return nil, err 228 } 229 230 profile, err := cli.LoadProfile(profilePath) 231 if err != nil { 232 return nil, err 233 } 234 235 for _, s := range profile.Subscriptions { 236 if s.IsDefault { 237 return &s, nil 238 } 239 } 240 241 return nil, fmt.Errorf("Unable to find default azure subscription id") 242 } 243 244 func azureAccountsClient(account string) (azblob.AccountsClient, error) { 245 var c azblob.AccountsClient 246 247 defaultSubscription, err := azureDefaultSubscription() 248 if err != nil { 249 return c, err 250 } 251 252 c = azblob.NewAccountsClient(defaultSubscription.ID) 253 254 authorizer, err := AzureAuthorizerConfig{ 255 Log: azbLog, 256 TenantId: defaultSubscription.TenantID, 257 }.Authorizer() 258 if err != nil { 259 return c, err 260 } 261 262 c.BaseClient.Authorizer = authorizer 263 return c, nil 264 } 265 266 func azureFindAccount(client azblob.AccountsClient, account string) (*azblob.Endpoints, string, error) { 267 accountsRes, err := client.List(context.TODO()) 268 if err != nil { 269 return nil, "", err 270 } 271 272 for _, acc := range *accountsRes.Value { 273 if *acc.Name == account { 274 // /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/... 275 parts := strings.SplitN(*acc.ID, "/", 6) 276 if len(parts) != 6 { 277 return nil, "", fmt.Errorf("Malformed account id: %v", *acc.ID) 278 } 279 return acc.PrimaryEndpoints, parts[4], nil 280 } 281 } 282 283 return nil, "", fmt.Errorf("Azure account not found: %v", account) 284 } 285 286 func AzureBlobConfig(endpoint string, location string, storageType string) (config AZBlobConfig, err error) { 287 if storageType != "blob" && storageType != "dfs" { 288 panic(fmt.Sprintf("unknown storage type: %v", storageType)) 289 } 290 291 account := os.Getenv("AZURE_STORAGE_ACCOUNT") 292 key := os.Getenv("AZURE_STORAGE_KEY") 293 configDir := os.Getenv("AZURE_CONFIG_DIR") 294 295 // check if the url contains the storage endpoint 296 at := strings.Index(location, "@") 297 if at != -1 { 298 storageEndpoint := "https://" + location[at+1:] 299 u, urlErr := url.Parse(storageEndpoint) 300 if urlErr == nil { 301 // if it's valid, then it overrides --endpoint 302 endpoint = storageEndpoint 303 config.Container = location[:at] 304 config.Prefix = strings.Trim(u.Path, "/") 305 } 306 } 307 308 // parse account from endpoint 309 if endpoint != "" && endpoint != "http://127.0.0.1:8080/devstoreaccount1/" { 310 var u *url.URL 311 u, err = url.Parse(endpoint) 312 if err != nil { 313 return 314 } 315 316 dot := strings.Index(u.Hostname(), ".") 317 if dot != -1 { 318 account = u.Hostname()[:dot] 319 } 320 } 321 322 if account == "" || key == "" { 323 if configDir == "" { 324 configDir, _ = homedir.Expand("~/.azure") 325 } 326 if config, err := ini.Load(configDir + "/config"); err == nil { 327 if sect, err := config.GetSection("storage"); err == nil { 328 if account == "" { 329 if k, err := sect.GetKey("account"); err == nil { 330 account = k.Value() 331 azbLog.Debugf("Using azure account: %v", account) 332 } 333 } 334 if key == "" { 335 if k, err := sect.GetKey("key"); err == nil { 336 key = k.Value() 337 } 338 } 339 } 340 } 341 } 342 // at this point I have to have the account 343 if account == "" { 344 err = fmt.Errorf("Missing account: configure via AZURE_STORAGE_ACCOUNT "+ 345 "or %v/config", configDir) 346 return 347 } 348 349 if endpoint == "" || key == "" { 350 var client azblob.AccountsClient 351 client, err = azureAccountsClient(account) 352 if err == nil { 353 var resourceGroup string 354 var endpoints *azblob.Endpoints 355 endpoints, resourceGroup, err = azureFindAccount(client, account) 356 if err != nil { 357 if key == "" { 358 err = fmt.Errorf("Missing key: configure via AZURE_STORAGE_KEY "+ 359 "or %v/config", configDir) 360 return 361 } 362 } else { 363 if storageType == "blob" { 364 endpoint = *endpoints.Blob 365 } else if storageType == "dfs" { 366 endpoint = *endpoints.Dfs 367 } 368 } 369 azbLog.Debugf("Using detected account endpoint: %v", endpoint) 370 371 if key == "" { 372 var keysRes azblob.AccountListKeysResult 373 keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account) 374 if err != nil || len(*keysRes.Keys) == 0 { 375 err = fmt.Errorf("Missing key: configure via AZURE_STORAGE_KEY "+ 376 "or %v/config", configDir) 377 return 378 } 379 380 // prefer full permission keys 381 for _, k := range *keysRes.Keys { 382 if k.Permissions == azblob.Full { 383 key = *k.Value 384 break 385 } 386 } 387 // if not just take the first one 388 key = *(*keysRes.Keys)[0].Value 389 } 390 } else { 391 if key == "" { 392 return 393 } else { 394 // we have the credential already, we 395 // can't look up the endpoint but we 396 // can guess that 397 err = nil 398 } 399 } 400 } 401 402 if endpoint == "" { 403 endpoint = "https://" + account + "." + storageType + "." + 404 azure.PublicCloud.StorageEndpointSuffix 405 azbLog.Infof("Unable to detect endpoint for account %v, using %v", 406 account, endpoint) 407 } 408 409 config.Init() 410 config.Endpoint = endpoint 411 config.AccountName = account 412 config.AccountKey = key 413 414 return 415 }