storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/iam-object-store.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"encoding/json"
    23  	"errors"
    24  	"path"
    25  	"strings"
    26  	"sync"
    27  	"time"
    28  	"unicode/utf8"
    29  
    30  	"storj.io/minio/cmd/config"
    31  	"storj.io/minio/cmd/logger"
    32  	"storj.io/minio/pkg/auth"
    33  	iampolicy "storj.io/minio/pkg/iam/policy"
    34  	"storj.io/minio/pkg/kms"
    35  	"storj.io/minio/pkg/madmin"
    36  )
    37  
    38  // IAMObjectStore implements IAMStorageAPI
    39  type IAMObjectStore struct {
    40  	// Protect assignment to objAPI
    41  	sync.RWMutex
    42  
    43  	objAPI ObjectLayer
    44  }
    45  
    46  func newIAMObjectStore(objAPI ObjectLayer) *IAMObjectStore {
    47  	return &IAMObjectStore{objAPI: objAPI}
    48  }
    49  
    50  func (iamOS *IAMObjectStore) lock() {
    51  	iamOS.Lock()
    52  }
    53  
    54  func (iamOS *IAMObjectStore) unlock() {
    55  	iamOS.Unlock()
    56  }
    57  
    58  func (iamOS *IAMObjectStore) rlock() {
    59  	iamOS.RLock()
    60  }
    61  
    62  func (iamOS *IAMObjectStore) runlock() {
    63  	iamOS.RUnlock()
    64  }
    65  
    66  // Migrate users directory in a single scan.
    67  //
    68  // 1. Migrate user policy from:
    69  //
    70  // `iamConfigUsersPrefix + "<username>/policy.json"`
    71  //
    72  // to:
    73  //
    74  // `iamConfigPolicyDBUsersPrefix + "<username>.json"`.
    75  //
    76  // 2. Add versioning to the policy json file in the new
    77  // location.
    78  //
    79  // 3. Migrate user identity json file to include version info.
    80  func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context, isSTS bool) error {
    81  	basePrefix := iamConfigUsersPrefix
    82  	if isSTS {
    83  		basePrefix = iamConfigSTSPrefix
    84  	}
    85  
    86  	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix) {
    87  		if item.Err != nil {
    88  			return item.Err
    89  		}
    90  
    91  		user := path.Dir(item.Item)
    92  		{
    93  			// 1. check if there is policy file in old location.
    94  			oldPolicyPath := pathJoin(basePrefix, user, iamPolicyFile)
    95  			var policyName string
    96  			if err := iamOS.loadIAMConfig(ctx, &policyName, oldPolicyPath); err != nil {
    97  				switch err {
    98  				case errConfigNotFound:
    99  					// This case means it is already
   100  					// migrated or there is no policy on
   101  					// user.
   102  				default:
   103  					// File may be corrupt or network error
   104  				}
   105  
   106  				// Nothing to do on the policy file,
   107  				// so move on to check the id file.
   108  				goto next
   109  			}
   110  
   111  			// 2. copy policy file to new location.
   112  			mp := newMappedPolicy(policyName)
   113  			userType := regularUser
   114  			if isSTS {
   115  				userType = stsUser
   116  			}
   117  			if err := iamOS.saveMappedPolicy(ctx, user, userType, false, mp); err != nil {
   118  				return err
   119  			}
   120  
   121  			// 3. delete policy file from old
   122  			// location. Ignore error.
   123  			iamOS.deleteIAMConfig(ctx, oldPolicyPath)
   124  		}
   125  	next:
   126  		// 4. check if user identity has old format.
   127  		identityPath := pathJoin(basePrefix, user, iamIdentityFile)
   128  		var cred auth.Credentials
   129  		if err := iamOS.loadIAMConfig(ctx, &cred, identityPath); err != nil {
   130  			switch err {
   131  			case errConfigNotFound:
   132  				// This should not happen.
   133  			default:
   134  				// File may be corrupt or network error
   135  			}
   136  			continue
   137  		}
   138  
   139  		// If the file is already in the new format,
   140  		// then the parsed auth.Credentials will have
   141  		// the zero value for the struct.
   142  		var zeroCred auth.Credentials
   143  		if cred.Equal(zeroCred) {
   144  			// nothing to do
   145  			continue
   146  		}
   147  
   148  		// Found a id file in old format. Copy value
   149  		// into new format and save it.
   150  		cred.AccessKey = user
   151  		u := newUserIdentity(cred)
   152  		if err := iamOS.saveIAMConfig(ctx, u, identityPath); err != nil {
   153  			logger.LogIf(ctx, err)
   154  			return err
   155  		}
   156  
   157  		// Nothing to delete as identity file location
   158  		// has not changed.
   159  	}
   160  	return nil
   161  
   162  }
   163  
   164  func (iamOS *IAMObjectStore) migrateToV1(ctx context.Context) error {
   165  	var iamFmt iamFormat
   166  	path := getIAMFormatFilePath()
   167  	if err := iamOS.loadIAMConfig(ctx, &iamFmt, path); err != nil {
   168  		switch err {
   169  		case errConfigNotFound:
   170  			// Need to migrate to V1.
   171  		default:
   172  			return err
   173  		}
   174  	} else {
   175  		if iamFmt.Version >= iamFormatVersion1 {
   176  			// Nothing to do.
   177  			return nil
   178  		}
   179  		// This case should not happen
   180  		// (i.e. Version is 0 or negative.)
   181  		return errors.New("got an invalid IAM format version")
   182  	}
   183  
   184  	// Migrate long-term users
   185  	if err := iamOS.migrateUsersConfigToV1(ctx, false); err != nil {
   186  		logger.LogIf(ctx, err)
   187  		return err
   188  	}
   189  	// Migrate STS users
   190  	if err := iamOS.migrateUsersConfigToV1(ctx, true); err != nil {
   191  		logger.LogIf(ctx, err)
   192  		return err
   193  	}
   194  	// Save iam format to version 1.
   195  	if err := iamOS.saveIAMConfig(ctx, newIAMFormatVersion1(), path); err != nil {
   196  		logger.LogIf(ctx, err)
   197  		return err
   198  	}
   199  	return nil
   200  }
   201  
   202  // Should be called under config migration lock
   203  func (iamOS *IAMObjectStore) migrateBackendFormat(ctx context.Context) error {
   204  	return iamOS.migrateToV1(ctx)
   205  }
   206  
   207  func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}, objPath string, opts ...options) error {
   208  	data, err := json.Marshal(item)
   209  	if err != nil {
   210  		return err
   211  	}
   212  	if GlobalKMS != nil {
   213  		data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
   214  			minioMetaBucket: path.Join(minioMetaBucket, objPath),
   215  		})
   216  		if err != nil {
   217  			return err
   218  		}
   219  	}
   220  	return saveConfig(ctx, iamOS.objAPI, objPath, data)
   221  }
   222  
   223  func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item interface{}, objPath string) error {
   224  	data, err := readConfig(ctx, iamOS.objAPI, objPath)
   225  	if err != nil {
   226  		return err
   227  	}
   228  	if !utf8.Valid(data) {
   229  		if GlobalKMS != nil {
   230  			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
   231  				minioMetaBucket: path.Join(minioMetaBucket, objPath),
   232  			})
   233  			if err != nil {
   234  				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
   235  				if err != nil {
   236  					return err
   237  				}
   238  			}
   239  		} else {
   240  			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
   241  			if err != nil {
   242  				return err
   243  			}
   244  		}
   245  	}
   246  	return json.Unmarshal(data, item)
   247  }
   248  
   249  func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error {
   250  	return deleteConfig(ctx, iamOS.objAPI, path)
   251  }
   252  
   253  func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
   254  	var p iampolicy.Policy
   255  	err := iamOS.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
   256  	if err != nil {
   257  		if err == errConfigNotFound {
   258  			return errNoSuchPolicy
   259  		}
   260  		return err
   261  	}
   262  	m[policy] = p
   263  	return nil
   264  }
   265  
   266  func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
   267  	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix) {
   268  		if item.Err != nil {
   269  			return item.Err
   270  		}
   271  
   272  		policyName := path.Dir(item.Item)
   273  		if err := iamOS.loadPolicyDoc(ctx, policyName, m); err != nil && err != errNoSuchPolicy {
   274  			return err
   275  		}
   276  	}
   277  	return nil
   278  }
   279  
   280  func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
   281  	var u UserIdentity
   282  	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
   283  	if err != nil {
   284  		if err == errConfigNotFound {
   285  			return errNoSuchUser
   286  		}
   287  		return err
   288  	}
   289  
   290  	if u.Credentials.IsExpired() {
   291  		// Delete expired identity - ignoring errors here.
   292  		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
   293  		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
   294  		return nil
   295  	}
   296  
   297  	if u.Credentials.AccessKey == "" {
   298  		u.Credentials.AccessKey = user
   299  	}
   300  
   301  	m[user] = u.Credentials
   302  	return nil
   303  }
   304  
   305  func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
   306  	var basePrefix string
   307  	switch userType {
   308  	case srvAccUser:
   309  		basePrefix = iamConfigServiceAccountsPrefix
   310  	case stsUser:
   311  		basePrefix = iamConfigSTSPrefix
   312  	default:
   313  		basePrefix = iamConfigUsersPrefix
   314  	}
   315  
   316  	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix) {
   317  		if item.Err != nil {
   318  			return item.Err
   319  		}
   320  
   321  		userName := path.Dir(item.Item)
   322  		if err := iamOS.loadUser(ctx, userName, userType, m); err != nil && err != errNoSuchUser {
   323  			return err
   324  		}
   325  	}
   326  	return nil
   327  }
   328  
   329  func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error {
   330  	var g GroupInfo
   331  	err := iamOS.loadIAMConfig(ctx, &g, getGroupInfoPath(group))
   332  	if err != nil {
   333  		if err == errConfigNotFound {
   334  			return errNoSuchGroup
   335  		}
   336  		return err
   337  	}
   338  	m[group] = g
   339  	return nil
   340  }
   341  
   342  func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
   343  	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigGroupsPrefix) {
   344  		if item.Err != nil {
   345  			return item.Err
   346  		}
   347  
   348  		group := path.Dir(item.Item)
   349  		if err := iamOS.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup {
   350  			return err
   351  		}
   352  	}
   353  	return nil
   354  }
   355  
   356  func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool,
   357  	m map[string]MappedPolicy) error {
   358  
   359  	var p MappedPolicy
   360  	err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
   361  	if err != nil {
   362  		if err == errConfigNotFound {
   363  			return errNoSuchPolicy
   364  		}
   365  		return err
   366  	}
   367  	m[name] = p
   368  	return nil
   369  }
   370  
   371  func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
   372  	var basePath string
   373  	if isGroup {
   374  		basePath = iamConfigPolicyDBGroupsPrefix
   375  	} else {
   376  		switch userType {
   377  		case srvAccUser:
   378  			basePath = iamConfigPolicyDBServiceAccountsPrefix
   379  		case stsUser:
   380  			basePath = iamConfigPolicyDBSTSUsersPrefix
   381  		default:
   382  			basePath = iamConfigPolicyDBUsersPrefix
   383  		}
   384  	}
   385  	for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePath) {
   386  		if item.Err != nil {
   387  			return item.Err
   388  		}
   389  
   390  		policyFile := item.Item
   391  		userOrGroupName := strings.TrimSuffix(policyFile, ".json")
   392  		if err := iamOS.loadMappedPolicy(ctx, userOrGroupName, userType, isGroup, m); err != nil && err != errNoSuchPolicy {
   393  			return err
   394  		}
   395  	}
   396  	return nil
   397  }
   398  
   399  // Refresh IAMSys. If an object layer is passed in use that, otherwise load from global.
   400  func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
   401  	return sys.Load(ctx, iamOS)
   402  }
   403  
   404  func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
   405  	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
   406  }
   407  
   408  func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
   409  	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
   410  }
   411  
   412  func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
   413  	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
   414  }
   415  
   416  func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {
   417  	return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name))
   418  }
   419  
   420  func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error {
   421  	err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name))
   422  	if err == errConfigNotFound {
   423  		err = errNoSuchPolicy
   424  	}
   425  	return err
   426  }
   427  
   428  func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
   429  	err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup))
   430  	if err == errConfigNotFound {
   431  		err = errNoSuchPolicy
   432  	}
   433  	return err
   434  }
   435  
   436  func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
   437  	err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType))
   438  	if err == errConfigNotFound {
   439  		err = errNoSuchUser
   440  	}
   441  	return err
   442  }
   443  
   444  func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error {
   445  	err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name))
   446  	if err == errConfigNotFound {
   447  		err = errNoSuchGroup
   448  	}
   449  	return err
   450  }
   451  
   452  // helper type for listIAMConfigItems
   453  type itemOrErr struct {
   454  	Item string
   455  	Err  error
   456  }
   457  
   458  // Lists files or dirs in the minioMetaBucket at the given path
   459  // prefix. If dirs is true, only directories are listed, otherwise
   460  // only objects are listed. All returned items have the pathPrefix
   461  // removed from their names.
   462  func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr {
   463  	ch := make(chan itemOrErr)
   464  
   465  	go func() {
   466  		defer close(ch)
   467  
   468  		// Allocate new results channel to receive ObjectInfo.
   469  		objInfoCh := make(chan ObjectInfo)
   470  
   471  		if err := objAPI.Walk(ctx, minioMetaBucket, pathPrefix, objInfoCh, ObjectOptions{}); err != nil {
   472  			select {
   473  			case ch <- itemOrErr{Err: err}:
   474  			case <-ctx.Done():
   475  			}
   476  			return
   477  		}
   478  
   479  		for obj := range objInfoCh {
   480  			item := strings.TrimPrefix(obj.Name, pathPrefix)
   481  			item = strings.TrimSuffix(item, SlashSeparator)
   482  			select {
   483  			case ch <- itemOrErr{Item: item}:
   484  			case <-ctx.Done():
   485  				return
   486  			}
   487  		}
   488  	}()
   489  
   490  	return ch
   491  }
   492  
   493  func (iamOS *IAMObjectStore) watch(ctx context.Context, sys *IAMSys) {
   494  	// Refresh IAMSys.
   495  	for {
   496  		time.Sleep(globalRefreshIAMInterval)
   497  		if err := iamOS.loadAll(ctx, sys); err != nil {
   498  			logger.LogIf(ctx, err)
   499  		}
   500  	}
   501  }