storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/admin-handlers.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"crypto/subtle"
    22  	"crypto/tls"
    23  	"encoding/json"
    24  	"errors"
    25  	"fmt"
    26  	"io"
    27  	"math/rand"
    28  	"net/http"
    29  	"net/url"
    30  	"os"
    31  	"path"
    32  	"runtime"
    33  	"sort"
    34  	"strconv"
    35  	"strings"
    36  	"time"
    37  
    38  	"github.com/gorilla/mux"
    39  
    40  	"storj.io/minio/cmd/config"
    41  	"storj.io/minio/cmd/crypto"
    42  	xhttp "storj.io/minio/cmd/http"
    43  	"storj.io/minio/cmd/logger"
    44  	"storj.io/minio/cmd/logger/message/log"
    45  	"storj.io/minio/pkg/auth"
    46  	"storj.io/minio/pkg/bandwidth"
    47  	"storj.io/minio/pkg/dsync"
    48  	"storj.io/minio/pkg/handlers"
    49  	iampolicy "storj.io/minio/pkg/iam/policy"
    50  	"storj.io/minio/pkg/kms"
    51  	"storj.io/minio/pkg/madmin"
    52  	xnet "storj.io/minio/pkg/net"
    53  	trace "storj.io/minio/pkg/trace"
    54  )
    55  
    56  const (
    57  	maxEConfigJSONSize = 262272
    58  )
    59  
    60  // Only valid query params for mgmt admin APIs.
    61  const (
    62  	mgmtBucket      = "bucket"
    63  	mgmtPrefix      = "prefix"
    64  	mgmtClientToken = "clientToken"
    65  	mgmtForceStart  = "forceStart"
    66  	mgmtForceStop   = "forceStop"
    67  )
    68  
    69  func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) {
    70  	if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil {
    71  		return us, err
    72  	}
    73  
    74  	us.CurrentVersion = Version
    75  	us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout)
    76  	return us, nil
    77  }
    78  
    79  // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
    80  // ----------
    81  // updates all minio servers and restarts them gracefully.
    82  func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
    83  	ctx := NewContext(r, w, "ServerUpdate")
    84  
    85  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
    86  
    87  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
    88  	if objectAPI == nil {
    89  		return
    90  	}
    91  
    92  	if globalInplaceUpdateDisabled {
    93  		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
    94  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
    95  		return
    96  	}
    97  
    98  	vars := mux.Vars(r)
    99  	updateURL := vars["updateURL"]
   100  	mode := getMinioMode()
   101  	if updateURL == "" {
   102  		updateURL = minioReleaseInfoURL
   103  		if runtime.GOOS == globalWindowsOSName {
   104  			updateURL = minioReleaseWindowsInfoURL
   105  		}
   106  	}
   107  
   108  	u, err := url.Parse(updateURL)
   109  	if err != nil {
   110  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   111  		return
   112  	}
   113  
   114  	content, err := downloadReleaseURL(u, updateTimeout, mode)
   115  	if err != nil {
   116  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   117  		return
   118  	}
   119  
   120  	sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content)
   121  	if err != nil {
   122  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   123  		return
   124  	}
   125  
   126  	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
   127  	crTime, err := GetCurrentReleaseTime()
   128  	if err != nil {
   129  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   130  		return
   131  	}
   132  
   133  	if lrTime.Sub(crTime) <= 0 {
   134  		updateStatus := madmin.ServerUpdateStatus{
   135  			CurrentVersion: Version,
   136  			UpdatedVersion: Version,
   137  		}
   138  
   139  		// Marshal API response
   140  		jsonBytes, err := json.Marshal(updateStatus)
   141  		if err != nil {
   142  			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   143  			return
   144  		}
   145  
   146  		writeSuccessResponseJSON(w, jsonBytes)
   147  		return
   148  	}
   149  
   150  	for _, nerr := range GlobalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
   151  		if nerr.Err != nil {
   152  			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
   153  			logger.LogIf(ctx, nerr.Err)
   154  			err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err)
   155  			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   156  			return
   157  		}
   158  	}
   159  
   160  	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
   161  	if err != nil {
   162  		err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err)
   163  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   164  		return
   165  	}
   166  
   167  	// Marshal API response
   168  	jsonBytes, err := json.Marshal(updateStatus)
   169  	if err != nil {
   170  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   171  		return
   172  	}
   173  
   174  	writeSuccessResponseJSON(w, jsonBytes)
   175  
   176  	// Notify all other MinIO peers signal service.
   177  	for _, nerr := range GlobalNotificationSys.SignalService(serviceRestart) {
   178  		if nerr.Err != nil {
   179  			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
   180  			logger.LogIf(ctx, nerr.Err)
   181  		}
   182  	}
   183  
   184  	globalServiceSignalCh <- serviceRestart
   185  }
   186  
   187  // ServiceHandler - POST /minio/admin/v3/service?action={action}
   188  // ----------
   189  // restarts/stops minio server gracefully. In a distributed setup,
   190  func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
   191  	ctx := NewContext(r, w, "Service")
   192  
   193  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   194  
   195  	vars := mux.Vars(r)
   196  	action := vars["action"]
   197  
   198  	var serviceSig serviceSignal
   199  	switch madmin.ServiceAction(action) {
   200  	case madmin.ServiceActionRestart:
   201  		serviceSig = serviceRestart
   202  	case madmin.ServiceActionStop:
   203  		serviceSig = serviceStop
   204  	default:
   205  		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.Application)
   206  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
   207  		return
   208  	}
   209  
   210  	var objectAPI ObjectLayer
   211  	switch serviceSig {
   212  	case serviceRestart:
   213  		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
   214  	case serviceStop:
   215  		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
   216  	}
   217  	if objectAPI == nil {
   218  		return
   219  	}
   220  
   221  	// Notify all other MinIO peers signal service.
   222  	for _, nerr := range GlobalNotificationSys.SignalService(serviceSig) {
   223  		if nerr.Err != nil {
   224  			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
   225  			logger.LogIf(ctx, nerr.Err)
   226  		}
   227  	}
   228  
   229  	// Reply to the client before restarting, stopping MinIO server.
   230  	writeSuccessResponseHeadersOnly(w)
   231  
   232  	globalServiceSignalCh <- serviceSig
   233  }
   234  
   235  // ServerProperties holds some server information such as, version, region
   236  // uptime, etc..
   237  type ServerProperties struct {
   238  	Uptime       int64    `json:"uptime"`
   239  	Version      string   `json:"version"`
   240  	CommitID     string   `json:"commitID"`
   241  	DeploymentID string   `json:"deploymentID"`
   242  	Region       string   `json:"region"`
   243  	SQSARN       []string `json:"sqsARN"`
   244  }
   245  
   246  // ServerConnStats holds transferred bytes from/to the server
   247  type ServerConnStats struct {
   248  	TotalInputBytes  uint64 `json:"transferred"`
   249  	TotalOutputBytes uint64 `json:"received"`
   250  	Throughput       uint64 `json:"throughput,omitempty"`
   251  	S3InputBytes     uint64 `json:"transferredS3"`
   252  	S3OutputBytes    uint64 `json:"receivedS3"`
   253  }
   254  
   255  // ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
   256  // including the average duration the call was spent.
   257  type ServerHTTPAPIStats struct {
   258  	APIStats map[string]int `json:"apiStats"`
   259  }
   260  
   261  // ServerHTTPStats holds all type of http operations performed to/from the server
   262  // including their average execution time.
   263  type ServerHTTPStats struct {
   264  	S3RequestsInQueue      int32              `json:"s3RequestsInQueue"`
   265  	CurrentS3Requests      ServerHTTPAPIStats `json:"currentS3Requests"`
   266  	TotalS3Requests        ServerHTTPAPIStats `json:"totalS3Requests"`
   267  	TotalS3Errors          ServerHTTPAPIStats `json:"totalS3Errors"`
   268  	TotalS3Canceled        ServerHTTPAPIStats `json:"totalS3Canceled"`
   269  	TotalS3RejectedAuth    uint64             `json:"totalS3RejectedAuth"`
   270  	TotalS3RejectedTime    uint64             `json:"totalS3RejectedTime"`
   271  	TotalS3RejectedHeader  uint64             `json:"totalS3RejectedHeader"`
   272  	TotalS3RejectedInvalid uint64             `json:"totalS3RejectedInvalid"`
   273  }
   274  
   275  // StorageInfoHandler - GET /minio/admin/v3/storageinfo
   276  // ----------
   277  // Get server information
   278  func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
   279  	ctx := NewContext(r, w, "StorageInfo")
   280  
   281  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   282  
   283  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
   284  	if objectAPI == nil {
   285  		return
   286  	}
   287  
   288  	// ignores any errors here.
   289  	storageInfo, _ := objectAPI.StorageInfo(ctx)
   290  
   291  	// Collect any disk healing.
   292  	healing, _ := getAggregatedBackgroundHealState(ctx, nil)
   293  	healDisks := make(map[string]struct{}, len(healing.HealDisks))
   294  	for _, disk := range healing.HealDisks {
   295  		healDisks[disk] = struct{}{}
   296  	}
   297  
   298  	// find all disks which belong to each respective endpoints
   299  	for i, disk := range storageInfo.Disks {
   300  		if _, ok := healDisks[disk.Endpoint]; ok {
   301  			storageInfo.Disks[i].Healing = true
   302  		}
   303  	}
   304  
   305  	// Marshal API response
   306  	jsonBytes, err := json.Marshal(storageInfo)
   307  	if err != nil {
   308  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   309  		return
   310  	}
   311  
   312  	// Reply with storage information (across nodes in a
   313  	// distributed setup) as json.
   314  	writeSuccessResponseJSON(w, jsonBytes)
   315  
   316  }
   317  
   318  // DataUsageInfoHandler - GET /minio/admin/v3/datausage
   319  // ----------
   320  // Get server/cluster data usage info
   321  func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
   322  	ctx := NewContext(r, w, "DataUsageInfo")
   323  
   324  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   325  
   326  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
   327  	if objectAPI == nil {
   328  		return
   329  	}
   330  
   331  	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
   332  	if err != nil {
   333  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   334  		return
   335  	}
   336  
   337  	dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
   338  	if err != nil {
   339  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   340  		return
   341  	}
   342  
   343  	writeSuccessResponseJSON(w, dataUsageInfoJSON)
   344  }
   345  
   346  func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
   347  	entry := &madmin.LockEntry{
   348  		Timestamp:  l.Timestamp,
   349  		Resource:   resource,
   350  		ServerList: []string{server},
   351  		Source:     l.Source,
   352  		Owner:      l.Owner,
   353  		ID:         l.UID,
   354  		Quorum:     l.Quorum,
   355  	}
   356  	if l.Writer {
   357  		entry.Type = "WRITE"
   358  	} else {
   359  		entry.Type = "READ"
   360  	}
   361  	return entry
   362  }
   363  
   364  func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
   365  	entryMap := make(map[string]*madmin.LockEntry)
   366  	for _, peerLock := range peerLocks {
   367  		if peerLock == nil {
   368  			continue
   369  		}
   370  		for k, v := range peerLock.Locks {
   371  			for _, lockReqInfo := range v {
   372  				if val, ok := entryMap[lockReqInfo.UID]; ok {
   373  					val.ServerList = append(val.ServerList, peerLock.Addr)
   374  				} else {
   375  					entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
   376  				}
   377  			}
   378  		}
   379  	}
   380  	var lockEntries madmin.LockEntries
   381  	for _, v := range entryMap {
   382  		if stale {
   383  			lockEntries = append(lockEntries, *v)
   384  			continue
   385  		}
   386  		if len(v.ServerList) >= v.Quorum {
   387  			lockEntries = append(lockEntries, *v)
   388  		}
   389  	}
   390  	sort.Sort(lockEntries)
   391  	return lockEntries
   392  }
   393  
   394  // PeerLocks holds server information result of one node
   395  type PeerLocks struct {
   396  	Addr  string
   397  	Locks map[string][]lockRequesterInfo
   398  }
   399  
   400  // ForceUnlockHandler force unlocks requested resource
   401  func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
   402  	ctx := NewContext(r, w, "ForceUnlock")
   403  
   404  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   405  
   406  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
   407  	if objectAPI == nil {
   408  		return
   409  	}
   410  
   411  	z, ok := objectAPI.(*erasureServerPools)
   412  	if !ok {
   413  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
   414  		return
   415  	}
   416  
   417  	vars := mux.Vars(r)
   418  
   419  	var args dsync.LockArgs
   420  	lockersMap := make(map[string]dsync.NetLocker)
   421  	for _, path := range strings.Split(vars["paths"], ",") {
   422  		if path == "" {
   423  			continue
   424  		}
   425  		args.Resources = append(args.Resources, path)
   426  		lockers, _ := z.serverPools[0].getHashedSet(path).getLockers()
   427  		for _, locker := range lockers {
   428  			if locker != nil {
   429  				lockersMap[locker.String()] = locker
   430  			}
   431  		}
   432  	}
   433  
   434  	for _, locker := range lockersMap {
   435  		locker.ForceUnlock(ctx, args)
   436  	}
   437  }
   438  
   439  // TopLocksHandler Get list of locks in use
   440  func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
   441  	ctx := NewContext(r, w, "TopLocks")
   442  
   443  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   444  
   445  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
   446  	if objectAPI == nil {
   447  		return
   448  	}
   449  
   450  	count := 10 // by default list only top 10 entries
   451  	if countStr := r.URL.Query().Get("count"); countStr != "" {
   452  		var err error
   453  		count, err = strconv.Atoi(countStr)
   454  		if err != nil {
   455  			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   456  			return
   457  		}
   458  	}
   459  	stale := r.URL.Query().Get("stale") == "true" // list also stale locks
   460  
   461  	peerLocks := GlobalNotificationSys.GetLocks(ctx, r)
   462  
   463  	topLocks := topLockEntries(peerLocks, stale)
   464  
   465  	// Marshal API response upto requested count.
   466  	if len(topLocks) > count && count > 0 {
   467  		topLocks = topLocks[:count]
   468  	}
   469  
   470  	jsonBytes, err := json.Marshal(topLocks)
   471  	if err != nil {
   472  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   473  		return
   474  	}
   475  
   476  	// Reply with storage information (across nodes in a
   477  	// distributed setup) as json.
   478  	writeSuccessResponseJSON(w, jsonBytes)
   479  }
   480  
   481  // StartProfilingResult contains the status of the starting
   482  // profiling action in a given server
   483  type StartProfilingResult struct {
   484  	NodeName string `json:"nodeName"`
   485  	Success  bool   `json:"success"`
   486  	Error    string `json:"error"`
   487  }
   488  
   489  // StartProfilingHandler - POST /minio/admin/v3/profiling/start?profilerType={profilerType}
   490  // ----------
   491  // Enable server profiling
   492  func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
   493  	ctx := NewContext(r, w, "StartProfiling")
   494  
   495  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   496  
   497  	// Validate request signature.
   498  	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
   499  	if adminAPIErr != ErrNone {
   500  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
   501  		return
   502  	}
   503  
   504  	if GlobalNotificationSys == nil {
   505  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
   506  		return
   507  	}
   508  
   509  	vars := mux.Vars(r)
   510  	profiles := strings.Split(vars["profilerType"], ",")
   511  	thisAddr, err := xnet.ParseHost(globalLocalNodeName)
   512  	if err != nil {
   513  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   514  		return
   515  	}
   516  
   517  	globalProfilerMu.Lock()
   518  	defer globalProfilerMu.Unlock()
   519  
   520  	if globalProfiler == nil {
   521  		globalProfiler = make(map[string]minioProfiler, 10)
   522  	}
   523  
   524  	// Stop profiler of all types if already running
   525  	for k, v := range globalProfiler {
   526  		for _, p := range profiles {
   527  			if p == k {
   528  				v.Stop()
   529  				delete(globalProfiler, k)
   530  			}
   531  		}
   532  	}
   533  
   534  	// Start profiling on remote servers.
   535  	var hostErrs []NotificationPeerErr
   536  	for _, profiler := range profiles {
   537  		hostErrs = append(hostErrs, GlobalNotificationSys.StartProfiling(profiler)...)
   538  
   539  		// Start profiling locally as well.
   540  		prof, err := startProfiler(profiler)
   541  		if err != nil {
   542  			hostErrs = append(hostErrs, NotificationPeerErr{
   543  				Host: *thisAddr,
   544  				Err:  err,
   545  			})
   546  		} else {
   547  			globalProfiler[profiler] = prof
   548  			hostErrs = append(hostErrs, NotificationPeerErr{
   549  				Host: *thisAddr,
   550  			})
   551  		}
   552  	}
   553  
   554  	var startProfilingResult []StartProfilingResult
   555  
   556  	for _, nerr := range hostErrs {
   557  		result := StartProfilingResult{NodeName: nerr.Host.String()}
   558  		if nerr.Err != nil {
   559  			result.Error = nerr.Err.Error()
   560  		} else {
   561  			result.Success = true
   562  		}
   563  		startProfilingResult = append(startProfilingResult, result)
   564  	}
   565  
   566  	// Create JSON result and send it to the client
   567  	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
   568  	if err != nil {
   569  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   570  		return
   571  	}
   572  
   573  	writeSuccessResponseJSON(w, startProfilingResultInBytes)
   574  }
   575  
   576  // dummyFileInfo represents a dummy representation of a profile data file
   577  // present only in memory, it helps to generate the zip stream.
   578  type dummyFileInfo struct {
   579  	name    string
   580  	size    int64
   581  	mode    os.FileMode
   582  	modTime time.Time
   583  	isDir   bool
   584  	sys     interface{}
   585  }
   586  
   587  func (f dummyFileInfo) Name() string       { return f.name }
   588  func (f dummyFileInfo) Size() int64        { return f.size }
   589  func (f dummyFileInfo) Mode() os.FileMode  { return f.mode }
   590  func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
   591  func (f dummyFileInfo) IsDir() bool        { return f.isDir }
   592  func (f dummyFileInfo) Sys() interface{}   { return f.sys }
   593  
   594  // DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
   595  // ----------
   596  // Download profiling information of all nodes in a zip format
   597  func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
   598  	ctx := NewContext(r, w, "DownloadProfiling")
   599  
   600  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   601  
   602  	// Validate request signature.
   603  	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
   604  	if adminAPIErr != ErrNone {
   605  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
   606  		return
   607  	}
   608  
   609  	if GlobalNotificationSys == nil {
   610  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
   611  		return
   612  	}
   613  
   614  	if !GlobalNotificationSys.DownloadProfilingData(ctx, w) {
   615  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL)
   616  		return
   617  	}
   618  }
   619  
   620  type healInitParams struct {
   621  	bucket, objPrefix     string
   622  	hs                    madmin.HealOpts
   623  	clientToken           string
   624  	forceStart, forceStop bool
   625  }
   626  
   627  // extractHealInitParams - Validates params for heal init API.
   628  func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
   629  	hip.bucket = vars[mgmtBucket]
   630  	hip.objPrefix = vars[mgmtPrefix]
   631  
   632  	if hip.bucket == "" {
   633  		if hip.objPrefix != "" {
   634  			// Bucket is required if object-prefix is given
   635  			err = ErrHealMissingBucket
   636  			return
   637  		}
   638  	} else if isReservedOrInvalidBucket(hip.bucket, false) {
   639  		err = ErrInvalidBucketName
   640  		return
   641  	}
   642  
   643  	// empty prefix is valid.
   644  	if !IsValidObjectPrefix(hip.objPrefix) {
   645  		err = ErrInvalidObjectName
   646  		return
   647  	}
   648  
   649  	if len(qParms[mgmtClientToken]) > 0 {
   650  		hip.clientToken = qParms[mgmtClientToken][0]
   651  	}
   652  	if _, ok := qParms[mgmtForceStart]; ok {
   653  		hip.forceStart = true
   654  	}
   655  	if _, ok := qParms[mgmtForceStop]; ok {
   656  		hip.forceStop = true
   657  	}
   658  
   659  	// Invalid request conditions:
   660  	//
   661  	//   Cannot have both forceStart and forceStop in the same
   662  	//   request; If clientToken is provided, request can only be
   663  	//   to continue receiving logs, so it cannot be start or
   664  	//   stop;
   665  	if (hip.forceStart && hip.forceStop) ||
   666  		(hip.clientToken != "" && (hip.forceStart || hip.forceStop)) {
   667  		err = ErrInvalidRequest
   668  		return
   669  	}
   670  
   671  	// ignore body if clientToken is provided
   672  	if hip.clientToken == "" {
   673  		jerr := json.NewDecoder(r).Decode(&hip.hs)
   674  		if jerr != nil {
   675  			logger.LogIf(GlobalContext, jerr, logger.Application)
   676  			err = ErrRequestBodyParse
   677  			return
   678  		}
   679  	}
   680  
   681  	err = ErrNone
   682  	return
   683  }
   684  
   685  // HealHandler - POST /minio/admin/v3/heal/
   686  // -----------
   687  // Start heal processing and return heal status items.
   688  //
   689  // On a successful heal sequence start, a unique client token is
   690  // returned. Subsequent requests to this endpoint providing the client
   691  // token will receive heal status records from the running heal
   692  // sequence.
   693  //
   694  // If no client token is provided, and a heal sequence is in progress
   695  // an error is returned with information about the running heal
   696  // sequence. However, if the force-start flag is provided, the server
   697  // aborts the running heal sequence and starts a new one.
   698  func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
   699  	ctx := NewContext(r, w, "Heal")
   700  
   701  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   702  
   703  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
   704  	if objectAPI == nil {
   705  		return
   706  	}
   707  
   708  	// Check if this setup has an erasure coded backend.
   709  	if !globalIsErasure {
   710  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
   711  		return
   712  	}
   713  
   714  	hip, errCode := extractHealInitParams(mux.Vars(r), r.URL.Query(), r.Body)
   715  	if errCode != ErrNone {
   716  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
   717  		return
   718  	}
   719  
   720  	// Analyze the heal token and route the request accordingly
   721  	token, success := proxyRequestByToken(ctx, w, r, hip.clientToken)
   722  	if success {
   723  		return
   724  	}
   725  	hip.clientToken = token
   726  	// if request was not successful, try this server locally if token
   727  	// is not found the call will fail anyways. if token is empty
   728  	// try this server to generate a new token.
   729  
   730  	type healResp struct {
   731  		respBytes []byte
   732  		apiErr    APIError
   733  		errBody   string
   734  	}
   735  
   736  	// Define a closure to start sending whitespace to client
   737  	// after 10s unless a response item comes in
   738  	keepConnLive := func(w http.ResponseWriter, r *http.Request, respCh chan healResp) {
   739  		ticker := time.NewTicker(time.Second * 10)
   740  		defer ticker.Stop()
   741  		started := false
   742  	forLoop:
   743  		for {
   744  			select {
   745  			case <-r.Context().Done():
   746  				return
   747  			case <-ticker.C:
   748  				if !started {
   749  					// Start writing response to client
   750  					started = true
   751  					setCommonHeaders(w)
   752  					setEventStreamHeaders(w)
   753  					// Set 200 OK status
   754  					w.WriteHeader(200)
   755  				}
   756  				// Send whitespace and keep connection open
   757  				w.Write([]byte(" "))
   758  				w.(http.Flusher).Flush()
   759  			case hr := <-respCh:
   760  				switch hr.apiErr {
   761  				case noError:
   762  					if started {
   763  						w.Write(hr.respBytes)
   764  						w.(http.Flusher).Flush()
   765  					} else {
   766  						writeSuccessResponseJSON(w, hr.respBytes)
   767  					}
   768  				default:
   769  					var errorRespJSON []byte
   770  					if hr.errBody == "" {
   771  						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
   772  							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
   773  							globalDeploymentID))
   774  					} else {
   775  						errorRespJSON = encodeResponseJSON(APIErrorResponse{
   776  							Code:      hr.apiErr.Code,
   777  							Message:   hr.errBody,
   778  							Resource:  r.URL.Path,
   779  							RequestID: w.Header().Get(xhttp.AmzRequestID),
   780  							HostID:    globalDeploymentID,
   781  						})
   782  					}
   783  					if !started {
   784  						setCommonHeaders(w)
   785  						w.Header().Set(xhttp.ContentType, string(mimeJSON))
   786  						w.WriteHeader(hr.apiErr.HTTPStatusCode)
   787  					}
   788  					w.Write(errorRespJSON)
   789  					w.(http.Flusher).Flush()
   790  				}
   791  				break forLoop
   792  			}
   793  		}
   794  	}
   795  
   796  	healPath := pathJoin(hip.bucket, hip.objPrefix)
   797  	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
   798  		nh, exists := globalAllHealState.getHealSequence(healPath)
   799  		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
   800  			clientToken := nh.clientToken
   801  			if globalIsDistErasure {
   802  				clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
   803  			}
   804  			b, err := json.Marshal(madmin.HealStartSuccess{
   805  				ClientToken:   clientToken,
   806  				ClientAddress: nh.clientAddress,
   807  				StartTime:     nh.startTime,
   808  			})
   809  			if err != nil {
   810  				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   811  				return
   812  			}
   813  			// Client token not specified but a heal sequence exists on a path,
   814  			// Send the token back to client.
   815  			writeSuccessResponseJSON(w, b)
   816  			return
   817  		}
   818  	}
   819  
   820  	if hip.clientToken != "" && !hip.forceStart && !hip.forceStop {
   821  		// Since clientToken is given, fetch heal status from running
   822  		// heal sequence.
   823  		respBytes, errCode := globalAllHealState.PopHealStatusJSON(
   824  			healPath, hip.clientToken)
   825  		if errCode != ErrNone {
   826  			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
   827  		} else {
   828  			writeSuccessResponseJSON(w, respBytes)
   829  		}
   830  		return
   831  	}
   832  
   833  	respCh := make(chan healResp)
   834  	switch {
   835  	case hip.forceStop:
   836  		go func() {
   837  			respBytes, apiErr := globalAllHealState.stopHealSequence(healPath)
   838  			hr := healResp{respBytes: respBytes, apiErr: apiErr}
   839  			respCh <- hr
   840  		}()
   841  	case hip.clientToken == "":
   842  		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
   843  		go func() {
   844  			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI)
   845  			hr := healResp{respBytes, apiErr, errMsg}
   846  			respCh <- hr
   847  		}()
   848  	}
   849  
   850  	// Due to the force-starting functionality, the Launch
   851  	// call above can take a long time - to keep the
   852  	// connection alive, we start sending whitespace
   853  	keepConnLive(w, r, respCh)
   854  }
   855  
   856  // getAggregatedBackgroundHealState returns the heal state of disks.
   857  // If no ObjectLayer is provided no set status is returned.
   858  func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmin.BgHealState, error) {
   859  	// Get local heal status first
   860  	bgHealStates, ok := getBackgroundHealStatus(ctx, o)
   861  	if !ok {
   862  		return bgHealStates, errServerNotInitialized
   863  	}
   864  
   865  	if globalIsDistErasure {
   866  		// Get heal status from other peers
   867  		peersHealStates, nerrs := GlobalNotificationSys.BackgroundHealStatus()
   868  		var errCount int
   869  		for _, nerr := range nerrs {
   870  			if nerr.Err != nil {
   871  				logger.LogIf(ctx, nerr.Err)
   872  				errCount++
   873  			}
   874  		}
   875  		if errCount == len(nerrs) {
   876  			return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy")
   877  		}
   878  		bgHealStates.Merge(peersHealStates...)
   879  	}
   880  
   881  	return bgHealStates, nil
   882  }
   883  
   884  func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
   885  	ctx := NewContext(r, w, "HealBackgroundStatus")
   886  
   887  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   888  
   889  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
   890  	if objectAPI == nil {
   891  		return
   892  	}
   893  
   894  	// Check if this setup has an erasure coded backend.
   895  	if !globalIsErasure {
   896  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
   897  		return
   898  	}
   899  
   900  	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI)
   901  	if err != nil {
   902  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   903  		return
   904  	}
   905  
   906  	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
   907  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   908  		return
   909  	}
   910  
   911  	w.(http.Flusher).Flush()
   912  }
   913  
   914  func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
   915  	var cred auth.Credentials
   916  	var adminAPIErr APIErrorCode
   917  	// Get current object layer instance.
   918  	objectAPI := newObjectLayerFn()
   919  	if objectAPI == nil || GlobalNotificationSys == nil {
   920  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
   921  		return nil, cred
   922  	}
   923  
   924  	// Validate request signature.
   925  	cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
   926  	if adminAPIErr != ErrNone {
   927  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
   928  		return nil, cred
   929  	}
   930  
   931  	return objectAPI, cred
   932  }
   933  
   934  // AdminError - is a generic error for all admin APIs.
   935  type AdminError struct {
   936  	Code       string
   937  	Message    string
   938  	StatusCode int
   939  }
   940  
   941  func (ae AdminError) Error() string {
   942  	return ae.Message
   943  }
   944  
   945  // Admin API errors
   946  const (
   947  	AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure"
   948  	AdminUpdateURLNotReachable   = "XMinioAdminUpdateURLNotReachable"
   949  	AdminUpdateApplyFailure      = "XMinioAdminUpdateApplyFailure"
   950  )
   951  
   952  // toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
   953  // specific error.
   954  func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
   955  	switch err {
   956  	case errErasureWriteQuorum:
   957  		return ErrAdminConfigNoQuorum
   958  	default:
   959  		return toAPIErrorCode(ctx, err)
   960  	}
   961  }
   962  
   963  func toAdminAPIErr(ctx context.Context, err error) APIError {
   964  	if err == nil {
   965  		return noError
   966  	}
   967  
   968  	var apiErr APIError
   969  	switch e := err.(type) {
   970  	case iampolicy.Error:
   971  		apiErr = APIError{
   972  			Code:           "XMinioMalformedIAMPolicy",
   973  			Description:    e.Error(),
   974  			HTTPStatusCode: http.StatusBadRequest,
   975  		}
   976  	case config.Error:
   977  		apiErr = APIError{
   978  			Code:           "XMinioConfigError",
   979  			Description:    e.Error(),
   980  			HTTPStatusCode: http.StatusBadRequest,
   981  		}
   982  	case AdminError:
   983  		apiErr = APIError{
   984  			Code:           e.Code,
   985  			Description:    e.Message,
   986  			HTTPStatusCode: e.StatusCode,
   987  		}
   988  	default:
   989  		switch {
   990  		case errors.Is(err, errConfigNotFound):
   991  			apiErr = APIError{
   992  				Code:           "XMinioConfigError",
   993  				Description:    err.Error(),
   994  				HTTPStatusCode: http.StatusNotFound,
   995  			}
   996  		case errors.Is(err, errIAMActionNotAllowed):
   997  			apiErr = APIError{
   998  				Code:           "XMinioIAMActionNotAllowed",
   999  				Description:    err.Error(),
  1000  				HTTPStatusCode: http.StatusForbidden,
  1001  			}
  1002  		case errors.Is(err, errIAMNotInitialized):
  1003  			apiErr = APIError{
  1004  				Code:           "XMinioIAMNotInitialized",
  1005  				Description:    err.Error(),
  1006  				HTTPStatusCode: http.StatusServiceUnavailable,
  1007  			}
  1008  		case errors.Is(err, crypto.ErrKESKeyExists):
  1009  			apiErr = APIError{
  1010  				Code:           "XMinioKMSKeyExists",
  1011  				Description:    err.Error(),
  1012  				HTTPStatusCode: http.StatusConflict,
  1013  			}
  1014  		default:
  1015  			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
  1016  		}
  1017  	}
  1018  	return apiErr
  1019  }
  1020  
  1021  // Returns true if the trace.Info should be traced,
  1022  // false if certain conditions are not met.
  1023  // - input entry is not of the type *trace.Info*
  1024  // - errOnly entries are to be traced, not status code 2xx, 3xx.
  1025  // - trace.Info type is asked by opts
  1026  func mustTrace(entry interface{}, opts madmin.ServiceTraceOpts) (shouldTrace bool) {
  1027  	trcInfo, ok := entry.(trace.Info)
  1028  	if !ok {
  1029  		return false
  1030  	}
  1031  
  1032  	// Override shouldTrace decision with errOnly filtering
  1033  	defer func() {
  1034  		if shouldTrace && opts.OnlyErrors {
  1035  			shouldTrace = trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
  1036  		}
  1037  	}()
  1038  
  1039  	if opts.Threshold > 0 {
  1040  		var latency time.Duration
  1041  		switch trcInfo.TraceType {
  1042  		case trace.OS:
  1043  			latency = trcInfo.OSStats.Duration
  1044  		case trace.Storage:
  1045  			latency = trcInfo.StorageStats.Duration
  1046  		case trace.HTTP:
  1047  			latency = trcInfo.CallStats.Latency
  1048  		}
  1049  		if latency < opts.Threshold {
  1050  			return false
  1051  		}
  1052  	}
  1053  
  1054  	if opts.Internal && trcInfo.TraceType == trace.HTTP && HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
  1055  		return true
  1056  	}
  1057  
  1058  	if opts.S3 && trcInfo.TraceType == trace.HTTP && !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) {
  1059  		return true
  1060  	}
  1061  
  1062  	if opts.Storage && trcInfo.TraceType == trace.Storage {
  1063  		return true
  1064  	}
  1065  
  1066  	return opts.OS && trcInfo.TraceType == trace.OS
  1067  }
  1068  
  1069  func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) {
  1070  	q := r.URL.Query()
  1071  
  1072  	opts.OnlyErrors = q.Get("err") == "true"
  1073  	opts.S3 = q.Get("s3") == "true"
  1074  	opts.Internal = q.Get("internal") == "true"
  1075  	opts.Storage = q.Get("storage") == "true"
  1076  	opts.OS = q.Get("os") == "true"
  1077  
  1078  	// Support deprecated 'all' query
  1079  	if q.Get("all") == "true" {
  1080  		opts.S3 = true
  1081  		opts.Internal = true
  1082  		opts.Storage = true
  1083  		opts.OS = true
  1084  	}
  1085  
  1086  	if t := q.Get("threshold"); t != "" {
  1087  		d, err := time.ParseDuration(t)
  1088  		if err != nil {
  1089  			return opts, err
  1090  		}
  1091  		opts.Threshold = d
  1092  	}
  1093  	return
  1094  }
  1095  
  1096  // TraceHandler - POST /minio/admin/v3/trace
  1097  // ----------
  1098  // The handler sends http trace to the connected HTTP client.
  1099  func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
  1100  	ctx := NewContext(r, w, "HTTPTrace")
  1101  
  1102  	// Validate request signature.
  1103  	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
  1104  	if adminAPIErr != ErrNone {
  1105  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
  1106  		return
  1107  	}
  1108  
  1109  	traceOpts, err := extractTraceOptions(r)
  1110  	if err != nil {
  1111  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
  1112  		return
  1113  	}
  1114  
  1115  	setEventStreamHeaders(w)
  1116  
  1117  	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
  1118  	// Use buffered channel to take care of burst sends or slow w.Write()
  1119  	traceCh := make(chan interface{}, 4000)
  1120  
  1121  	peers, _ := newPeerRestClients(globalEndpoints)
  1122  
  1123  	globalTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
  1124  		return mustTrace(entry, traceOpts)
  1125  	})
  1126  
  1127  	for _, peer := range peers {
  1128  		if peer == nil {
  1129  			continue
  1130  		}
  1131  		peer.Trace(traceCh, ctx.Done(), traceOpts)
  1132  	}
  1133  
  1134  	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
  1135  	defer keepAliveTicker.Stop()
  1136  
  1137  	enc := json.NewEncoder(w)
  1138  	for {
  1139  		select {
  1140  		case entry := <-traceCh:
  1141  			if err := enc.Encode(entry); err != nil {
  1142  				return
  1143  			}
  1144  			w.(http.Flusher).Flush()
  1145  		case <-keepAliveTicker.C:
  1146  			if _, err := w.Write([]byte(" ")); err != nil {
  1147  				return
  1148  			}
  1149  			w.(http.Flusher).Flush()
  1150  		case <-ctx.Done():
  1151  			return
  1152  		}
  1153  	}
  1154  }
  1155  
  1156  // The handler sends console logs to the connected HTTP client.
  1157  func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
  1158  	ctx := NewContext(r, w, "ConsoleLog")
  1159  
  1160  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1161  
  1162  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
  1163  	if objectAPI == nil {
  1164  		return
  1165  	}
  1166  	node := r.URL.Query().Get("node")
  1167  	// limit buffered console entries if client requested it.
  1168  	limitStr := r.URL.Query().Get("limit")
  1169  	limitLines, err := strconv.Atoi(limitStr)
  1170  	if err != nil {
  1171  		limitLines = 10
  1172  	}
  1173  
  1174  	logKind := r.URL.Query().Get("logType")
  1175  	if logKind == "" {
  1176  		logKind = string(logger.All)
  1177  	}
  1178  	logKind = strings.ToUpper(logKind)
  1179  
  1180  	// Avoid reusing tcp connection if read timeout is hit
  1181  	// This is needed to make r.Context().Done() work as
  1182  	// expected in case of read timeout
  1183  	w.Header().Set("Connection", "close")
  1184  
  1185  	setEventStreamHeaders(w)
  1186  
  1187  	logCh := make(chan interface{}, 4000)
  1188  
  1189  	peers, _ := newPeerRestClients(globalEndpoints)
  1190  
  1191  	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
  1192  
  1193  	for _, peer := range peers {
  1194  		if peer == nil {
  1195  			continue
  1196  		}
  1197  		if node == "" || strings.EqualFold(peer.host.Name, node) {
  1198  			peer.ConsoleLog(logCh, ctx.Done())
  1199  		}
  1200  	}
  1201  
  1202  	enc := json.NewEncoder(w)
  1203  
  1204  	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
  1205  	defer keepAliveTicker.Stop()
  1206  
  1207  	for {
  1208  		select {
  1209  		case entry := <-logCh:
  1210  			log, ok := entry.(log.Info)
  1211  			if ok && log.SendLog(node, logKind) {
  1212  				if err := enc.Encode(log); err != nil {
  1213  					return
  1214  				}
  1215  				w.(http.Flusher).Flush()
  1216  			}
  1217  		case <-keepAliveTicker.C:
  1218  			if _, err := w.Write([]byte(" ")); err != nil {
  1219  				return
  1220  			}
  1221  			w.(http.Flusher).Flush()
  1222  		case <-ctx.Done():
  1223  			return
  1224  		}
  1225  	}
  1226  }
  1227  
  1228  // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
  1229  func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
  1230  	ctx := NewContext(r, w, "KMSCreateKey")
  1231  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1232  
  1233  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
  1234  	if objectAPI == nil {
  1235  		return
  1236  	}
  1237  
  1238  	if GlobalKMS == nil {
  1239  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
  1240  		return
  1241  	}
  1242  
  1243  	if err := GlobalKMS.CreateKey(r.URL.Query().Get("key-id")); err != nil {
  1244  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
  1245  		return
  1246  	}
  1247  	writeSuccessResponseHeadersOnly(w)
  1248  }
  1249  
  1250  // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
  1251  func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
  1252  	ctx := NewContext(r, w, "KMSKeyStatus")
  1253  
  1254  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1255  
  1256  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
  1257  	if objectAPI == nil {
  1258  		return
  1259  	}
  1260  
  1261  	if GlobalKMS == nil {
  1262  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
  1263  		return
  1264  	}
  1265  	stat, err := GlobalKMS.Stat()
  1266  	if err != nil {
  1267  		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
  1268  		return
  1269  	}
  1270  
  1271  	keyID := r.URL.Query().Get("key-id")
  1272  	if keyID == "" {
  1273  		keyID = stat.DefaultKey
  1274  	}
  1275  	var response = madmin.KMSKeyStatus{
  1276  		KeyID: keyID,
  1277  	}
  1278  
  1279  	kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
  1280  	// 1. Generate a new key using the KMS.
  1281  	key, err := GlobalKMS.GenerateKey(keyID, kmsContext)
  1282  	if err != nil {
  1283  		response.EncryptionErr = err.Error()
  1284  		resp, err := json.Marshal(response)
  1285  		if err != nil {
  1286  			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
  1287  			return
  1288  		}
  1289  		writeSuccessResponseJSON(w, resp)
  1290  		return
  1291  	}
  1292  
  1293  	// 2. Verify that we can indeed decrypt the (encrypted) key
  1294  	decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Plaintext, kmsContext)
  1295  	if err != nil {
  1296  		response.DecryptionErr = err.Error()
  1297  		resp, err := json.Marshal(response)
  1298  		if err != nil {
  1299  			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
  1300  			return
  1301  		}
  1302  		writeSuccessResponseJSON(w, resp)
  1303  		return
  1304  	}
  1305  
  1306  	// 3. Compare generated key with decrypted key
  1307  	if subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1 {
  1308  		response.DecryptionErr = "The generated and the decrypted data key do not match"
  1309  		resp, err := json.Marshal(response)
  1310  		if err != nil {
  1311  			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
  1312  			return
  1313  		}
  1314  		writeSuccessResponseJSON(w, resp)
  1315  		return
  1316  	}
  1317  
  1318  	resp, err := json.Marshal(response)
  1319  	if err != nil {
  1320  		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
  1321  		return
  1322  	}
  1323  	writeSuccessResponseJSON(w, resp)
  1324  }
  1325  
  1326  // HealthInfoHandler - GET /minio/admin/v3/healthinfo
  1327  // ----------
  1328  // Get server health info
  1329  func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
  1330  	ctx := NewContext(r, w, "HealthInfo")
  1331  
  1332  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1333  
  1334  	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
  1335  	if objectAPI == nil {
  1336  		return
  1337  	}
  1338  
  1339  	query := r.URL.Query()
  1340  	healthInfo := madmin.HealthInfo{}
  1341  	healthInfoCh := make(chan madmin.HealthInfo)
  1342  
  1343  	enc := json.NewEncoder(w)
  1344  	partialWrite := func(oinfo madmin.HealthInfo) {
  1345  		healthInfoCh <- oinfo
  1346  	}
  1347  
  1348  	setCommonHeaders(w)
  1349  
  1350  	setEventStreamHeaders(w)
  1351  
  1352  	w.WriteHeader(http.StatusOK)
  1353  
  1354  	errResp := func(err error) {
  1355  		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
  1356  			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
  1357  		encodedErrorResponse := EncodeResponse(errorResponse)
  1358  		healthInfo.Error = string(encodedErrorResponse)
  1359  		logger.LogIf(ctx, enc.Encode(healthInfo))
  1360  	}
  1361  
  1362  	deadline := 3600 * time.Second
  1363  	if dstr := r.URL.Query().Get("deadline"); dstr != "" {
  1364  		var err error
  1365  		deadline, err = time.ParseDuration(dstr)
  1366  		if err != nil {
  1367  			errResp(err)
  1368  			return
  1369  		}
  1370  	}
  1371  
  1372  	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
  1373  	defer cancel()
  1374  
  1375  	var err error
  1376  	nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
  1377  	ctx, err = nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
  1378  	if err != nil { // returns a locked lock
  1379  		errResp(err)
  1380  		return
  1381  	}
  1382  	defer nsLock.Unlock()
  1383  
  1384  	go func() {
  1385  		defer close(healthInfoCh)
  1386  
  1387  		if cpu := query.Get("syscpu"); cpu == "true" {
  1388  			cpuInfo := getLocalCPUInfo(deadlinedCtx, r)
  1389  
  1390  			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo)
  1391  			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, GlobalNotificationSys.CPUInfo(deadlinedCtx)...)
  1392  			partialWrite(healthInfo)
  1393  		}
  1394  
  1395  		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
  1396  			diskHwInfo := getLocalDiskHwInfo(deadlinedCtx, r)
  1397  
  1398  			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, diskHwInfo)
  1399  			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, GlobalNotificationSys.DiskHwInfo(deadlinedCtx)...)
  1400  			partialWrite(healthInfo)
  1401  		}
  1402  
  1403  		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
  1404  			osInfo := getLocalOsInfo(deadlinedCtx, r)
  1405  
  1406  			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, osInfo)
  1407  			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, GlobalNotificationSys.OsInfo(deadlinedCtx)...)
  1408  			partialWrite(healthInfo)
  1409  		}
  1410  
  1411  		if mem := query.Get("sysmem"); mem == "true" {
  1412  			memInfo := getLocalMemInfo(deadlinedCtx, r)
  1413  
  1414  			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, memInfo)
  1415  			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, GlobalNotificationSys.MemInfo(deadlinedCtx)...)
  1416  			partialWrite(healthInfo)
  1417  		}
  1418  
  1419  		if proc := query.Get("sysprocess"); proc == "true" {
  1420  			procInfo := getLocalProcInfo(deadlinedCtx, r)
  1421  
  1422  			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, procInfo)
  1423  			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, GlobalNotificationSys.ProcInfo(deadlinedCtx)...)
  1424  			partialWrite(healthInfo)
  1425  		}
  1426  
  1427  		if config := query.Get("minioconfig"); config == "true" {
  1428  			cfg, err := readServerConfig(ctx, objectAPI)
  1429  			logger.LogIf(ctx, err)
  1430  			healthInfo.Minio.Config = cfg
  1431  			partialWrite(healthInfo)
  1432  		}
  1433  
  1434  		if drive := query.Get("perfdrive"); drive == "true" {
  1435  			// Get drive perf details from local server's drive(s)
  1436  			drivePerfSerial := getLocalDrives(deadlinedCtx, false, globalEndpoints, r)
  1437  			drivePerfParallel := getLocalDrives(deadlinedCtx, true, globalEndpoints, r)
  1438  
  1439  			errStr := ""
  1440  			if drivePerfSerial.Error != "" {
  1441  				errStr = "serial: " + drivePerfSerial.Error
  1442  			}
  1443  			if drivePerfParallel.Error != "" {
  1444  				errStr = errStr + " parallel: " + drivePerfParallel.Error
  1445  			}
  1446  
  1447  			driveInfo := madmin.ServerDrivesInfo{
  1448  				Addr:     drivePerfSerial.Addr,
  1449  				Serial:   drivePerfSerial.Serial,
  1450  				Parallel: drivePerfParallel.Parallel,
  1451  				Error:    errStr,
  1452  			}
  1453  			healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, driveInfo)
  1454  			partialWrite(healthInfo)
  1455  
  1456  			// Notify all other MinIO peers to report drive perf numbers
  1457  			driveInfos := GlobalNotificationSys.DrivePerfInfoChan(deadlinedCtx)
  1458  			for obd := range driveInfos {
  1459  				healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, obd)
  1460  				partialWrite(healthInfo)
  1461  			}
  1462  			partialWrite(healthInfo)
  1463  		}
  1464  
  1465  		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
  1466  			healthInfo.Perf.Net = append(healthInfo.Perf.Net, GlobalNotificationSys.NetInfo(deadlinedCtx))
  1467  			partialWrite(healthInfo)
  1468  
  1469  			netInfos := GlobalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
  1470  			for netInfo := range netInfos {
  1471  				healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
  1472  				partialWrite(healthInfo)
  1473  			}
  1474  			partialWrite(healthInfo)
  1475  
  1476  			healthInfo.Perf.NetParallel = GlobalNotificationSys.NetPerfParallelInfo(deadlinedCtx)
  1477  			partialWrite(healthInfo)
  1478  		}
  1479  
  1480  	}()
  1481  
  1482  	ticker := time.NewTicker(5 * time.Second)
  1483  	defer ticker.Stop()
  1484  
  1485  	for {
  1486  		select {
  1487  		case oinfo, ok := <-healthInfoCh:
  1488  			if !ok {
  1489  				return
  1490  			}
  1491  			logger.LogIf(ctx, enc.Encode(oinfo))
  1492  			w.(http.Flusher).Flush()
  1493  		case <-ticker.C:
  1494  			if _, err := w.Write([]byte(" ")); err != nil {
  1495  				return
  1496  			}
  1497  			w.(http.Flusher).Flush()
  1498  		case <-deadlinedCtx.Done():
  1499  			w.(http.Flusher).Flush()
  1500  			return
  1501  		}
  1502  	}
  1503  
  1504  }
  1505  
  1506  // BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
  1507  // ----------
  1508  // Get bandwidth consumption information
  1509  func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
  1510  	ctx := NewContext(r, w, "BandwidthMonitor")
  1511  
  1512  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1513  
  1514  	// Validate request signature.
  1515  	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
  1516  	if adminAPIErr != ErrNone {
  1517  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
  1518  		return
  1519  	}
  1520  
  1521  	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
  1522  
  1523  	setEventStreamHeaders(w)
  1524  	reportCh := make(chan bandwidth.Report)
  1525  	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
  1526  	defer keepAliveTicker.Stop()
  1527  	bucketsRequestedString := r.URL.Query().Get("buckets")
  1528  	bucketsRequested := strings.Split(bucketsRequestedString, ",")
  1529  	go func() {
  1530  		defer close(reportCh)
  1531  		for {
  1532  			select {
  1533  			case <-ctx.Done():
  1534  				return
  1535  			case reportCh <- GlobalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...):
  1536  				time.Sleep(time.Duration(rnd.Float64() * float64(2*time.Second)))
  1537  			}
  1538  		}
  1539  	}()
  1540  	for {
  1541  		select {
  1542  		case report, ok := <-reportCh:
  1543  			if !ok {
  1544  				return
  1545  			}
  1546  			if err := json.NewEncoder(w).Encode(report); err != nil {
  1547  				writeErrorResponseJSON(ctx, w, ToAPIError(ctx, err), r.URL)
  1548  				return
  1549  			}
  1550  			w.(http.Flusher).Flush()
  1551  		case <-keepAliveTicker.C:
  1552  			if _, err := w.Write([]byte(" ")); err != nil {
  1553  				return
  1554  			}
  1555  			w.(http.Flusher).Flush()
  1556  		case <-ctx.Done():
  1557  			return
  1558  		}
  1559  	}
  1560  }
  1561  
  1562  // ServerInfoHandler - GET /minio/admin/v3/info
  1563  // ----------
  1564  // Get server information
  1565  func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
  1566  	ctx := NewContext(r, w, "ServerInfo")
  1567  
  1568  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
  1569  
  1570  	// Validate request signature.
  1571  	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
  1572  	if adminAPIErr != ErrNone {
  1573  		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
  1574  		return
  1575  	}
  1576  
  1577  	kmsStat := fetchKMSStatus()
  1578  
  1579  	ldap := madmin.LDAP{}
  1580  	if globalLDAPConfig.Enabled {
  1581  		ldapConn, err := globalLDAPConfig.Connect()
  1582  		if err != nil {
  1583  			ldap.Status = string(madmin.ItemOffline)
  1584  		} else if ldapConn == nil {
  1585  			ldap.Status = "Not Configured"
  1586  		} else {
  1587  			// Close ldap connection to avoid leaks.
  1588  			ldapConn.Close()
  1589  			ldap.Status = string(madmin.ItemOnline)
  1590  		}
  1591  	}
  1592  
  1593  	log, audit := fetchLoggerInfo()
  1594  
  1595  	// Get the notification target info
  1596  	notifyTarget := fetchLambdaInfo()
  1597  
  1598  	local := getLocalServerProperty(globalEndpoints, r)
  1599  	servers := GlobalNotificationSys.ServerInfo()
  1600  	servers = append(servers, local)
  1601  
  1602  	assignPoolNumbers(servers)
  1603  
  1604  	var backend interface{}
  1605  	mode := madmin.ItemInitializing
  1606  
  1607  	buckets := madmin.Buckets{}
  1608  	objects := madmin.Objects{}
  1609  	usage := madmin.Usage{}
  1610  
  1611  	objectAPI := newObjectLayerFn()
  1612  	if objectAPI != nil {
  1613  		mode = madmin.ItemOnline
  1614  
  1615  		// Load data usage
  1616  		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
  1617  		if err == nil {
  1618  			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
  1619  			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
  1620  			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
  1621  		} else {
  1622  			buckets = madmin.Buckets{Error: err.Error()}
  1623  			objects = madmin.Objects{Error: err.Error()}
  1624  			usage = madmin.Usage{Error: err.Error()}
  1625  		}
  1626  
  1627  		// Fetching the backend information
  1628  		backendInfo := objectAPI.BackendInfo()
  1629  		if backendInfo.Type == madmin.Erasure {
  1630  			// Calculate the number of online/offline disks of all nodes
  1631  			var allDisks []madmin.Disk
  1632  			for _, s := range servers {
  1633  				allDisks = append(allDisks, s.Disks...)
  1634  			}
  1635  			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
  1636  
  1637  			backend = madmin.ErasureBackend{
  1638  				Type:             madmin.ErasureType,
  1639  				OnlineDisks:      onlineDisks.Sum(),
  1640  				OfflineDisks:     offlineDisks.Sum(),
  1641  				StandardSCParity: backendInfo.StandardSCParity,
  1642  				RRSCParity:       backendInfo.RRSCParity,
  1643  			}
  1644  		} else {
  1645  			backend = madmin.FSBackend{
  1646  				Type: madmin.FsType,
  1647  			}
  1648  		}
  1649  	}
  1650  
  1651  	domain := globalDomainNames
  1652  	services := madmin.Services{
  1653  		KMS:           kmsStat,
  1654  		LDAP:          ldap,
  1655  		Logger:        log,
  1656  		Audit:         audit,
  1657  		Notifications: notifyTarget,
  1658  	}
  1659  
  1660  	infoMsg := madmin.InfoMessage{
  1661  		Mode:         string(mode),
  1662  		Domain:       domain,
  1663  		Region:       globalServerRegion,
  1664  		SQSARN:       GlobalNotificationSys.GetARNList(false),
  1665  		DeploymentID: globalDeploymentID,
  1666  		Buckets:      buckets,
  1667  		Objects:      objects,
  1668  		Usage:        usage,
  1669  		Services:     services,
  1670  		Backend:      backend,
  1671  		Servers:      servers,
  1672  	}
  1673  
  1674  	// Marshal API response
  1675  	jsonBytes, err := json.Marshal(infoMsg)
  1676  	if err != nil {
  1677  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
  1678  		return
  1679  	}
  1680  
  1681  	// Reply with storage information (across nodes in a
  1682  	// distributed setup) as json.
  1683  	writeSuccessResponseJSON(w, jsonBytes)
  1684  }
  1685  
  1686  func assignPoolNumbers(servers []madmin.ServerProperties) {
  1687  	for i := range servers {
  1688  		for idx, ge := range globalEndpoints {
  1689  			for _, endpoint := range ge.Endpoints {
  1690  				if servers[i].Endpoint == endpoint.Host {
  1691  					servers[i].PoolNumber = idx + 1
  1692  				} else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil {
  1693  					if host.Name == endpoint.Hostname() {
  1694  						servers[i].PoolNumber = idx + 1
  1695  					}
  1696  				}
  1697  			}
  1698  		}
  1699  	}
  1700  }
  1701  
  1702  func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
  1703  
  1704  	lambdaMap := make(map[string][]madmin.TargetIDStatus)
  1705  
  1706  	for _, tgt := range globalConfigTargetList.Targets() {
  1707  		targetIDStatus := make(map[string]madmin.Status)
  1708  		active, _ := tgt.IsActive()
  1709  		targetID := tgt.ID()
  1710  		if active {
  1711  			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
  1712  		} else {
  1713  			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
  1714  		}
  1715  		list := lambdaMap[targetID.Name]
  1716  		list = append(list, targetIDStatus)
  1717  		lambdaMap[targetID.Name] = list
  1718  	}
  1719  
  1720  	for _, tgt := range globalEnvTargetList.Targets() {
  1721  		targetIDStatus := make(map[string]madmin.Status)
  1722  		active, _ := tgt.IsActive()
  1723  		targetID := tgt.ID()
  1724  		if active {
  1725  			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)}
  1726  		} else {
  1727  			targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)}
  1728  		}
  1729  		list := lambdaMap[targetID.Name]
  1730  		list = append(list, targetIDStatus)
  1731  		lambdaMap[targetID.Name] = list
  1732  	}
  1733  
  1734  	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
  1735  	counter := 0
  1736  	for key, value := range lambdaMap {
  1737  		v := make(map[string][]madmin.TargetIDStatus)
  1738  		v[key] = value
  1739  		notify[counter] = v
  1740  		counter++
  1741  	}
  1742  	return notify
  1743  }
  1744  
  1745  // fetchKMSStatus fetches KMS-related status information.
  1746  func fetchKMSStatus() madmin.KMS {
  1747  	kmsStat := madmin.KMS{}
  1748  	if GlobalKMS == nil {
  1749  		kmsStat.Status = "disabled"
  1750  		return kmsStat
  1751  	}
  1752  
  1753  	stat, err := GlobalKMS.Stat()
  1754  	if err != nil {
  1755  		kmsStat.Status = string(madmin.ItemOffline)
  1756  		return kmsStat
  1757  	}
  1758  	if len(stat.Endpoints) == 0 {
  1759  		kmsStat.Status = stat.Name
  1760  		return kmsStat
  1761  	}
  1762  	if err := checkConnection(stat.Endpoints[0], 15*time.Second); err != nil {
  1763  		kmsStat.Status = string(madmin.ItemOffline)
  1764  		return kmsStat
  1765  	}
  1766  	kmsStat.Status = string(madmin.ItemOnline)
  1767  
  1768  	kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation
  1769  	// 1. Generate a new key using the KMS.
  1770  	key, err := GlobalKMS.GenerateKey("", kmsContext)
  1771  	if err != nil {
  1772  		kmsStat.Encrypt = fmt.Sprintf("Encryption failed: %v", err)
  1773  	} else {
  1774  		kmsStat.Encrypt = "success"
  1775  	}
  1776  
  1777  	// 2. Verify that we can indeed decrypt the (encrypted) key
  1778  	decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Ciphertext, kmsContext)
  1779  	switch {
  1780  	case err != nil:
  1781  		kmsStat.Decrypt = fmt.Sprintf("Decryption failed: %v", err)
  1782  	case subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1:
  1783  		kmsStat.Decrypt = "Decryption failed: decrypted key does not match generated key"
  1784  	default:
  1785  		kmsStat.Decrypt = "success"
  1786  	}
  1787  	return kmsStat
  1788  }
  1789  
  1790  // fetchLoggerDetails return log info
  1791  func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
  1792  	var loggerInfo []madmin.Logger
  1793  	var auditloggerInfo []madmin.Audit
  1794  	for _, target := range logger.Targets {
  1795  		if target.Endpoint() != "" {
  1796  			tgt := target.String()
  1797  			err := checkConnection(target.Endpoint(), 15*time.Second)
  1798  			if err == nil {
  1799  				mapLog := make(map[string]madmin.Status)
  1800  				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
  1801  				loggerInfo = append(loggerInfo, mapLog)
  1802  			} else {
  1803  				mapLog := make(map[string]madmin.Status)
  1804  				mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
  1805  				loggerInfo = append(loggerInfo, mapLog)
  1806  			}
  1807  		}
  1808  	}
  1809  
  1810  	for _, target := range logger.AuditTargets {
  1811  		if target.Endpoint() != "" {
  1812  			tgt := target.String()
  1813  			err := checkConnection(target.Endpoint(), 15*time.Second)
  1814  			if err == nil {
  1815  				mapAudit := make(map[string]madmin.Status)
  1816  				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOnline)}
  1817  				auditloggerInfo = append(auditloggerInfo, mapAudit)
  1818  			} else {
  1819  				mapAudit := make(map[string]madmin.Status)
  1820  				mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOffline)}
  1821  				auditloggerInfo = append(auditloggerInfo, mapAudit)
  1822  			}
  1823  		}
  1824  	}
  1825  
  1826  	return loggerInfo, auditloggerInfo
  1827  }
  1828  
  1829  // checkConnection - ping an endpoint , return err in case of no connection
  1830  func checkConnection(endpointStr string, timeout time.Duration) error {
  1831  	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
  1832  	defer cancel()
  1833  
  1834  	client := &http.Client{Transport: &http.Transport{
  1835  		Proxy:                 http.ProxyFromEnvironment,
  1836  		DialContext:           xhttp.NewCustomDialContext(timeout),
  1837  		ResponseHeaderTimeout: 5 * time.Second,
  1838  		TLSHandshakeTimeout:   5 * time.Second,
  1839  		ExpectContinueTimeout: 5 * time.Second,
  1840  		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
  1841  		// Go net/http automatically unzip if content-type is
  1842  		// gzip disable this feature, as we are always interested
  1843  		// in raw stream.
  1844  		DisableCompression: true,
  1845  	}}
  1846  	defer client.CloseIdleConnections()
  1847  
  1848  	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
  1849  	if err != nil {
  1850  		return err
  1851  	}
  1852  
  1853  	resp, err := client.Do(req)
  1854  	if err != nil {
  1855  		return err
  1856  	}
  1857  	defer xhttp.DrainBody(resp.Body)
  1858  	return nil
  1859  }