storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/admin-handlers.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "crypto/subtle" 22 "crypto/tls" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io" 27 "math/rand" 28 "net/http" 29 "net/url" 30 "os" 31 "path" 32 "runtime" 33 "sort" 34 "strconv" 35 "strings" 36 "time" 37 38 "github.com/gorilla/mux" 39 40 "storj.io/minio/cmd/config" 41 "storj.io/minio/cmd/crypto" 42 xhttp "storj.io/minio/cmd/http" 43 "storj.io/minio/cmd/logger" 44 "storj.io/minio/cmd/logger/message/log" 45 "storj.io/minio/pkg/auth" 46 "storj.io/minio/pkg/bandwidth" 47 "storj.io/minio/pkg/dsync" 48 "storj.io/minio/pkg/handlers" 49 iampolicy "storj.io/minio/pkg/iam/policy" 50 "storj.io/minio/pkg/kms" 51 "storj.io/minio/pkg/madmin" 52 xnet "storj.io/minio/pkg/net" 53 trace "storj.io/minio/pkg/trace" 54 ) 55 56 const ( 57 maxEConfigJSONSize = 262272 58 ) 59 60 // Only valid query params for mgmt admin APIs. 61 const ( 62 mgmtBucket = "bucket" 63 mgmtPrefix = "prefix" 64 mgmtClientToken = "clientToken" 65 mgmtForceStart = "forceStart" 66 mgmtForceStop = "forceStop" 67 ) 68 69 func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, releaseInfo string, mode string) (us madmin.ServerUpdateStatus, err error) { 70 if err = doUpdate(u, lrTime, sha256Sum, releaseInfo, mode); err != nil { 71 return us, err 72 } 73 74 us.CurrentVersion = Version 75 us.UpdatedVersion = lrTime.Format(minioReleaseTagTimeLayout) 76 return us, nil 77 } 78 79 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL} 80 // ---------- 81 // updates all minio servers and restarts them gracefully. 82 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) { 83 ctx := NewContext(r, w, "ServerUpdate") 84 85 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 86 87 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction) 88 if objectAPI == nil { 89 return 90 } 91 92 if globalInplaceUpdateDisabled { 93 // if MINIO_UPDATE=off - inplace update is disabled, mostly in containers. 94 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) 95 return 96 } 97 98 vars := mux.Vars(r) 99 updateURL := vars["updateURL"] 100 mode := getMinioMode() 101 if updateURL == "" { 102 updateURL = minioReleaseInfoURL 103 if runtime.GOOS == globalWindowsOSName { 104 updateURL = minioReleaseWindowsInfoURL 105 } 106 } 107 108 u, err := url.Parse(updateURL) 109 if err != nil { 110 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 111 return 112 } 113 114 content, err := downloadReleaseURL(u, updateTimeout, mode) 115 if err != nil { 116 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 117 return 118 } 119 120 sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content) 121 if err != nil { 122 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 123 return 124 } 125 126 u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo 127 crTime, err := GetCurrentReleaseTime() 128 if err != nil { 129 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 130 return 131 } 132 133 if lrTime.Sub(crTime) <= 0 { 134 updateStatus := madmin.ServerUpdateStatus{ 135 CurrentVersion: Version, 136 UpdatedVersion: Version, 137 } 138 139 // Marshal API response 140 jsonBytes, err := json.Marshal(updateStatus) 141 if err != nil { 142 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 143 return 144 } 145 146 writeSuccessResponseJSON(w, jsonBytes) 147 return 148 } 149 150 for _, nerr := range GlobalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) { 151 if nerr.Err != nil { 152 logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) 153 logger.LogIf(ctx, nerr.Err) 154 err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err) 155 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 156 return 157 } 158 } 159 160 updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode) 161 if err != nil { 162 err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err) 163 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 164 return 165 } 166 167 // Marshal API response 168 jsonBytes, err := json.Marshal(updateStatus) 169 if err != nil { 170 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 171 return 172 } 173 174 writeSuccessResponseJSON(w, jsonBytes) 175 176 // Notify all other MinIO peers signal service. 177 for _, nerr := range GlobalNotificationSys.SignalService(serviceRestart) { 178 if nerr.Err != nil { 179 logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) 180 logger.LogIf(ctx, nerr.Err) 181 } 182 } 183 184 globalServiceSignalCh <- serviceRestart 185 } 186 187 // ServiceHandler - POST /minio/admin/v3/service?action={action} 188 // ---------- 189 // restarts/stops minio server gracefully. In a distributed setup, 190 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) { 191 ctx := NewContext(r, w, "Service") 192 193 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 194 195 vars := mux.Vars(r) 196 action := vars["action"] 197 198 var serviceSig serviceSignal 199 switch madmin.ServiceAction(action) { 200 case madmin.ServiceActionRestart: 201 serviceSig = serviceRestart 202 case madmin.ServiceActionStop: 203 serviceSig = serviceStop 204 default: 205 logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.Application) 206 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) 207 return 208 } 209 210 var objectAPI ObjectLayer 211 switch serviceSig { 212 case serviceRestart: 213 objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction) 214 case serviceStop: 215 objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction) 216 } 217 if objectAPI == nil { 218 return 219 } 220 221 // Notify all other MinIO peers signal service. 222 for _, nerr := range GlobalNotificationSys.SignalService(serviceSig) { 223 if nerr.Err != nil { 224 logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) 225 logger.LogIf(ctx, nerr.Err) 226 } 227 } 228 229 // Reply to the client before restarting, stopping MinIO server. 230 writeSuccessResponseHeadersOnly(w) 231 232 globalServiceSignalCh <- serviceSig 233 } 234 235 // ServerProperties holds some server information such as, version, region 236 // uptime, etc.. 237 type ServerProperties struct { 238 Uptime int64 `json:"uptime"` 239 Version string `json:"version"` 240 CommitID string `json:"commitID"` 241 DeploymentID string `json:"deploymentID"` 242 Region string `json:"region"` 243 SQSARN []string `json:"sqsARN"` 244 } 245 246 // ServerConnStats holds transferred bytes from/to the server 247 type ServerConnStats struct { 248 TotalInputBytes uint64 `json:"transferred"` 249 TotalOutputBytes uint64 `json:"received"` 250 Throughput uint64 `json:"throughput,omitempty"` 251 S3InputBytes uint64 `json:"transferredS3"` 252 S3OutputBytes uint64 `json:"receivedS3"` 253 } 254 255 // ServerHTTPAPIStats holds total number of HTTP operations from/to the server, 256 // including the average duration the call was spent. 257 type ServerHTTPAPIStats struct { 258 APIStats map[string]int `json:"apiStats"` 259 } 260 261 // ServerHTTPStats holds all type of http operations performed to/from the server 262 // including their average execution time. 263 type ServerHTTPStats struct { 264 S3RequestsInQueue int32 `json:"s3RequestsInQueue"` 265 CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` 266 TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` 267 TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` 268 TotalS3Canceled ServerHTTPAPIStats `json:"totalS3Canceled"` 269 TotalS3RejectedAuth uint64 `json:"totalS3RejectedAuth"` 270 TotalS3RejectedTime uint64 `json:"totalS3RejectedTime"` 271 TotalS3RejectedHeader uint64 `json:"totalS3RejectedHeader"` 272 TotalS3RejectedInvalid uint64 `json:"totalS3RejectedInvalid"` 273 } 274 275 // StorageInfoHandler - GET /minio/admin/v3/storageinfo 276 // ---------- 277 // Get server information 278 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) { 279 ctx := NewContext(r, w, "StorageInfo") 280 281 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 282 283 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction) 284 if objectAPI == nil { 285 return 286 } 287 288 // ignores any errors here. 289 storageInfo, _ := objectAPI.StorageInfo(ctx) 290 291 // Collect any disk healing. 292 healing, _ := getAggregatedBackgroundHealState(ctx, nil) 293 healDisks := make(map[string]struct{}, len(healing.HealDisks)) 294 for _, disk := range healing.HealDisks { 295 healDisks[disk] = struct{}{} 296 } 297 298 // find all disks which belong to each respective endpoints 299 for i, disk := range storageInfo.Disks { 300 if _, ok := healDisks[disk.Endpoint]; ok { 301 storageInfo.Disks[i].Healing = true 302 } 303 } 304 305 // Marshal API response 306 jsonBytes, err := json.Marshal(storageInfo) 307 if err != nil { 308 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 309 return 310 } 311 312 // Reply with storage information (across nodes in a 313 // distributed setup) as json. 314 writeSuccessResponseJSON(w, jsonBytes) 315 316 } 317 318 // DataUsageInfoHandler - GET /minio/admin/v3/datausage 319 // ---------- 320 // Get server/cluster data usage info 321 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) { 322 ctx := NewContext(r, w, "DataUsageInfo") 323 324 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 325 326 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction) 327 if objectAPI == nil { 328 return 329 } 330 331 dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) 332 if err != nil { 333 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 334 return 335 } 336 337 dataUsageInfoJSON, err := json.Marshal(dataUsageInfo) 338 if err != nil { 339 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 340 return 341 } 342 343 writeSuccessResponseJSON(w, dataUsageInfoJSON) 344 } 345 346 func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry { 347 entry := &madmin.LockEntry{ 348 Timestamp: l.Timestamp, 349 Resource: resource, 350 ServerList: []string{server}, 351 Source: l.Source, 352 Owner: l.Owner, 353 ID: l.UID, 354 Quorum: l.Quorum, 355 } 356 if l.Writer { 357 entry.Type = "WRITE" 358 } else { 359 entry.Type = "READ" 360 } 361 return entry 362 } 363 364 func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries { 365 entryMap := make(map[string]*madmin.LockEntry) 366 for _, peerLock := range peerLocks { 367 if peerLock == nil { 368 continue 369 } 370 for k, v := range peerLock.Locks { 371 for _, lockReqInfo := range v { 372 if val, ok := entryMap[lockReqInfo.UID]; ok { 373 val.ServerList = append(val.ServerList, peerLock.Addr) 374 } else { 375 entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr) 376 } 377 } 378 } 379 } 380 var lockEntries madmin.LockEntries 381 for _, v := range entryMap { 382 if stale { 383 lockEntries = append(lockEntries, *v) 384 continue 385 } 386 if len(v.ServerList) >= v.Quorum { 387 lockEntries = append(lockEntries, *v) 388 } 389 } 390 sort.Sort(lockEntries) 391 return lockEntries 392 } 393 394 // PeerLocks holds server information result of one node 395 type PeerLocks struct { 396 Addr string 397 Locks map[string][]lockRequesterInfo 398 } 399 400 // ForceUnlockHandler force unlocks requested resource 401 func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) { 402 ctx := NewContext(r, w, "ForceUnlock") 403 404 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 405 406 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction) 407 if objectAPI == nil { 408 return 409 } 410 411 z, ok := objectAPI.(*erasureServerPools) 412 if !ok { 413 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) 414 return 415 } 416 417 vars := mux.Vars(r) 418 419 var args dsync.LockArgs 420 lockersMap := make(map[string]dsync.NetLocker) 421 for _, path := range strings.Split(vars["paths"], ",") { 422 if path == "" { 423 continue 424 } 425 args.Resources = append(args.Resources, path) 426 lockers, _ := z.serverPools[0].getHashedSet(path).getLockers() 427 for _, locker := range lockers { 428 if locker != nil { 429 lockersMap[locker.String()] = locker 430 } 431 } 432 } 433 434 for _, locker := range lockersMap { 435 locker.ForceUnlock(ctx, args) 436 } 437 } 438 439 // TopLocksHandler Get list of locks in use 440 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) { 441 ctx := NewContext(r, w, "TopLocks") 442 443 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 444 445 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction) 446 if objectAPI == nil { 447 return 448 } 449 450 count := 10 // by default list only top 10 entries 451 if countStr := r.URL.Query().Get("count"); countStr != "" { 452 var err error 453 count, err = strconv.Atoi(countStr) 454 if err != nil { 455 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 456 return 457 } 458 } 459 stale := r.URL.Query().Get("stale") == "true" // list also stale locks 460 461 peerLocks := GlobalNotificationSys.GetLocks(ctx, r) 462 463 topLocks := topLockEntries(peerLocks, stale) 464 465 // Marshal API response upto requested count. 466 if len(topLocks) > count && count > 0 { 467 topLocks = topLocks[:count] 468 } 469 470 jsonBytes, err := json.Marshal(topLocks) 471 if err != nil { 472 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 473 return 474 } 475 476 // Reply with storage information (across nodes in a 477 // distributed setup) as json. 478 writeSuccessResponseJSON(w, jsonBytes) 479 } 480 481 // StartProfilingResult contains the status of the starting 482 // profiling action in a given server 483 type StartProfilingResult struct { 484 NodeName string `json:"nodeName"` 485 Success bool `json:"success"` 486 Error string `json:"error"` 487 } 488 489 // StartProfilingHandler - POST /minio/admin/v3/profiling/start?profilerType={profilerType} 490 // ---------- 491 // Enable server profiling 492 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) { 493 ctx := NewContext(r, w, "StartProfiling") 494 495 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 496 497 // Validate request signature. 498 _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "") 499 if adminAPIErr != ErrNone { 500 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 501 return 502 } 503 504 if GlobalNotificationSys == nil { 505 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) 506 return 507 } 508 509 vars := mux.Vars(r) 510 profiles := strings.Split(vars["profilerType"], ",") 511 thisAddr, err := xnet.ParseHost(globalLocalNodeName) 512 if err != nil { 513 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 514 return 515 } 516 517 globalProfilerMu.Lock() 518 defer globalProfilerMu.Unlock() 519 520 if globalProfiler == nil { 521 globalProfiler = make(map[string]minioProfiler, 10) 522 } 523 524 // Stop profiler of all types if already running 525 for k, v := range globalProfiler { 526 for _, p := range profiles { 527 if p == k { 528 v.Stop() 529 delete(globalProfiler, k) 530 } 531 } 532 } 533 534 // Start profiling on remote servers. 535 var hostErrs []NotificationPeerErr 536 for _, profiler := range profiles { 537 hostErrs = append(hostErrs, GlobalNotificationSys.StartProfiling(profiler)...) 538 539 // Start profiling locally as well. 540 prof, err := startProfiler(profiler) 541 if err != nil { 542 hostErrs = append(hostErrs, NotificationPeerErr{ 543 Host: *thisAddr, 544 Err: err, 545 }) 546 } else { 547 globalProfiler[profiler] = prof 548 hostErrs = append(hostErrs, NotificationPeerErr{ 549 Host: *thisAddr, 550 }) 551 } 552 } 553 554 var startProfilingResult []StartProfilingResult 555 556 for _, nerr := range hostErrs { 557 result := StartProfilingResult{NodeName: nerr.Host.String()} 558 if nerr.Err != nil { 559 result.Error = nerr.Err.Error() 560 } else { 561 result.Success = true 562 } 563 startProfilingResult = append(startProfilingResult, result) 564 } 565 566 // Create JSON result and send it to the client 567 startProfilingResultInBytes, err := json.Marshal(startProfilingResult) 568 if err != nil { 569 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 570 return 571 } 572 573 writeSuccessResponseJSON(w, startProfilingResultInBytes) 574 } 575 576 // dummyFileInfo represents a dummy representation of a profile data file 577 // present only in memory, it helps to generate the zip stream. 578 type dummyFileInfo struct { 579 name string 580 size int64 581 mode os.FileMode 582 modTime time.Time 583 isDir bool 584 sys interface{} 585 } 586 587 func (f dummyFileInfo) Name() string { return f.name } 588 func (f dummyFileInfo) Size() int64 { return f.size } 589 func (f dummyFileInfo) Mode() os.FileMode { return f.mode } 590 func (f dummyFileInfo) ModTime() time.Time { return f.modTime } 591 func (f dummyFileInfo) IsDir() bool { return f.isDir } 592 func (f dummyFileInfo) Sys() interface{} { return f.sys } 593 594 // DownloadProfilingHandler - POST /minio/admin/v3/profiling/download 595 // ---------- 596 // Download profiling information of all nodes in a zip format 597 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) { 598 ctx := NewContext(r, w, "DownloadProfiling") 599 600 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 601 602 // Validate request signature. 603 _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "") 604 if adminAPIErr != ErrNone { 605 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 606 return 607 } 608 609 if GlobalNotificationSys == nil { 610 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) 611 return 612 } 613 614 if !GlobalNotificationSys.DownloadProfilingData(ctx, w) { 615 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL) 616 return 617 } 618 } 619 620 type healInitParams struct { 621 bucket, objPrefix string 622 hs madmin.HealOpts 623 clientToken string 624 forceStart, forceStop bool 625 } 626 627 // extractHealInitParams - Validates params for heal init API. 628 func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) { 629 hip.bucket = vars[mgmtBucket] 630 hip.objPrefix = vars[mgmtPrefix] 631 632 if hip.bucket == "" { 633 if hip.objPrefix != "" { 634 // Bucket is required if object-prefix is given 635 err = ErrHealMissingBucket 636 return 637 } 638 } else if isReservedOrInvalidBucket(hip.bucket, false) { 639 err = ErrInvalidBucketName 640 return 641 } 642 643 // empty prefix is valid. 644 if !IsValidObjectPrefix(hip.objPrefix) { 645 err = ErrInvalidObjectName 646 return 647 } 648 649 if len(qParms[mgmtClientToken]) > 0 { 650 hip.clientToken = qParms[mgmtClientToken][0] 651 } 652 if _, ok := qParms[mgmtForceStart]; ok { 653 hip.forceStart = true 654 } 655 if _, ok := qParms[mgmtForceStop]; ok { 656 hip.forceStop = true 657 } 658 659 // Invalid request conditions: 660 // 661 // Cannot have both forceStart and forceStop in the same 662 // request; If clientToken is provided, request can only be 663 // to continue receiving logs, so it cannot be start or 664 // stop; 665 if (hip.forceStart && hip.forceStop) || 666 (hip.clientToken != "" && (hip.forceStart || hip.forceStop)) { 667 err = ErrInvalidRequest 668 return 669 } 670 671 // ignore body if clientToken is provided 672 if hip.clientToken == "" { 673 jerr := json.NewDecoder(r).Decode(&hip.hs) 674 if jerr != nil { 675 logger.LogIf(GlobalContext, jerr, logger.Application) 676 err = ErrRequestBodyParse 677 return 678 } 679 } 680 681 err = ErrNone 682 return 683 } 684 685 // HealHandler - POST /minio/admin/v3/heal/ 686 // ----------- 687 // Start heal processing and return heal status items. 688 // 689 // On a successful heal sequence start, a unique client token is 690 // returned. Subsequent requests to this endpoint providing the client 691 // token will receive heal status records from the running heal 692 // sequence. 693 // 694 // If no client token is provided, and a heal sequence is in progress 695 // an error is returned with information about the running heal 696 // sequence. However, if the force-start flag is provided, the server 697 // aborts the running heal sequence and starts a new one. 698 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) { 699 ctx := NewContext(r, w, "Heal") 700 701 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 702 703 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction) 704 if objectAPI == nil { 705 return 706 } 707 708 // Check if this setup has an erasure coded backend. 709 if !globalIsErasure { 710 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL) 711 return 712 } 713 714 hip, errCode := extractHealInitParams(mux.Vars(r), r.URL.Query(), r.Body) 715 if errCode != ErrNone { 716 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) 717 return 718 } 719 720 // Analyze the heal token and route the request accordingly 721 token, success := proxyRequestByToken(ctx, w, r, hip.clientToken) 722 if success { 723 return 724 } 725 hip.clientToken = token 726 // if request was not successful, try this server locally if token 727 // is not found the call will fail anyways. if token is empty 728 // try this server to generate a new token. 729 730 type healResp struct { 731 respBytes []byte 732 apiErr APIError 733 errBody string 734 } 735 736 // Define a closure to start sending whitespace to client 737 // after 10s unless a response item comes in 738 keepConnLive := func(w http.ResponseWriter, r *http.Request, respCh chan healResp) { 739 ticker := time.NewTicker(time.Second * 10) 740 defer ticker.Stop() 741 started := false 742 forLoop: 743 for { 744 select { 745 case <-r.Context().Done(): 746 return 747 case <-ticker.C: 748 if !started { 749 // Start writing response to client 750 started = true 751 setCommonHeaders(w) 752 setEventStreamHeaders(w) 753 // Set 200 OK status 754 w.WriteHeader(200) 755 } 756 // Send whitespace and keep connection open 757 w.Write([]byte(" ")) 758 w.(http.Flusher).Flush() 759 case hr := <-respCh: 760 switch hr.apiErr { 761 case noError: 762 if started { 763 w.Write(hr.respBytes) 764 w.(http.Flusher).Flush() 765 } else { 766 writeSuccessResponseJSON(w, hr.respBytes) 767 } 768 default: 769 var errorRespJSON []byte 770 if hr.errBody == "" { 771 errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr, 772 r.URL.Path, w.Header().Get(xhttp.AmzRequestID), 773 globalDeploymentID)) 774 } else { 775 errorRespJSON = encodeResponseJSON(APIErrorResponse{ 776 Code: hr.apiErr.Code, 777 Message: hr.errBody, 778 Resource: r.URL.Path, 779 RequestID: w.Header().Get(xhttp.AmzRequestID), 780 HostID: globalDeploymentID, 781 }) 782 } 783 if !started { 784 setCommonHeaders(w) 785 w.Header().Set(xhttp.ContentType, string(mimeJSON)) 786 w.WriteHeader(hr.apiErr.HTTPStatusCode) 787 } 788 w.Write(errorRespJSON) 789 w.(http.Flusher).Flush() 790 } 791 break forLoop 792 } 793 } 794 } 795 796 healPath := pathJoin(hip.bucket, hip.objPrefix) 797 if hip.clientToken == "" && !hip.forceStart && !hip.forceStop { 798 nh, exists := globalAllHealState.getHealSequence(healPath) 799 if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 { 800 clientToken := nh.clientToken 801 if globalIsDistErasure { 802 clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints)) 803 } 804 b, err := json.Marshal(madmin.HealStartSuccess{ 805 ClientToken: clientToken, 806 ClientAddress: nh.clientAddress, 807 StartTime: nh.startTime, 808 }) 809 if err != nil { 810 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 811 return 812 } 813 // Client token not specified but a heal sequence exists on a path, 814 // Send the token back to client. 815 writeSuccessResponseJSON(w, b) 816 return 817 } 818 } 819 820 if hip.clientToken != "" && !hip.forceStart && !hip.forceStop { 821 // Since clientToken is given, fetch heal status from running 822 // heal sequence. 823 respBytes, errCode := globalAllHealState.PopHealStatusJSON( 824 healPath, hip.clientToken) 825 if errCode != ErrNone { 826 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) 827 } else { 828 writeSuccessResponseJSON(w, respBytes) 829 } 830 return 831 } 832 833 respCh := make(chan healResp) 834 switch { 835 case hip.forceStop: 836 go func() { 837 respBytes, apiErr := globalAllHealState.stopHealSequence(healPath) 838 hr := healResp{respBytes: respBytes, apiErr: apiErr} 839 respCh <- hr 840 }() 841 case hip.clientToken == "": 842 nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart) 843 go func() { 844 respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI) 845 hr := healResp{respBytes, apiErr, errMsg} 846 respCh <- hr 847 }() 848 } 849 850 // Due to the force-starting functionality, the Launch 851 // call above can take a long time - to keep the 852 // connection alive, we start sending whitespace 853 keepConnLive(w, r, respCh) 854 } 855 856 // getAggregatedBackgroundHealState returns the heal state of disks. 857 // If no ObjectLayer is provided no set status is returned. 858 func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmin.BgHealState, error) { 859 // Get local heal status first 860 bgHealStates, ok := getBackgroundHealStatus(ctx, o) 861 if !ok { 862 return bgHealStates, errServerNotInitialized 863 } 864 865 if globalIsDistErasure { 866 // Get heal status from other peers 867 peersHealStates, nerrs := GlobalNotificationSys.BackgroundHealStatus() 868 var errCount int 869 for _, nerr := range nerrs { 870 if nerr.Err != nil { 871 logger.LogIf(ctx, nerr.Err) 872 errCount++ 873 } 874 } 875 if errCount == len(nerrs) { 876 return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy") 877 } 878 bgHealStates.Merge(peersHealStates...) 879 } 880 881 return bgHealStates, nil 882 } 883 884 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) { 885 ctx := NewContext(r, w, "HealBackgroundStatus") 886 887 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 888 889 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction) 890 if objectAPI == nil { 891 return 892 } 893 894 // Check if this setup has an erasure coded backend. 895 if !globalIsErasure { 896 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL) 897 return 898 } 899 900 aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI) 901 if err != nil { 902 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 903 return 904 } 905 906 if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil { 907 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 908 return 909 } 910 911 w.(http.Flusher).Flush() 912 } 913 914 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) { 915 var cred auth.Credentials 916 var adminAPIErr APIErrorCode 917 // Get current object layer instance. 918 objectAPI := newObjectLayerFn() 919 if objectAPI == nil || GlobalNotificationSys == nil { 920 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) 921 return nil, cred 922 } 923 924 // Validate request signature. 925 cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "") 926 if adminAPIErr != ErrNone { 927 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 928 return nil, cred 929 } 930 931 return objectAPI, cred 932 } 933 934 // AdminError - is a generic error for all admin APIs. 935 type AdminError struct { 936 Code string 937 Message string 938 StatusCode int 939 } 940 941 func (ae AdminError) Error() string { 942 return ae.Message 943 } 944 945 // Admin API errors 946 const ( 947 AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure" 948 AdminUpdateURLNotReachable = "XMinioAdminUpdateURLNotReachable" 949 AdminUpdateApplyFailure = "XMinioAdminUpdateApplyFailure" 950 ) 951 952 // toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API 953 // specific error. 954 func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode { 955 switch err { 956 case errErasureWriteQuorum: 957 return ErrAdminConfigNoQuorum 958 default: 959 return toAPIErrorCode(ctx, err) 960 } 961 } 962 963 func toAdminAPIErr(ctx context.Context, err error) APIError { 964 if err == nil { 965 return noError 966 } 967 968 var apiErr APIError 969 switch e := err.(type) { 970 case iampolicy.Error: 971 apiErr = APIError{ 972 Code: "XMinioMalformedIAMPolicy", 973 Description: e.Error(), 974 HTTPStatusCode: http.StatusBadRequest, 975 } 976 case config.Error: 977 apiErr = APIError{ 978 Code: "XMinioConfigError", 979 Description: e.Error(), 980 HTTPStatusCode: http.StatusBadRequest, 981 } 982 case AdminError: 983 apiErr = APIError{ 984 Code: e.Code, 985 Description: e.Message, 986 HTTPStatusCode: e.StatusCode, 987 } 988 default: 989 switch { 990 case errors.Is(err, errConfigNotFound): 991 apiErr = APIError{ 992 Code: "XMinioConfigError", 993 Description: err.Error(), 994 HTTPStatusCode: http.StatusNotFound, 995 } 996 case errors.Is(err, errIAMActionNotAllowed): 997 apiErr = APIError{ 998 Code: "XMinioIAMActionNotAllowed", 999 Description: err.Error(), 1000 HTTPStatusCode: http.StatusForbidden, 1001 } 1002 case errors.Is(err, errIAMNotInitialized): 1003 apiErr = APIError{ 1004 Code: "XMinioIAMNotInitialized", 1005 Description: err.Error(), 1006 HTTPStatusCode: http.StatusServiceUnavailable, 1007 } 1008 case errors.Is(err, crypto.ErrKESKeyExists): 1009 apiErr = APIError{ 1010 Code: "XMinioKMSKeyExists", 1011 Description: err.Error(), 1012 HTTPStatusCode: http.StatusConflict, 1013 } 1014 default: 1015 apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err) 1016 } 1017 } 1018 return apiErr 1019 } 1020 1021 // Returns true if the trace.Info should be traced, 1022 // false if certain conditions are not met. 1023 // - input entry is not of the type *trace.Info* 1024 // - errOnly entries are to be traced, not status code 2xx, 3xx. 1025 // - trace.Info type is asked by opts 1026 func mustTrace(entry interface{}, opts madmin.ServiceTraceOpts) (shouldTrace bool) { 1027 trcInfo, ok := entry.(trace.Info) 1028 if !ok { 1029 return false 1030 } 1031 1032 // Override shouldTrace decision with errOnly filtering 1033 defer func() { 1034 if shouldTrace && opts.OnlyErrors { 1035 shouldTrace = trcInfo.RespInfo.StatusCode >= http.StatusBadRequest 1036 } 1037 }() 1038 1039 if opts.Threshold > 0 { 1040 var latency time.Duration 1041 switch trcInfo.TraceType { 1042 case trace.OS: 1043 latency = trcInfo.OSStats.Duration 1044 case trace.Storage: 1045 latency = trcInfo.StorageStats.Duration 1046 case trace.HTTP: 1047 latency = trcInfo.CallStats.Latency 1048 } 1049 if latency < opts.Threshold { 1050 return false 1051 } 1052 } 1053 1054 if opts.Internal && trcInfo.TraceType == trace.HTTP && HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) { 1055 return true 1056 } 1057 1058 if opts.S3 && trcInfo.TraceType == trace.HTTP && !HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) { 1059 return true 1060 } 1061 1062 if opts.Storage && trcInfo.TraceType == trace.Storage { 1063 return true 1064 } 1065 1066 return opts.OS && trcInfo.TraceType == trace.OS 1067 } 1068 1069 func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) { 1070 q := r.URL.Query() 1071 1072 opts.OnlyErrors = q.Get("err") == "true" 1073 opts.S3 = q.Get("s3") == "true" 1074 opts.Internal = q.Get("internal") == "true" 1075 opts.Storage = q.Get("storage") == "true" 1076 opts.OS = q.Get("os") == "true" 1077 1078 // Support deprecated 'all' query 1079 if q.Get("all") == "true" { 1080 opts.S3 = true 1081 opts.Internal = true 1082 opts.Storage = true 1083 opts.OS = true 1084 } 1085 1086 if t := q.Get("threshold"); t != "" { 1087 d, err := time.ParseDuration(t) 1088 if err != nil { 1089 return opts, err 1090 } 1091 opts.Threshold = d 1092 } 1093 return 1094 } 1095 1096 // TraceHandler - POST /minio/admin/v3/trace 1097 // ---------- 1098 // The handler sends http trace to the connected HTTP client. 1099 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { 1100 ctx := NewContext(r, w, "HTTPTrace") 1101 1102 // Validate request signature. 1103 _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "") 1104 if adminAPIErr != ErrNone { 1105 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 1106 return 1107 } 1108 1109 traceOpts, err := extractTraceOptions(r) 1110 if err != nil { 1111 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) 1112 return 1113 } 1114 1115 setEventStreamHeaders(w) 1116 1117 // Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers. 1118 // Use buffered channel to take care of burst sends or slow w.Write() 1119 traceCh := make(chan interface{}, 4000) 1120 1121 peers, _ := newPeerRestClients(globalEndpoints) 1122 1123 globalTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool { 1124 return mustTrace(entry, traceOpts) 1125 }) 1126 1127 for _, peer := range peers { 1128 if peer == nil { 1129 continue 1130 } 1131 peer.Trace(traceCh, ctx.Done(), traceOpts) 1132 } 1133 1134 keepAliveTicker := time.NewTicker(500 * time.Millisecond) 1135 defer keepAliveTicker.Stop() 1136 1137 enc := json.NewEncoder(w) 1138 for { 1139 select { 1140 case entry := <-traceCh: 1141 if err := enc.Encode(entry); err != nil { 1142 return 1143 } 1144 w.(http.Flusher).Flush() 1145 case <-keepAliveTicker.C: 1146 if _, err := w.Write([]byte(" ")); err != nil { 1147 return 1148 } 1149 w.(http.Flusher).Flush() 1150 case <-ctx.Done(): 1151 return 1152 } 1153 } 1154 } 1155 1156 // The handler sends console logs to the connected HTTP client. 1157 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) { 1158 ctx := NewContext(r, w, "ConsoleLog") 1159 1160 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1161 1162 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction) 1163 if objectAPI == nil { 1164 return 1165 } 1166 node := r.URL.Query().Get("node") 1167 // limit buffered console entries if client requested it. 1168 limitStr := r.URL.Query().Get("limit") 1169 limitLines, err := strconv.Atoi(limitStr) 1170 if err != nil { 1171 limitLines = 10 1172 } 1173 1174 logKind := r.URL.Query().Get("logType") 1175 if logKind == "" { 1176 logKind = string(logger.All) 1177 } 1178 logKind = strings.ToUpper(logKind) 1179 1180 // Avoid reusing tcp connection if read timeout is hit 1181 // This is needed to make r.Context().Done() work as 1182 // expected in case of read timeout 1183 w.Header().Set("Connection", "close") 1184 1185 setEventStreamHeaders(w) 1186 1187 logCh := make(chan interface{}, 4000) 1188 1189 peers, _ := newPeerRestClients(globalEndpoints) 1190 1191 globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil) 1192 1193 for _, peer := range peers { 1194 if peer == nil { 1195 continue 1196 } 1197 if node == "" || strings.EqualFold(peer.host.Name, node) { 1198 peer.ConsoleLog(logCh, ctx.Done()) 1199 } 1200 } 1201 1202 enc := json.NewEncoder(w) 1203 1204 keepAliveTicker := time.NewTicker(500 * time.Millisecond) 1205 defer keepAliveTicker.Stop() 1206 1207 for { 1208 select { 1209 case entry := <-logCh: 1210 log, ok := entry.(log.Info) 1211 if ok && log.SendLog(node, logKind) { 1212 if err := enc.Encode(log); err != nil { 1213 return 1214 } 1215 w.(http.Flusher).Flush() 1216 } 1217 case <-keepAliveTicker.C: 1218 if _, err := w.Write([]byte(" ")); err != nil { 1219 return 1220 } 1221 w.(http.Flusher).Flush() 1222 case <-ctx.Done(): 1223 return 1224 } 1225 } 1226 } 1227 1228 // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id> 1229 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) { 1230 ctx := NewContext(r, w, "KMSCreateKey") 1231 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1232 1233 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction) 1234 if objectAPI == nil { 1235 return 1236 } 1237 1238 if GlobalKMS == nil { 1239 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) 1240 return 1241 } 1242 1243 if err := GlobalKMS.CreateKey(r.URL.Query().Get("key-id")); err != nil { 1244 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 1245 return 1246 } 1247 writeSuccessResponseHeadersOnly(w) 1248 } 1249 1250 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id> 1251 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) { 1252 ctx := NewContext(r, w, "KMSKeyStatus") 1253 1254 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1255 1256 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction) 1257 if objectAPI == nil { 1258 return 1259 } 1260 1261 if GlobalKMS == nil { 1262 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) 1263 return 1264 } 1265 stat, err := GlobalKMS.Stat() 1266 if err != nil { 1267 writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) 1268 return 1269 } 1270 1271 keyID := r.URL.Query().Get("key-id") 1272 if keyID == "" { 1273 keyID = stat.DefaultKey 1274 } 1275 var response = madmin.KMSKeyStatus{ 1276 KeyID: keyID, 1277 } 1278 1279 kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation 1280 // 1. Generate a new key using the KMS. 1281 key, err := GlobalKMS.GenerateKey(keyID, kmsContext) 1282 if err != nil { 1283 response.EncryptionErr = err.Error() 1284 resp, err := json.Marshal(response) 1285 if err != nil { 1286 writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) 1287 return 1288 } 1289 writeSuccessResponseJSON(w, resp) 1290 return 1291 } 1292 1293 // 2. Verify that we can indeed decrypt the (encrypted) key 1294 decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Plaintext, kmsContext) 1295 if err != nil { 1296 response.DecryptionErr = err.Error() 1297 resp, err := json.Marshal(response) 1298 if err != nil { 1299 writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) 1300 return 1301 } 1302 writeSuccessResponseJSON(w, resp) 1303 return 1304 } 1305 1306 // 3. Compare generated key with decrypted key 1307 if subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1 { 1308 response.DecryptionErr = "The generated and the decrypted data key do not match" 1309 resp, err := json.Marshal(response) 1310 if err != nil { 1311 writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) 1312 return 1313 } 1314 writeSuccessResponseJSON(w, resp) 1315 return 1316 } 1317 1318 resp, err := json.Marshal(response) 1319 if err != nil { 1320 writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) 1321 return 1322 } 1323 writeSuccessResponseJSON(w, resp) 1324 } 1325 1326 // HealthInfoHandler - GET /minio/admin/v3/healthinfo 1327 // ---------- 1328 // Get server health info 1329 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) { 1330 ctx := NewContext(r, w, "HealthInfo") 1331 1332 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1333 1334 objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction) 1335 if objectAPI == nil { 1336 return 1337 } 1338 1339 query := r.URL.Query() 1340 healthInfo := madmin.HealthInfo{} 1341 healthInfoCh := make(chan madmin.HealthInfo) 1342 1343 enc := json.NewEncoder(w) 1344 partialWrite := func(oinfo madmin.HealthInfo) { 1345 healthInfoCh <- oinfo 1346 } 1347 1348 setCommonHeaders(w) 1349 1350 setEventStreamHeaders(w) 1351 1352 w.WriteHeader(http.StatusOK) 1353 1354 errResp := func(err error) { 1355 errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(), 1356 w.Header().Get(xhttp.AmzRequestID), globalDeploymentID) 1357 encodedErrorResponse := EncodeResponse(errorResponse) 1358 healthInfo.Error = string(encodedErrorResponse) 1359 logger.LogIf(ctx, enc.Encode(healthInfo)) 1360 } 1361 1362 deadline := 3600 * time.Second 1363 if dstr := r.URL.Query().Get("deadline"); dstr != "" { 1364 var err error 1365 deadline, err = time.ParseDuration(dstr) 1366 if err != nil { 1367 errResp(err) 1368 return 1369 } 1370 } 1371 1372 deadlinedCtx, cancel := context.WithTimeout(ctx, deadline) 1373 defer cancel() 1374 1375 var err error 1376 nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress") 1377 ctx, err = nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)) 1378 if err != nil { // returns a locked lock 1379 errResp(err) 1380 return 1381 } 1382 defer nsLock.Unlock() 1383 1384 go func() { 1385 defer close(healthInfoCh) 1386 1387 if cpu := query.Get("syscpu"); cpu == "true" { 1388 cpuInfo := getLocalCPUInfo(deadlinedCtx, r) 1389 1390 healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo) 1391 healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, GlobalNotificationSys.CPUInfo(deadlinedCtx)...) 1392 partialWrite(healthInfo) 1393 } 1394 1395 if diskHw := query.Get("sysdiskhw"); diskHw == "true" { 1396 diskHwInfo := getLocalDiskHwInfo(deadlinedCtx, r) 1397 1398 healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, diskHwInfo) 1399 healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, GlobalNotificationSys.DiskHwInfo(deadlinedCtx)...) 1400 partialWrite(healthInfo) 1401 } 1402 1403 if osInfo := query.Get("sysosinfo"); osInfo == "true" { 1404 osInfo := getLocalOsInfo(deadlinedCtx, r) 1405 1406 healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, osInfo) 1407 healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, GlobalNotificationSys.OsInfo(deadlinedCtx)...) 1408 partialWrite(healthInfo) 1409 } 1410 1411 if mem := query.Get("sysmem"); mem == "true" { 1412 memInfo := getLocalMemInfo(deadlinedCtx, r) 1413 1414 healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, memInfo) 1415 healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, GlobalNotificationSys.MemInfo(deadlinedCtx)...) 1416 partialWrite(healthInfo) 1417 } 1418 1419 if proc := query.Get("sysprocess"); proc == "true" { 1420 procInfo := getLocalProcInfo(deadlinedCtx, r) 1421 1422 healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, procInfo) 1423 healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, GlobalNotificationSys.ProcInfo(deadlinedCtx)...) 1424 partialWrite(healthInfo) 1425 } 1426 1427 if config := query.Get("minioconfig"); config == "true" { 1428 cfg, err := readServerConfig(ctx, objectAPI) 1429 logger.LogIf(ctx, err) 1430 healthInfo.Minio.Config = cfg 1431 partialWrite(healthInfo) 1432 } 1433 1434 if drive := query.Get("perfdrive"); drive == "true" { 1435 // Get drive perf details from local server's drive(s) 1436 drivePerfSerial := getLocalDrives(deadlinedCtx, false, globalEndpoints, r) 1437 drivePerfParallel := getLocalDrives(deadlinedCtx, true, globalEndpoints, r) 1438 1439 errStr := "" 1440 if drivePerfSerial.Error != "" { 1441 errStr = "serial: " + drivePerfSerial.Error 1442 } 1443 if drivePerfParallel.Error != "" { 1444 errStr = errStr + " parallel: " + drivePerfParallel.Error 1445 } 1446 1447 driveInfo := madmin.ServerDrivesInfo{ 1448 Addr: drivePerfSerial.Addr, 1449 Serial: drivePerfSerial.Serial, 1450 Parallel: drivePerfParallel.Parallel, 1451 Error: errStr, 1452 } 1453 healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, driveInfo) 1454 partialWrite(healthInfo) 1455 1456 // Notify all other MinIO peers to report drive perf numbers 1457 driveInfos := GlobalNotificationSys.DrivePerfInfoChan(deadlinedCtx) 1458 for obd := range driveInfos { 1459 healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, obd) 1460 partialWrite(healthInfo) 1461 } 1462 partialWrite(healthInfo) 1463 } 1464 1465 if net := query.Get("perfnet"); net == "true" && globalIsDistErasure { 1466 healthInfo.Perf.Net = append(healthInfo.Perf.Net, GlobalNotificationSys.NetInfo(deadlinedCtx)) 1467 partialWrite(healthInfo) 1468 1469 netInfos := GlobalNotificationSys.DispatchNetPerfChan(deadlinedCtx) 1470 for netInfo := range netInfos { 1471 healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo) 1472 partialWrite(healthInfo) 1473 } 1474 partialWrite(healthInfo) 1475 1476 healthInfo.Perf.NetParallel = GlobalNotificationSys.NetPerfParallelInfo(deadlinedCtx) 1477 partialWrite(healthInfo) 1478 } 1479 1480 }() 1481 1482 ticker := time.NewTicker(5 * time.Second) 1483 defer ticker.Stop() 1484 1485 for { 1486 select { 1487 case oinfo, ok := <-healthInfoCh: 1488 if !ok { 1489 return 1490 } 1491 logger.LogIf(ctx, enc.Encode(oinfo)) 1492 w.(http.Flusher).Flush() 1493 case <-ticker.C: 1494 if _, err := w.Write([]byte(" ")); err != nil { 1495 return 1496 } 1497 w.(http.Flusher).Flush() 1498 case <-deadlinedCtx.Done(): 1499 w.(http.Flusher).Flush() 1500 return 1501 } 1502 } 1503 1504 } 1505 1506 // BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth 1507 // ---------- 1508 // Get bandwidth consumption information 1509 func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) { 1510 ctx := NewContext(r, w, "BandwidthMonitor") 1511 1512 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1513 1514 // Validate request signature. 1515 _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "") 1516 if adminAPIErr != ErrNone { 1517 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 1518 return 1519 } 1520 1521 rnd := rand.New(rand.NewSource(time.Now().UnixNano())) 1522 1523 setEventStreamHeaders(w) 1524 reportCh := make(chan bandwidth.Report) 1525 keepAliveTicker := time.NewTicker(500 * time.Millisecond) 1526 defer keepAliveTicker.Stop() 1527 bucketsRequestedString := r.URL.Query().Get("buckets") 1528 bucketsRequested := strings.Split(bucketsRequestedString, ",") 1529 go func() { 1530 defer close(reportCh) 1531 for { 1532 select { 1533 case <-ctx.Done(): 1534 return 1535 case reportCh <- GlobalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...): 1536 time.Sleep(time.Duration(rnd.Float64() * float64(2*time.Second))) 1537 } 1538 } 1539 }() 1540 for { 1541 select { 1542 case report, ok := <-reportCh: 1543 if !ok { 1544 return 1545 } 1546 if err := json.NewEncoder(w).Encode(report); err != nil { 1547 writeErrorResponseJSON(ctx, w, ToAPIError(ctx, err), r.URL) 1548 return 1549 } 1550 w.(http.Flusher).Flush() 1551 case <-keepAliveTicker.C: 1552 if _, err := w.Write([]byte(" ")); err != nil { 1553 return 1554 } 1555 w.(http.Flusher).Flush() 1556 case <-ctx.Done(): 1557 return 1558 } 1559 } 1560 } 1561 1562 // ServerInfoHandler - GET /minio/admin/v3/info 1563 // ---------- 1564 // Get server information 1565 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) { 1566 ctx := NewContext(r, w, "ServerInfo") 1567 1568 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 1569 1570 // Validate request signature. 1571 _, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "") 1572 if adminAPIErr != ErrNone { 1573 writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) 1574 return 1575 } 1576 1577 kmsStat := fetchKMSStatus() 1578 1579 ldap := madmin.LDAP{} 1580 if globalLDAPConfig.Enabled { 1581 ldapConn, err := globalLDAPConfig.Connect() 1582 if err != nil { 1583 ldap.Status = string(madmin.ItemOffline) 1584 } else if ldapConn == nil { 1585 ldap.Status = "Not Configured" 1586 } else { 1587 // Close ldap connection to avoid leaks. 1588 ldapConn.Close() 1589 ldap.Status = string(madmin.ItemOnline) 1590 } 1591 } 1592 1593 log, audit := fetchLoggerInfo() 1594 1595 // Get the notification target info 1596 notifyTarget := fetchLambdaInfo() 1597 1598 local := getLocalServerProperty(globalEndpoints, r) 1599 servers := GlobalNotificationSys.ServerInfo() 1600 servers = append(servers, local) 1601 1602 assignPoolNumbers(servers) 1603 1604 var backend interface{} 1605 mode := madmin.ItemInitializing 1606 1607 buckets := madmin.Buckets{} 1608 objects := madmin.Objects{} 1609 usage := madmin.Usage{} 1610 1611 objectAPI := newObjectLayerFn() 1612 if objectAPI != nil { 1613 mode = madmin.ItemOnline 1614 1615 // Load data usage 1616 dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) 1617 if err == nil { 1618 buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} 1619 objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} 1620 usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} 1621 } else { 1622 buckets = madmin.Buckets{Error: err.Error()} 1623 objects = madmin.Objects{Error: err.Error()} 1624 usage = madmin.Usage{Error: err.Error()} 1625 } 1626 1627 // Fetching the backend information 1628 backendInfo := objectAPI.BackendInfo() 1629 if backendInfo.Type == madmin.Erasure { 1630 // Calculate the number of online/offline disks of all nodes 1631 var allDisks []madmin.Disk 1632 for _, s := range servers { 1633 allDisks = append(allDisks, s.Disks...) 1634 } 1635 onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks) 1636 1637 backend = madmin.ErasureBackend{ 1638 Type: madmin.ErasureType, 1639 OnlineDisks: onlineDisks.Sum(), 1640 OfflineDisks: offlineDisks.Sum(), 1641 StandardSCParity: backendInfo.StandardSCParity, 1642 RRSCParity: backendInfo.RRSCParity, 1643 } 1644 } else { 1645 backend = madmin.FSBackend{ 1646 Type: madmin.FsType, 1647 } 1648 } 1649 } 1650 1651 domain := globalDomainNames 1652 services := madmin.Services{ 1653 KMS: kmsStat, 1654 LDAP: ldap, 1655 Logger: log, 1656 Audit: audit, 1657 Notifications: notifyTarget, 1658 } 1659 1660 infoMsg := madmin.InfoMessage{ 1661 Mode: string(mode), 1662 Domain: domain, 1663 Region: globalServerRegion, 1664 SQSARN: GlobalNotificationSys.GetARNList(false), 1665 DeploymentID: globalDeploymentID, 1666 Buckets: buckets, 1667 Objects: objects, 1668 Usage: usage, 1669 Services: services, 1670 Backend: backend, 1671 Servers: servers, 1672 } 1673 1674 // Marshal API response 1675 jsonBytes, err := json.Marshal(infoMsg) 1676 if err != nil { 1677 writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) 1678 return 1679 } 1680 1681 // Reply with storage information (across nodes in a 1682 // distributed setup) as json. 1683 writeSuccessResponseJSON(w, jsonBytes) 1684 } 1685 1686 func assignPoolNumbers(servers []madmin.ServerProperties) { 1687 for i := range servers { 1688 for idx, ge := range globalEndpoints { 1689 for _, endpoint := range ge.Endpoints { 1690 if servers[i].Endpoint == endpoint.Host { 1691 servers[i].PoolNumber = idx + 1 1692 } else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil { 1693 if host.Name == endpoint.Hostname() { 1694 servers[i].PoolNumber = idx + 1 1695 } 1696 } 1697 } 1698 } 1699 } 1700 } 1701 1702 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus { 1703 1704 lambdaMap := make(map[string][]madmin.TargetIDStatus) 1705 1706 for _, tgt := range globalConfigTargetList.Targets() { 1707 targetIDStatus := make(map[string]madmin.Status) 1708 active, _ := tgt.IsActive() 1709 targetID := tgt.ID() 1710 if active { 1711 targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)} 1712 } else { 1713 targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)} 1714 } 1715 list := lambdaMap[targetID.Name] 1716 list = append(list, targetIDStatus) 1717 lambdaMap[targetID.Name] = list 1718 } 1719 1720 for _, tgt := range globalEnvTargetList.Targets() { 1721 targetIDStatus := make(map[string]madmin.Status) 1722 active, _ := tgt.IsActive() 1723 targetID := tgt.ID() 1724 if active { 1725 targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)} 1726 } else { 1727 targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)} 1728 } 1729 list := lambdaMap[targetID.Name] 1730 list = append(list, targetIDStatus) 1731 lambdaMap[targetID.Name] = list 1732 } 1733 1734 notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap)) 1735 counter := 0 1736 for key, value := range lambdaMap { 1737 v := make(map[string][]madmin.TargetIDStatus) 1738 v[key] = value 1739 notify[counter] = v 1740 counter++ 1741 } 1742 return notify 1743 } 1744 1745 // fetchKMSStatus fetches KMS-related status information. 1746 func fetchKMSStatus() madmin.KMS { 1747 kmsStat := madmin.KMS{} 1748 if GlobalKMS == nil { 1749 kmsStat.Status = "disabled" 1750 return kmsStat 1751 } 1752 1753 stat, err := GlobalKMS.Stat() 1754 if err != nil { 1755 kmsStat.Status = string(madmin.ItemOffline) 1756 return kmsStat 1757 } 1758 if len(stat.Endpoints) == 0 { 1759 kmsStat.Status = stat.Name 1760 return kmsStat 1761 } 1762 if err := checkConnection(stat.Endpoints[0], 15*time.Second); err != nil { 1763 kmsStat.Status = string(madmin.ItemOffline) 1764 return kmsStat 1765 } 1766 kmsStat.Status = string(madmin.ItemOnline) 1767 1768 kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation 1769 // 1. Generate a new key using the KMS. 1770 key, err := GlobalKMS.GenerateKey("", kmsContext) 1771 if err != nil { 1772 kmsStat.Encrypt = fmt.Sprintf("Encryption failed: %v", err) 1773 } else { 1774 kmsStat.Encrypt = "success" 1775 } 1776 1777 // 2. Verify that we can indeed decrypt the (encrypted) key 1778 decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Ciphertext, kmsContext) 1779 switch { 1780 case err != nil: 1781 kmsStat.Decrypt = fmt.Sprintf("Decryption failed: %v", err) 1782 case subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1: 1783 kmsStat.Decrypt = "Decryption failed: decrypted key does not match generated key" 1784 default: 1785 kmsStat.Decrypt = "success" 1786 } 1787 return kmsStat 1788 } 1789 1790 // fetchLoggerDetails return log info 1791 func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) { 1792 var loggerInfo []madmin.Logger 1793 var auditloggerInfo []madmin.Audit 1794 for _, target := range logger.Targets { 1795 if target.Endpoint() != "" { 1796 tgt := target.String() 1797 err := checkConnection(target.Endpoint(), 15*time.Second) 1798 if err == nil { 1799 mapLog := make(map[string]madmin.Status) 1800 mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOnline)} 1801 loggerInfo = append(loggerInfo, mapLog) 1802 } else { 1803 mapLog := make(map[string]madmin.Status) 1804 mapLog[tgt] = madmin.Status{Status: string(madmin.ItemOffline)} 1805 loggerInfo = append(loggerInfo, mapLog) 1806 } 1807 } 1808 } 1809 1810 for _, target := range logger.AuditTargets { 1811 if target.Endpoint() != "" { 1812 tgt := target.String() 1813 err := checkConnection(target.Endpoint(), 15*time.Second) 1814 if err == nil { 1815 mapAudit := make(map[string]madmin.Status) 1816 mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOnline)} 1817 auditloggerInfo = append(auditloggerInfo, mapAudit) 1818 } else { 1819 mapAudit := make(map[string]madmin.Status) 1820 mapAudit[tgt] = madmin.Status{Status: string(madmin.ItemOffline)} 1821 auditloggerInfo = append(auditloggerInfo, mapAudit) 1822 } 1823 } 1824 } 1825 1826 return loggerInfo, auditloggerInfo 1827 } 1828 1829 // checkConnection - ping an endpoint , return err in case of no connection 1830 func checkConnection(endpointStr string, timeout time.Duration) error { 1831 ctx, cancel := context.WithTimeout(GlobalContext, timeout) 1832 defer cancel() 1833 1834 client := &http.Client{Transport: &http.Transport{ 1835 Proxy: http.ProxyFromEnvironment, 1836 DialContext: xhttp.NewCustomDialContext(timeout), 1837 ResponseHeaderTimeout: 5 * time.Second, 1838 TLSHandshakeTimeout: 5 * time.Second, 1839 ExpectContinueTimeout: 5 * time.Second, 1840 TLSClientConfig: &tls.Config{RootCAs: globalRootCAs}, 1841 // Go net/http automatically unzip if content-type is 1842 // gzip disable this feature, as we are always interested 1843 // in raw stream. 1844 DisableCompression: true, 1845 }} 1846 defer client.CloseIdleConnections() 1847 1848 req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil) 1849 if err != nil { 1850 return err 1851 } 1852 1853 resp, err := client.Do(req) 1854 if err != nil { 1855 return err 1856 } 1857 defer xhttp.DrainBody(resp.Body) 1858 return nil 1859 }