github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/cmd/admin-handlers-site-replication.go (about)

     1  // Copyright (c) 2015-2021 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package cmd
    19  
    20  import (
    21  	"bytes"
    22  	"context"
    23  	"encoding/gob"
    24  	"encoding/json"
    25  	"errors"
    26  	"io"
    27  	"net/http"
    28  	"strings"
    29  	"sync/atomic"
    30  	"time"
    31  
    32  	"github.com/dustin/go-humanize"
    33  	"github.com/minio/madmin-go/v3"
    34  	xioutil "github.com/minio/minio/internal/ioutil"
    35  	"github.com/minio/minio/internal/logger"
    36  	"github.com/minio/mux"
    37  	"github.com/minio/pkg/v2/policy"
    38  )
    39  
    40  // SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
    41  func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
    42  	ctx := r.Context()
    43  
    44  	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
    45  	if objectAPI == nil {
    46  		return
    47  	}
    48  
    49  	var sites []madmin.PeerSite
    50  	if err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey); err != nil {
    51  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
    52  		return
    53  	}
    54  
    55  	opts := getSRAddOptions(r)
    56  	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
    57  	if err != nil {
    58  		logger.LogIf(ctx, err)
    59  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
    60  		return
    61  	}
    62  
    63  	body, err := json.Marshal(status)
    64  	if err != nil {
    65  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
    66  		return
    67  	}
    68  
    69  	writeSuccessResponseJSON(w, body)
    70  }
    71  
    72  func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
    73  	opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
    74  	return
    75  }
    76  
    77  // SRPeerJoin - PUT /minio/admin/v3/site-replication/join
    78  //
    79  // used internally to tell current cluster to enable SR with
    80  // the provided peer clusters and service account.
    81  func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
    82  	ctx := r.Context()
    83  
    84  	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
    85  	if objectAPI == nil {
    86  		return
    87  	}
    88  
    89  	var joinArg madmin.SRPeerJoinReq
    90  	if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
    91  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
    92  		return
    93  	}
    94  
    95  	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
    96  		logger.LogIf(ctx, err)
    97  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
    98  		return
    99  	}
   100  }
   101  
   102  // SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
   103  func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
   104  	ctx := r.Context()
   105  
   106  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
   107  	if objectAPI == nil {
   108  		return
   109  	}
   110  
   111  	vars := mux.Vars(r)
   112  	bucket := vars["bucket"]
   113  	operation := madmin.BktOp(vars["operation"])
   114  
   115  	var err error
   116  	switch operation {
   117  	default:
   118  		err = errSRInvalidRequest(errInvalidArgument)
   119  	case madmin.MakeWithVersioningBktOp:
   120  		createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt")))
   121  		if cerr != nil {
   122  			createdAt = timeSentinel
   123  		}
   124  
   125  		opts := MakeBucketOptions{
   126  			LockEnabled:       r.Form.Get("lockEnabled") == "true",
   127  			VersioningEnabled: r.Form.Get("versioningEnabled") == "true",
   128  			ForceCreate:       r.Form.Get("forceCreate") == "true",
   129  			CreatedAt:         createdAt,
   130  		}
   131  		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
   132  	case madmin.ConfigureReplBktOp:
   133  		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
   134  	case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp:
   135  		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
   136  			Force:      operation == madmin.ForceDeleteBucketBktOp,
   137  			SRDeleteOp: getSRBucketDeleteOp(true),
   138  		})
   139  	case madmin.PurgeDeletedBucketOp:
   140  		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
   141  	}
   142  	if err != nil {
   143  		logger.LogIf(ctx, err)
   144  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   145  		return
   146  	}
   147  }
   148  
   149  // SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
   150  func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
   151  	ctx := r.Context()
   152  
   153  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
   154  	if objectAPI == nil {
   155  		return
   156  	}
   157  
   158  	var item madmin.SRIAMItem
   159  	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
   160  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   161  		return
   162  	}
   163  
   164  	var err error
   165  	switch item.Type {
   166  	default:
   167  		err = errSRInvalidRequest(errInvalidArgument)
   168  	case madmin.SRIAMItemPolicy:
   169  		if item.Policy == nil {
   170  			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
   171  		} else {
   172  			policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
   173  			if perr != nil {
   174  				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
   175  				return
   176  			}
   177  			if policy.IsEmpty() {
   178  				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
   179  			} else {
   180  				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt)
   181  			}
   182  		}
   183  	case madmin.SRIAMItemSvcAcc:
   184  		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
   185  	case madmin.SRIAMItemPolicyMapping:
   186  		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
   187  	case madmin.SRIAMItemSTSAcc:
   188  		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
   189  	case madmin.SRIAMItemIAMUser:
   190  		err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
   191  	case madmin.SRIAMItemGroupInfo:
   192  		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
   193  	}
   194  	if err != nil {
   195  		logger.LogIf(ctx, err)
   196  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   197  		return
   198  	}
   199  }
   200  
   201  // SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
   202  func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
   203  	ctx := r.Context()
   204  
   205  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
   206  	if objectAPI == nil {
   207  		return
   208  	}
   209  
   210  	var item madmin.SRBucketMeta
   211  	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
   212  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   213  		return
   214  	}
   215  
   216  	if item.Bucket == "" {
   217  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
   218  		return
   219  	}
   220  
   221  	var err error
   222  	switch item.Type {
   223  	default:
   224  		err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
   225  	case madmin.SRBucketMetaTypePolicy:
   226  		if item.Policy == nil {
   227  			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
   228  		} else {
   229  			bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
   230  			if berr != nil {
   231  				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
   232  				return
   233  			}
   234  			if bktPolicy.IsEmpty() {
   235  				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
   236  			} else {
   237  				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt)
   238  			}
   239  		}
   240  	case madmin.SRBucketMetaTypeQuotaConfig:
   241  		if item.Quota == nil {
   242  			err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt)
   243  		} else {
   244  			quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
   245  			if err != nil {
   246  				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   247  				return
   248  			}
   249  			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
   250  				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   251  				return
   252  			}
   253  		}
   254  	case madmin.SRBucketMetaTypeVersionConfig:
   255  		err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt)
   256  	case madmin.SRBucketMetaTypeTags:
   257  		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
   258  	case madmin.SRBucketMetaTypeObjectLockConfig:
   259  		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
   260  	case madmin.SRBucketMetaTypeSSEConfig:
   261  		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
   262  	case madmin.SRBucketMetaLCConfig:
   263  		err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
   264  	}
   265  	if err != nil {
   266  		logger.LogIf(ctx, err)
   267  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   268  		return
   269  	}
   270  }
   271  
   272  // SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
   273  func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
   274  	ctx := r.Context()
   275  
   276  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
   277  	if objectAPI == nil {
   278  		return
   279  	}
   280  
   281  	info, err := globalSiteReplicationSys.GetClusterInfo(ctx)
   282  	if err != nil {
   283  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   284  		return
   285  	}
   286  
   287  	if err = json.NewEncoder(w).Encode(info); err != nil {
   288  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   289  		return
   290  	}
   291  }
   292  
   293  func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
   294  	ctx := r.Context()
   295  
   296  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
   297  	if objectAPI == nil {
   298  		return
   299  	}
   300  
   301  	idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx)
   302  	if err := json.NewEncoder(w).Encode(idpSettings); err != nil {
   303  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   304  		return
   305  	}
   306  }
   307  
   308  func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
   309  	data, err := io.ReadAll(body)
   310  	if err != nil {
   311  		return SRError{
   312  			Cause: err,
   313  			Code:  ErrSiteReplicationInvalidRequest,
   314  		}
   315  	}
   316  	if encryptionKey != "" {
   317  		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
   318  		if err != nil {
   319  			logger.LogIf(ctx, err)
   320  			return SRError{
   321  				Cause: err,
   322  				Code:  ErrSiteReplicationInvalidRequest,
   323  			}
   324  		}
   325  	}
   326  	return json.Unmarshal(data, v)
   327  }
   328  
   329  // SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
   330  func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
   331  	ctx := r.Context()
   332  
   333  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
   334  	if objectAPI == nil {
   335  		return
   336  	}
   337  	opts := getSRStatusOptions(r)
   338  	// default options to all if status options are unset for backward compatibility
   339  	var dfltOpts madmin.SRStatusOptions
   340  	if opts == dfltOpts {
   341  		opts.Buckets = true
   342  		opts.Users = true
   343  		opts.Policies = true
   344  		opts.Groups = true
   345  		opts.ILMExpiryRules = true
   346  	}
   347  	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
   348  	if err != nil {
   349  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   350  		return
   351  	}
   352  
   353  	if err = json.NewEncoder(w).Encode(info); err != nil {
   354  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   355  		return
   356  	}
   357  }
   358  
   359  // SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
   360  func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
   361  	ctx := r.Context()
   362  
   363  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
   364  	if objectAPI == nil {
   365  		return
   366  	}
   367  
   368  	opts := getSRStatusOptions(r)
   369  	info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI, opts)
   370  	if err != nil {
   371  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   372  		return
   373  	}
   374  
   375  	if err = json.NewEncoder(w).Encode(info); err != nil {
   376  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   377  		return
   378  	}
   379  }
   380  
   381  // SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
   382  func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
   383  	ctx := r.Context()
   384  
   385  	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
   386  	if objectAPI == nil {
   387  		return
   388  	}
   389  	var site madmin.PeerInfo
   390  	err := parseJSONBody(ctx, r.Body, &site, cred.SecretKey)
   391  	if err != nil {
   392  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   393  		return
   394  	}
   395  
   396  	opts := getSREditOptions(r)
   397  	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
   398  	if err != nil {
   399  		logger.LogIf(ctx, err)
   400  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   401  		return
   402  	}
   403  	body, err := json.Marshal(status)
   404  	if err != nil {
   405  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   406  		return
   407  	}
   408  
   409  	writeSuccessResponseJSON(w, body)
   410  }
   411  
   412  func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
   413  	opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
   414  	opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
   415  	return
   416  }
   417  
   418  // SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
   419  //
   420  // used internally to tell current cluster to update endpoint for peer
   421  func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
   422  	ctx := r.Context()
   423  
   424  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
   425  	if objectAPI == nil {
   426  		return
   427  	}
   428  
   429  	var pi madmin.PeerInfo
   430  	if err := parseJSONBody(ctx, r.Body, &pi, ""); err != nil {
   431  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   432  		return
   433  	}
   434  
   435  	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
   436  		logger.LogIf(ctx, err)
   437  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   438  		return
   439  	}
   440  }
   441  
   442  // SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
   443  //
   444  // used internally to tell current cluster to update site replication state
   445  func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
   446  	ctx := r.Context()
   447  
   448  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
   449  	if objectAPI == nil {
   450  		return
   451  	}
   452  
   453  	var state madmin.SRStateEditReq
   454  	if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
   455  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   456  		return
   457  	}
   458  	if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
   459  		logger.LogIf(ctx, err)
   460  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   461  		return
   462  	}
   463  }
   464  
   465  func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
   466  	q := r.Form
   467  	opts.Buckets = q.Get("buckets") == "true"
   468  	opts.Policies = q.Get("policies") == "true"
   469  	opts.Groups = q.Get("groups") == "true"
   470  	opts.Users = q.Get("users") == "true"
   471  	opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
   472  	opts.PeerState = q.Get("peer-state") == "true"
   473  	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
   474  	opts.EntityValue = q.Get("entityvalue")
   475  	opts.ShowDeleted = q.Get("showDeleted") == "true"
   476  	opts.Metrics = q.Get("metrics") == "true"
   477  	return
   478  }
   479  
   480  // SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
   481  func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
   482  	ctx := r.Context()
   483  
   484  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
   485  	if objectAPI == nil {
   486  		return
   487  	}
   488  	var rreq madmin.SRRemoveReq
   489  	err := parseJSONBody(ctx, r.Body, &rreq, "")
   490  	if err != nil {
   491  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   492  		return
   493  	}
   494  	status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
   495  	if err != nil {
   496  		logger.LogIf(ctx, err)
   497  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   498  		return
   499  	}
   500  
   501  	body, err := json.Marshal(status)
   502  	if err != nil {
   503  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   504  		return
   505  	}
   506  	writeSuccessResponseJSON(w, body)
   507  }
   508  
   509  // SRPeerRemove - PUT /minio/admin/v3/site-replication/peer/remove
   510  //
   511  // used internally to tell current cluster to update endpoint for peer
   512  func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
   513  	ctx := r.Context()
   514  
   515  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
   516  	if objectAPI == nil {
   517  		return
   518  	}
   519  
   520  	var req madmin.SRRemoveReq
   521  	if err := parseJSONBody(ctx, r.Body, &req, ""); err != nil {
   522  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   523  		return
   524  	}
   525  
   526  	if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
   527  		logger.LogIf(ctx, err)
   528  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   529  		return
   530  	}
   531  }
   532  
   533  // SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
   534  func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
   535  	ctx := r.Context()
   536  
   537  	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction)
   538  	if objectAPI == nil {
   539  		return
   540  	}
   541  
   542  	var peerSite madmin.PeerInfo
   543  	if err := parseJSONBody(ctx, r.Body, &peerSite, ""); err != nil {
   544  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   545  		return
   546  	}
   547  	vars := mux.Vars(r)
   548  	op := madmin.SiteResyncOp(vars["operation"])
   549  	var (
   550  		status madmin.SRResyncOpStatus
   551  		err    error
   552  	)
   553  	switch op {
   554  	case madmin.SiteResyncStart:
   555  		status, err = globalSiteReplicationSys.startResync(ctx, objectAPI, peerSite)
   556  	case madmin.SiteResyncCancel:
   557  		status, err = globalSiteReplicationSys.cancelResync(ctx, objectAPI, peerSite)
   558  	default:
   559  		err = errSRInvalidRequest(errInvalidArgument)
   560  	}
   561  	if err != nil {
   562  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   563  		return
   564  	}
   565  	body, err := json.Marshal(status)
   566  	if err != nil {
   567  		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
   568  		return
   569  	}
   570  	writeSuccessResponseJSON(w, body)
   571  }
   572  
   573  // SiteReplicationDevNull - everything goes to io.Discard
   574  // [POST] /minio/admin/v3/site-replication/devnull
   575  func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
   576  	ctx := r.Context()
   577  
   578  	globalSiteNetPerfRX.Connect()
   579  	defer globalSiteNetPerfRX.Disconnect()
   580  
   581  	connectTime := time.Now()
   582  	for {
   583  		n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
   584  		atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
   585  		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
   586  			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
   587  			// would mean the network is not stable. Logging here will help in debugging network issues.
   588  			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
   589  				logger.LogIf(ctx, err)
   590  			}
   591  		}
   592  		if err != nil {
   593  			if errors.Is(err, io.EOF) {
   594  				w.WriteHeader(http.StatusNoContent)
   595  			} else {
   596  				w.WriteHeader(http.StatusBadRequest)
   597  			}
   598  			break
   599  		}
   600  	}
   601  }
   602  
   603  // SiteReplicationNetPerf - everything goes to io.Discard
   604  // [POST] /minio/admin/v3/site-replication/netperf
   605  func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
   606  	durationStr := r.Form.Get(peerRESTDuration)
   607  	duration, _ := time.ParseDuration(durationStr)
   608  	if duration < globalNetPerfMinDuration {
   609  		duration = globalNetPerfMinDuration
   610  	}
   611  	result := siteNetperf(r.Context(), duration)
   612  	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
   613  }