github.com/verrazzano/verrazzano@v1.7.1/cluster-operator/controllers/vmc/rancher.go (about)

     1  // Copyright (c) 2021, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  package vmc
     5  
     6  import (
     7  	"context"
     8  	"crypto/md5" //nolint:gosec //#gosec G501 // package used for caching only, not security
     9  	"fmt"
    10  	"io"
    11  	"net/http"
    12  
    13  	"github.com/verrazzano/verrazzano/cluster-operator/apis/clusters/v1alpha1"
    14  	internalcapi "github.com/verrazzano/verrazzano/cluster-operator/internal/capi"
    15  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    16  	"k8s.io/apimachinery/pkg/types"
    17  
    18  	"github.com/Jeffail/gabs/v2"
    19  	cons "github.com/verrazzano/verrazzano/pkg/constants"
    20  	"github.com/verrazzano/verrazzano/pkg/httputil"
    21  	"github.com/verrazzano/verrazzano/pkg/log/vzlog"
    22  	"github.com/verrazzano/verrazzano/pkg/mcconstants"
    23  	"github.com/verrazzano/verrazzano/pkg/rancherutil"
    24  	"github.com/verrazzano/verrazzano/platform-operator/constants"
    25  	corev1 "k8s.io/api/core/v1"
    26  	"k8s.io/apimachinery/pkg/api/equality"
    27  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    28  	"k8s.io/apimachinery/pkg/runtime/schema"
    29  	"k8s.io/apimachinery/pkg/util/json"
    30  	"k8s.io/apimachinery/pkg/util/yaml"
    31  	"sigs.k8s.io/controller-runtime/pkg/client"
    32  	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    33  )
    34  
    35  const (
    36  	rancherNamespace   = "cattle-system"
    37  	rancherIngressName = "rancher"
    38  	rancherTLSSecret   = "tls-rancher-ingress" //nolint:gosec //#gosec G101
    39  
    40  	clusterPath          = "/v3/cluster"
    41  	clustersPath         = "/v3/clusters"
    42  	clustersByNamePath   = "/v3/clusters?name="
    43  	clusterRegTokenPath  = "/v3/clusterregistrationtoken" //nolint:gosec //#gosec G101
    44  	manifestPath         = "/v3/import/"
    45  	loginPath            = "/v3-public/localProviders/local?action=login"
    46  	secretPathTemplate   = "/api/v1/namespaces/%s/secrets/%s" //nolint:gosec //#gosec G101
    47  	secretCreateTemplate = "/api/v1/namespaces/%s/secrets"    //nolint:gosec //#gosec G101
    48  
    49  	k8sClustersPath = "/k8s/clusters/"
    50  
    51  	rancherClusterStateActive   = "active"
    52  	rancherClusterStateInactive = "inactive"
    53  )
    54  
    55  type RancherCluster struct {
    56  	Name string
    57  	ID   string
    58  }
    59  
    60  // RegisterManagedClusterWithRancher registers a managed cluster with Rancher and returns a chunk of YAML that
    61  // must be applied on the managed cluster to complete the registration.
    62  func RegisterManagedClusterWithRancher(rc *rancherutil.RancherConfig, clusterName string, rancherClusterID string, log vzlog.VerrazzanoLogger) (string, string, error) {
    63  	clusterID := rancherClusterID
    64  	var err error
    65  	if clusterID == "" {
    66  		log.Oncef("Registering managed cluster in Rancher with name: %s", clusterName)
    67  		clusterID, err = ImportClusterToRancher(rc, clusterName, nil, log)
    68  		if err != nil {
    69  			log.Errorf("Failed to import cluster to Rancher: %v", err)
    70  			return "", "", err
    71  		}
    72  	}
    73  
    74  	log.Oncef("Getting registration YAML from Rancher for cluster %s with id %s", clusterName, clusterID)
    75  	regYAML, err := getRegistrationYAMLFromRancher(rc, clusterID, log)
    76  	if err != nil {
    77  		log.Errorf("Failed to get registration YAML from Rancher: %v", err)
    78  		return "", "", err
    79  	}
    80  
    81  	return regYAML, clusterID, nil
    82  }
    83  
    84  // ImportClusterToRancher uses the Rancher API to import the cluster. The cluster will show as "pending" until the registration
    85  // YAML is applied on the managed cluster.
    86  func ImportClusterToRancher(rc *rancherutil.RancherConfig, clusterName string, labels map[string]string, log vzlog.VerrazzanoLogger) (string, error) {
    87  	action := http.MethodPost
    88  
    89  	payload, err := makeClusterPayload(clusterName, labels)
    90  	if err != nil {
    91  		return "", err
    92  	}
    93  
    94  	reqURL := rc.BaseURL + clusterPath
    95  	headers := map[string]string{"Content-Type": "application/json"}
    96  	headers["Authorization"] = "Bearer " + rc.APIAccessToken
    97  
    98  	response, responseBody, err := rancherutil.SendRequest(action, reqURL, headers, payload, rc, log)
    99  
   100  	if response != nil && response.StatusCode == http.StatusUnprocessableEntity {
   101  		// if we've already imported this cluster, we get an HTTP 422, so attempt to fetch the existing cluster
   102  		// and get the cluster ID from the response
   103  		log.Debugf("Cluster %s already registered with Rancher, attempting to fetch it", clusterName)
   104  		clusterID, err := GetClusterIDFromRancher(rc, clusterName, log)
   105  		if err != nil {
   106  			return "", err
   107  		}
   108  		return clusterID, nil
   109  	}
   110  
   111  	if err != nil {
   112  		return "", err
   113  	}
   114  
   115  	err = httputil.ValidateResponseCode(response, http.StatusCreated)
   116  	if err != nil {
   117  		return "", err
   118  	}
   119  	log.Oncef("Successfully registered managed cluster in Rancher with name: %s", clusterName)
   120  
   121  	return httputil.ExtractFieldFromResponseBodyOrReturnError(responseBody, "id", "unable to find cluster id in Rancher response")
   122  }
   123  
   124  // makeClusterPayload returns the payload for Rancher cluster creation, given a cluster name
   125  // and labels to apply to it
   126  func makeClusterPayload(clusterName string, labels map[string]string) (string, error) {
   127  	labelsJSONString, err := makeLabelsJSONString(labels)
   128  	if err != nil {
   129  		return "", err
   130  	}
   131  	payload := `{"type": "cluster",
   132  		"name":"` + clusterName + `",
   133  		"dockerRootDir": "/var/lib/docker",
   134  		"enableClusterAlerting": "false",
   135  		"enableClusterMonitoring": "false",
   136  		"enableNetworkPolicy": "false"`
   137  
   138  	if len(labelsJSONString) > 0 {
   139  		payload = fmt.Sprintf(`%s, "labels": %s }`, payload, labelsJSONString)
   140  	} else {
   141  		payload = fmt.Sprintf("%s}", payload)
   142  	}
   143  	return payload, nil
   144  }
   145  
   146  func makeLabelsJSONString(labels map[string]string) (string, error) {
   147  	if len(labels) == 0 {
   148  		return "", nil
   149  	}
   150  	labelsJSON, err := json.Marshal(labels)
   151  	if err != nil {
   152  		return "", err
   153  	}
   154  	return string(labelsJSON), nil
   155  }
   156  
   157  // DeleteClusterFromRancher uses the Rancher API to delete a cluster in Rancher.
   158  func DeleteClusterFromRancher(rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) (bool, error) {
   159  	action := http.MethodDelete
   160  	reqURL := rc.BaseURL + clustersPath + "/" + clusterID
   161  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   162  
   163  	response, _, err := rancherutil.SendRequest(action, reqURL, headers, "", rc, log)
   164  
   165  	if response != nil && response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNotFound {
   166  		return false, fmt.Errorf("tried to delete cluster from Rancher but failed, response code: %d", response.StatusCode)
   167  	}
   168  
   169  	if err != nil {
   170  		return false, err
   171  	}
   172  
   173  	log.Oncef("Successfully deleted cluster %s from Rancher", clusterID)
   174  	return true, nil
   175  }
   176  
   177  // GetClusterIDFromRancher attempts to fetch the cluster from Rancher by name and pull out the cluster ID
   178  func GetClusterIDFromRancher(rc *rancherutil.RancherConfig, clusterName string, log vzlog.VerrazzanoLogger) (string, error) {
   179  	action := http.MethodGet
   180  
   181  	reqURL := rc.BaseURL + clustersByNamePath + clusterName
   182  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   183  
   184  	response, responseBody, err := rancherutil.SendRequest(action, reqURL, headers, "", rc, log)
   185  
   186  	if response != nil && response.StatusCode != http.StatusOK {
   187  		return "", fmt.Errorf("tried to get cluster from Rancher but failed, response code: %d", response.StatusCode)
   188  	}
   189  
   190  	if err != nil {
   191  		return "", err
   192  	}
   193  
   194  	return httputil.ExtractFieldFromResponseBodyOrReturnError(responseBody, "data.0.id", "unable to find clusterId in Rancher response")
   195  }
   196  
   197  // GetAllClustersInRancher returns cluster information for every cluster registered with Rancher
   198  func GetAllClustersInRancher(rc *rancherutil.RancherConfig, log vzlog.VerrazzanoLogger) ([]RancherCluster, []byte, error) {
   199  	reqURL := rc.BaseURL + clustersPath
   200  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   201  
   202  	hash := md5.New() //nolint:gosec //#gosec G401
   203  	clusters := []RancherCluster{}
   204  	for {
   205  		response, responseBody, err := rancherutil.SendRequest(http.MethodGet, reqURL, headers, "", rc, log)
   206  		if response != nil && response.StatusCode != http.StatusOK {
   207  			return nil, nil, fmt.Errorf("Unable to get clusters from Rancher, response code: %d", response.StatusCode)
   208  		}
   209  
   210  		if err != nil {
   211  			return nil, nil, err
   212  		}
   213  
   214  		// parse the response and iterate over the items
   215  		jsonString, err := gabs.ParseJSON([]byte(responseBody))
   216  		if err != nil {
   217  			return nil, nil, err
   218  		}
   219  
   220  		var items []interface{}
   221  		var ok bool
   222  		if items, ok = jsonString.Path("data").Data().([]interface{}); !ok {
   223  			return nil, nil, fmt.Errorf("Unable to find expected data in Rancher clusters response: %v", jsonString)
   224  		}
   225  
   226  		for _, item := range items {
   227  			var i map[string]interface{}
   228  			var ok bool
   229  			if i, ok = item.(map[string]interface{}); !ok {
   230  				log.Infof("Expected item to be of type 'map[string]interface{}': %s", responseBody)
   231  				continue
   232  			}
   233  			var name, id interface{}
   234  			if name, ok = i["name"]; !ok {
   235  				log.Infof("Expected to find 'name' field in Rancher cluster data: %s", responseBody)
   236  				continue
   237  			}
   238  			if id, ok = i["id"]; !ok {
   239  				log.Infof("Expected to find 'id' field in Rancher cluster data: %s", responseBody)
   240  				continue
   241  			}
   242  			cluster := RancherCluster{Name: name.(string), ID: id.(string)}
   243  			clusters = append(clusters, cluster)
   244  		}
   245  
   246  		// add this response body to the hash
   247  		io.WriteString(hash, responseBody)
   248  
   249  		// if there is a "next page" link then use that to make another request
   250  		if reqURL, err = httputil.ExtractFieldFromResponseBodyOrReturnError(responseBody, "pagination.next", ""); err != nil {
   251  			break
   252  		}
   253  	}
   254  
   255  	// unfortunately Rancher does not support ETags, so we return a hash of the response bodies which allows the caller to know if
   256  	// there were any changes to the clusters
   257  	return clusters, hash.Sum(nil), nil
   258  }
   259  
   260  // isManagedClusterActiveInRancher returns true if the managed cluster is active
   261  func isManagedClusterActiveInRancher(rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) (bool, error) {
   262  	reqURL := rc.BaseURL + clustersPath + "/" + clusterID
   263  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   264  
   265  	response, responseBody, err := rancherutil.SendRequest(http.MethodGet, reqURL, headers, "", rc, log)
   266  
   267  	if response != nil && response.StatusCode != http.StatusOK {
   268  		return false, fmt.Errorf("tried to get cluster from Rancher but failed, response code: %d", response.StatusCode)
   269  	}
   270  
   271  	if err != nil {
   272  		return false, err
   273  	}
   274  
   275  	state, err := httputil.ExtractFieldFromResponseBodyOrReturnError(responseBody, "state", "unable to find cluster state in Rancher response")
   276  	if err != nil {
   277  		return false, err
   278  	}
   279  	agentImage, err := httputil.ExtractFieldFromResponseBodyOrReturnError(responseBody, "agentImage", "unable to find agent image in Rancher response")
   280  	if err != nil {
   281  		return false, err
   282  	}
   283  
   284  	// Rancher temporarily sets the state of a new cluster to "active" before setting it to "pending", so we also check for the "agentImage" field
   285  	// to know that the cluster is really active
   286  	return state == rancherClusterStateActive && len(agentImage) > 0, nil
   287  }
   288  
   289  // getCACertFromManagedCluster attempts to get the CA cert from the managed cluster using the Rancher API proxy. It first checks for
   290  // the Rancher TLS secret and if that is not found it looks for the Verrazzano system TLS secret.
   291  func getCACertFromManagedCluster(rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) (string, error) {
   292  	// first look for the Rancher TLS secret
   293  	caCert, err := GetCACertFromManagedClusterSecret(rc, clusterID, rancherNamespace, cons.RancherTLSCA, cons.RancherTLSCAKey, log)
   294  	if err != nil {
   295  		return "", err
   296  	}
   297  
   298  	if caCert != "" {
   299  		return caCert, nil
   300  	}
   301  
   302  	// didn't find the Rancher secret so next look for the verrazzano-tls secret
   303  	caCert, err = GetCACertFromManagedClusterSecret(rc, clusterID, cons.VerrazzanoSystemNamespace, constants.VerrazzanoIngressSecret, mcconstants.CaCrtKey, log)
   304  	if err != nil {
   305  		return "", err
   306  	}
   307  
   308  	if caCert != "" {
   309  		return caCert, nil
   310  	}
   311  
   312  	return "", nil
   313  }
   314  
   315  // GetCACertFromManagedClusterSecret attempts to get the CA cert from a secret on the managed cluster using the Rancher API proxy
   316  func GetCACertFromManagedClusterSecret(rc *rancherutil.RancherConfig, clusterID, namespace, secretName, secretKey string, log vzlog.VerrazzanoLogger) (string, error) {
   317  	const k8sAPISecretPattern = "%s/api/v1/namespaces/%s/secrets/%s" //nolint:gosec //#gosec G101
   318  
   319  	// use the Rancher API proxy on the managed cluster to fetch the secret
   320  	baseReqURL := rc.BaseURL + k8sClustersPath + clusterID
   321  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   322  
   323  	reqURL := fmt.Sprintf(k8sAPISecretPattern, baseReqURL, namespace, secretName)
   324  	response, responseBody, err := rancherutil.SendRequest(http.MethodGet, reqURL, headers, "", rc, log)
   325  
   326  	if response != nil {
   327  		if response.StatusCode == http.StatusNotFound {
   328  			return "", nil
   329  		}
   330  		if response.StatusCode != http.StatusOK {
   331  			return "", fmt.Errorf("tried to get managed cluster CA cert %s/%s from Rancher but failed, response code: %d", namespace, secretName, response.StatusCode)
   332  		}
   333  	}
   334  	if err != nil {
   335  		return "", err
   336  	}
   337  
   338  	// parse the response and pull out the secretKey value from the secret data
   339  	jsonString, err := gabs.ParseJSON([]byte(responseBody))
   340  	if err != nil {
   341  		return "", err
   342  	}
   343  
   344  	if data, ok := jsonString.Path("data").Data().(map[string]interface{}); ok {
   345  		if caCert, ok := data[secretKey].(string); ok {
   346  			return caCert, nil
   347  		}
   348  	}
   349  
   350  	return "", nil
   351  }
   352  
   353  // isNamespaceCreated attempts to ascertain whether the given namesapce is created in the cluster
   354  func isNamespaceCreated(vmc *v1alpha1.VerrazzanoManagedCluster, r *VerrazzanoManagedClusterReconciler, clusterID, namespace string) (bool, error) {
   355  	if vmc.Status.ClusterRef != nil {
   356  		cluster := &unstructured.Unstructured{}
   357  		cluster.SetGroupVersionKind(internalcapi.GVKCAPICluster)
   358  		err := r.Get(context.TODO(), types.NamespacedName{Namespace: vmc.Status.ClusterRef.Namespace, Name: vmc.Status.ClusterRef.Name}, cluster)
   359  		if err != nil && !apierrors.IsNotFound(err) {
   360  			return false, err
   361  		}
   362  		workloadClient, err := r.getWorkloadClusterClient(cluster)
   363  		if err != nil {
   364  			r.log.Errorf("Error getting workload cluster %s client: %v", cluster.GetName(), err)
   365  			return false, err
   366  		}
   367  		err = workloadClient.Get(context.TODO(),
   368  			types.NamespacedName{Name: constants.VerrazzanoSystemNamespace},
   369  			&corev1.Namespace{})
   370  		if err != nil {
   371  			if !apierrors.IsNotFound(err) {
   372  				return false, err
   373  			}
   374  			return false, nil
   375  		}
   376  		return true, nil
   377  	}
   378  
   379  	const k8sAPISecretPattern = "%s/api/v1/namespaces/%s" //nolint:gosec //#gosec G101
   380  
   381  	rc, err := rancherutil.NewAdminRancherConfig(r.Client, r.RancherIngressHost, r.log)
   382  	if err != nil || rc == nil {
   383  		return false, err
   384  	}
   385  
   386  	isActive, err := isManagedClusterActiveInRancher(rc, clusterID, r.log)
   387  	if err != nil || !isActive {
   388  		return false, err
   389  	}
   390  
   391  	// use the Rancher API proxy on the managed cluster to fetch the secret
   392  	baseReqURL := rc.BaseURL + k8sClustersPath + clusterID
   393  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   394  
   395  	reqURL := fmt.Sprintf(k8sAPISecretPattern, baseReqURL, namespace)
   396  	response, _, err := rancherutil.SendRequest(http.MethodGet, reqURL, headers, "", rc, r.log)
   397  
   398  	if response != nil {
   399  		if response.StatusCode == http.StatusNotFound {
   400  			return false, nil
   401  		}
   402  		if response.StatusCode != http.StatusOK {
   403  			return false, fmt.Errorf("tried to get namespace %s via Rancher from clsuter %s but failed, response code: %d", namespace, clusterID, response.StatusCode)
   404  		}
   405  	}
   406  	if err != nil {
   407  		return false, err
   408  	}
   409  
   410  	return true, nil
   411  }
   412  
   413  // getRegistrationYAMLFromRancher creates a registration token in Rancher for the managed cluster and uses the
   414  // returned token to fetch the registration (manifest) YAML.
   415  func getRegistrationYAMLFromRancher(rc *rancherutil.RancherConfig, rancherClusterID string, log vzlog.VerrazzanoLogger) (string, error) {
   416  	headers := map[string]string{"Content-Type": "application/json"}
   417  	headers["Authorization"] = "Bearer " + rc.APIAccessToken
   418  
   419  	var token string
   420  	token, err := getRegistrationTokenFromRancher(rc, rancherClusterID, log)
   421  	if err != nil {
   422  		log.Oncef("Unable to fetch existing cluster registration token for cluster: %s with reason: %v, continuing to create a new token", rancherClusterID, err)
   423  	}
   424  
   425  	if token == "" {
   426  		action := http.MethodPost
   427  		payload := `{"type": "clusterRegistrationToken", "clusterId": "` + rancherClusterID + `"}`
   428  		reqURL := rc.BaseURL + clusterRegTokenPath
   429  
   430  		log.Infof("Creating cluster registration token for cluster %s", rancherClusterID)
   431  		response, manifestContent, err := rancherutil.SendRequest(action, reqURL, headers, payload, rc, log)
   432  		if err != nil {
   433  			return "", err
   434  		}
   435  
   436  		err = httputil.ValidateResponseCode(response, http.StatusCreated)
   437  		if err != nil {
   438  			return "", err
   439  		}
   440  
   441  		// get the manifest token from the response, construct a URL, and fetch its contents
   442  		token, err = httputil.ExtractFieldFromResponseBodyOrReturnError(manifestContent, "token", "unable to find manifest token in Rancher response")
   443  		if err != nil {
   444  			return "", err
   445  		}
   446  	}
   447  	// Rancher 2.5.x added the cluster ID to the manifest URL.
   448  	manifestURL := rc.BaseURL + manifestPath + token + "_" + rancherClusterID + ".yaml"
   449  
   450  	action := http.MethodGet
   451  	response, manifestContent, err := rancherutil.SendRequest(action, manifestURL, headers, "", rc, log)
   452  
   453  	if err != nil {
   454  		return "", err
   455  	}
   456  
   457  	err = httputil.ValidateResponseCode(response, http.StatusOK)
   458  	if err != nil {
   459  		return "", err
   460  	}
   461  
   462  	return manifestContent, nil
   463  }
   464  
   465  type ClusterRegistrationTokens struct {
   466  	ClusterID string `json:"clusterId"`
   467  	State     string `json:"state"`
   468  	Token     string `json:"token"`
   469  }
   470  
   471  func getRegistrationTokenFromRancher(rc *rancherutil.RancherConfig, rancherClusterID string, log vzlog.VerrazzanoLogger) (string, error) {
   472  
   473  	action := http.MethodGet
   474  	reqURL := rc.BaseURL + clusterRegTokenPath + "?state=active&clusterId=" + rancherClusterID
   475  	headers := map[string]string{"Content-Type": "application/json"}
   476  	headers["Authorization"] = "Bearer " + rc.APIAccessToken
   477  
   478  	response, manifestContent, err := rancherutil.SendRequest(action, reqURL, headers, "{}", rc, log)
   479  	if err != nil {
   480  		return "", err
   481  	}
   482  
   483  	err = httputil.ValidateResponseCode(response, http.StatusOK)
   484  	if err != nil {
   485  		return "", err
   486  	}
   487  
   488  	data, err := httputil.ExtractFieldFromResponseBodyOrReturnError(manifestContent, "data", "unable to find data token in Rancher response")
   489  	if err != nil {
   490  		return "", err
   491  	}
   492  
   493  	var items []ClusterRegistrationTokens
   494  	json.Unmarshal([]byte(data), &items)
   495  	for _, item := range items {
   496  		if item.ClusterID == rancherClusterID && item.State == "active" {
   497  			log.Oncef("ClusterRegistrationToken exists for the cluster %s", rancherClusterID)
   498  			return item.Token, nil
   499  		}
   500  	}
   501  
   502  	log.Oncef("No existing ClusterRegistrationToken found for cluster %s", rancherClusterID)
   503  	return "", nil
   504  }
   505  
   506  // createOrUpdateSecretRancherProxy simulates the controllerutil create or update function through the Rancher Proxy API for secrets
   507  func createOrUpdateSecretRancherProxy(secret *corev1.Secret, rc *rancherutil.RancherConfig, clusterID string, f controllerutil.MutateFn, log vzlog.VerrazzanoLogger) (controllerutil.OperationResult, error) {
   508  	log.Debugf("Creating or Updating Secret %s/%s", secret.GetNamespace(), secret.GetName())
   509  	if err := rancherSecretGet(secret, rc, clusterID, log); err != nil {
   510  		if !apierrors.IsNotFound(err) {
   511  			return controllerutil.OperationResultNone, err
   512  		}
   513  		if err := rancherSecretMutate(f, secret, log); err != nil {
   514  			return controllerutil.OperationResultNone, err
   515  		}
   516  		if err := rancherSecretCreate(secret, rc, clusterID, log); err != nil {
   517  			return controllerutil.OperationResultNone, err
   518  		}
   519  		return controllerutil.OperationResultCreated, nil
   520  	}
   521  
   522  	existingSec := secret.DeepCopyObject()
   523  	if err := rancherSecretMutate(f, secret, log); err != nil {
   524  		return controllerutil.OperationResultNone, err
   525  	}
   526  	if equality.Semantic.DeepEqual(existingSec, secret) {
   527  		return controllerutil.OperationResultNone, nil
   528  	}
   529  	if err := rancherSecretUpdate(secret, rc, clusterID, log); err != nil {
   530  		return controllerutil.OperationResultNone, err
   531  	}
   532  	return controllerutil.OperationResultUpdated, nil
   533  }
   534  
   535  // rancherSecretMutate mutates the rancher secret from the given Mutate function
   536  func rancherSecretMutate(f controllerutil.MutateFn, secret *corev1.Secret, log vzlog.VerrazzanoLogger) error {
   537  	key := client.ObjectKeyFromObject(secret)
   538  	if err := f(); err != nil {
   539  		return err
   540  	}
   541  	if newKey := client.ObjectKeyFromObject(secret); key != newKey {
   542  		return log.ErrorfNewErr("MutateFn cannot mutate secret name and/or secret namespace")
   543  	}
   544  	return nil
   545  }
   546  
   547  // rancherSecretGet simulates a client get request through the Rancher proxy for secrets
   548  func rancherSecretGet(secret *corev1.Secret, rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) error {
   549  	if secret == nil {
   550  		return log.ErrorNewErr("Failed to get secret, nil value passed to get request")
   551  	}
   552  	reqURL := constructSecretURL(secret, rc.Host, clusterID, false)
   553  	headers := map[string]string{"Authorization": "Bearer " + rc.APIAccessToken}
   554  	resp, body, err := rancherutil.SendRequest(http.MethodGet, reqURL, headers, "", rc, log)
   555  	if err != nil && (resp == nil || resp.StatusCode != 404) {
   556  		return err
   557  	}
   558  	if resp == nil {
   559  		return log.ErrorfNewErr("Failed to find response from GET request %s", secret.GetNamespace(), secret.GetName(), reqURL)
   560  	}
   561  	if resp.StatusCode == http.StatusNotFound {
   562  		return apierrors.NewNotFound(schema.ParseGroupResource("Secret"), secret.GetName())
   563  	}
   564  	if resp.StatusCode != http.StatusOK {
   565  		return log.ErrorfNewErr("Failed to get secret %s/%s from GET request %s with code %d", secret.GetNamespace(), secret.GetName(), reqURL, resp.StatusCode)
   566  	}
   567  
   568  	// Unmarshall the response body into the secret object, simulating a typical Get request
   569  	err = yaml.Unmarshal([]byte(body), secret)
   570  	if err != nil {
   571  		return log.ErrorfNewErr("Failed to unmarshall response body into secret %s/%s from GET request %s: %v", secret.GetNamespace(), secret.GetName(), reqURL, err)
   572  	}
   573  	return nil
   574  }
   575  
   576  // rancherSecretCreate simulates a client create request through the Rancher proxy for secrets
   577  func rancherSecretCreate(secret *corev1.Secret, rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) error {
   578  	if secret == nil {
   579  		return log.ErrorNewErr("Failed to create secret, nil value passed to create request")
   580  	}
   581  	reqURL := constructSecretURL(secret, rc.Host, clusterID, true)
   582  	payload, err := json.Marshal(secret)
   583  	if err != nil {
   584  		return log.ErrorfNewErr("Failed to marshall secret %s/%s: %v", secret.GetNamespace(), secret.GetName(), err)
   585  	}
   586  	headers := map[string]string{
   587  		"Authorization": "Bearer " + rc.APIAccessToken,
   588  		"Content-Type":  "application/json",
   589  	}
   590  	resp, _, err := rancherutil.SendRequest(http.MethodPost, reqURL, headers, string(payload), rc, log)
   591  	if err != nil {
   592  		return err
   593  	}
   594  
   595  	if resp == nil {
   596  		return log.ErrorfNewErr("Failed to find response from POST request %s", secret.GetNamespace(), secret.GetName(), reqURL)
   597  	}
   598  	if resp.StatusCode != http.StatusCreated {
   599  		return log.ErrorfNewErr("Failed to create secret %s/%s from POST request %s with code %d", secret.GetNamespace(), secret.GetName(), reqURL, resp.StatusCode)
   600  	}
   601  	return nil
   602  }
   603  
   604  // rancherSecretUpdate simulates a client update request through the Rancher proxy for secrets
   605  func rancherSecretUpdate(secret *corev1.Secret, rc *rancherutil.RancherConfig, clusterID string, log vzlog.VerrazzanoLogger) error {
   606  	if secret == nil {
   607  		return log.ErrorNewErr("Failed to update secret, nil value passed to update request")
   608  	}
   609  	reqURL := constructSecretURL(secret, rc.Host, clusterID, false)
   610  	payload, err := json.Marshal(secret)
   611  	if err != nil {
   612  		return log.ErrorfNewErr("Failed to marshall secret %s/%s: %v", secret.GetNamespace(), secret.GetName(), err)
   613  	}
   614  	headers := map[string]string{
   615  		"Authorization": "Bearer " + rc.APIAccessToken,
   616  		"Content-Type":  "application/json",
   617  	}
   618  	resp, _, err := rancherutil.SendRequest(http.MethodPut, reqURL, headers, string(payload), rc, log)
   619  	if err != nil {
   620  		return err
   621  	}
   622  
   623  	if resp == nil {
   624  		return log.ErrorfNewErr("Failed to find response from PUT request %s", secret.GetNamespace(), secret.GetName(), reqURL)
   625  	}
   626  	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
   627  		return log.ErrorfNewErr("Failed to create secret %s/%s from PUT request %s with code %d", secret.GetNamespace(), secret.GetName(), reqURL, resp.StatusCode)
   628  	}
   629  	return nil
   630  }
   631  
   632  // constructSecretURL returns a formatted url string from path requirements and objects
   633  func constructSecretURL(secret *corev1.Secret, host, clusterID string, create bool) string {
   634  	if create {
   635  		return "https://" + host + k8sClustersPath + clusterID + fmt.Sprintf(secretCreateTemplate, secret.GetNamespace())
   636  	}
   637  	return "https://" + host + k8sClustersPath + clusterID + fmt.Sprintf(secretPathTemplate, secret.GetNamespace(), secret.GetName())
   638  }