github.com/oam-dev/cluster-gateway@v1.9.0/pkg/addon/controllers/health.go (about)

     1  package controllers
     2  
     3  import (
     4  	"context"
     5  	"time"
     6  
     7  	"k8s.io/apimachinery/pkg/api/meta"
     8  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
     9  	"k8s.io/client-go/kubernetes"
    10  	"k8s.io/client-go/rest"
    11  	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
    12  	ctrl "sigs.k8s.io/controller-runtime"
    13  	"sigs.k8s.io/controller-runtime/pkg/client"
    14  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    15  
    16  	multicluster "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/transport"
    17  	"github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1"
    18  	"github.com/oam-dev/cluster-gateway/pkg/common"
    19  	"github.com/oam-dev/cluster-gateway/pkg/event"
    20  	"github.com/oam-dev/cluster-gateway/pkg/generated/clientset/versioned"
    21  )
    22  
    23  var (
    24  	healthLog = ctrl.Log.WithName("ClusterGatewayHealthProber")
    25  )
    26  var _ reconcile.Reconciler = &ClusterGatewayHealthProber{}
    27  
    28  type ClusterGatewayHealthProber struct {
    29  	multiClusterRestClient rest.Interface
    30  	gatewayClient          versioned.Interface
    31  	runtimeClient          client.Client
    32  }
    33  
    34  func SetupClusterGatewayHealthProberWithManager(mgr ctrl.Manager) error {
    35  	gatewayClient, err := versioned.NewForConfig(mgr.GetConfig())
    36  	if err != nil {
    37  		return err
    38  	}
    39  	copied := rest.CopyConfig(mgr.GetConfig())
    40  	copied.WrapTransport = multicluster.NewClusterGatewayRoundTripper
    41  	multiClusterClient, err := kubernetes.NewForConfig(copied)
    42  	if err != nil {
    43  		return err
    44  	}
    45  	prober := &ClusterGatewayHealthProber{
    46  		multiClusterRestClient: multiClusterClient.Discovery().RESTClient(),
    47  		gatewayClient:          gatewayClient,
    48  		runtimeClient:          mgr.GetClient(),
    49  	}
    50  	ch, handler := event.AddOnHealthResyncHandler(mgr.GetClient(), time.Second)
    51  	return ctrl.NewControllerManagedBy(mgr).
    52  		For(&addonv1alpha1.ManagedClusterAddOn{}).
    53  		WatchesRawSource(ch, handler).
    54  		Complete(prober)
    55  }
    56  
    57  func (c *ClusterGatewayHealthProber) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
    58  	if request.Name != common.AddonName {
    59  		return reconcile.Result{}, nil
    60  	}
    61  	clusterName := request.Namespace
    62  	gw, err := c.gatewayClient.ClusterV1alpha1().
    63  		ClusterGateways().
    64  		GetHealthiness(ctx, clusterName, metav1.GetOptions{})
    65  	if err != nil {
    66  		return reconcile.Result{}, err
    67  	}
    68  	resp, healthErr := c.multiClusterRestClient.
    69  		Get().
    70  		AbsPath("healthz").
    71  		DoRaw(multicluster.WithMultiClusterContext(context.TODO(), clusterName))
    72  	healthy := string(resp) == "ok" && healthErr == nil
    73  	if !healthy {
    74  		healthErrMsg := ""
    75  		if healthErr != nil {
    76  			healthErrMsg = healthErr.Error()
    77  		}
    78  		healthLog.Info("Cluster unhealthy", "cluster", clusterName,
    79  			"body", string(resp),
    80  			"error", healthErrMsg)
    81  	}
    82  	if healthy != gw.Status.Healthy {
    83  		gw.Status.Healthy = healthy
    84  		if !healthy {
    85  			if healthErr != nil {
    86  				gw.Status.HealthyReason = v1alpha1.HealthyReasonType(healthErr.Error())
    87  			}
    88  		} else {
    89  			gw.Status.HealthyReason = ""
    90  		}
    91  		healthLog.Info("Updating cluster healthiness",
    92  			"cluster", clusterName,
    93  			"healthy", healthy)
    94  		_, err = c.gatewayClient.ClusterV1alpha1().
    95  			ClusterGateways().
    96  			UpdateHealthiness(ctx, gw, metav1.UpdateOptions{})
    97  		if err != nil {
    98  			return reconcile.Result{}, err
    99  		}
   100  	}
   101  
   102  	addon := &addonv1alpha1.ManagedClusterAddOn{}
   103  	if err := c.runtimeClient.Get(ctx, request.NamespacedName, addon); err != nil {
   104  		return reconcile.Result{}, err
   105  	}
   106  	if healthy != meta.IsStatusConditionTrue(addon.Status.Conditions, addonv1alpha1.ManagedClusterAddOnConditionAvailable) {
   107  		healthLog.Info("Updating addon healthiness",
   108  			"cluster", clusterName,
   109  			"healthy", healthy)
   110  		healthyStatus := metav1.ConditionTrue
   111  		if !healthy {
   112  			healthyStatus = metav1.ConditionFalse
   113  		}
   114  		if healthy {
   115  			meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{
   116  				Type:    addonv1alpha1.ManagedClusterAddOnConditionAvailable,
   117  				Status:  healthyStatus,
   118  				Reason:  "SuccessfullyProbedHealthz",
   119  				Message: "Returned OK",
   120  			})
   121  		} else {
   122  			errMsg := "Unknown"
   123  			if healthErr != nil {
   124  				errMsg = healthErr.Error()
   125  			} else if len(string(resp)) > 0 {
   126  				errMsg = string(resp)
   127  			}
   128  			meta.SetStatusCondition(&addon.Status.Conditions, metav1.Condition{
   129  				Type:    addonv1alpha1.ManagedClusterAddOnConditionAvailable,
   130  				Status:  healthyStatus,
   131  				Reason:  "FailedProbingHealthz",
   132  				Message: errMsg,
   133  			})
   134  		}
   135  		if err := c.runtimeClient.Status().Update(ctx, addon); err != nil {
   136  			return reconcile.Result{}, err
   137  		}
   138  	}
   139  
   140  	if !healthy {
   141  		return reconcile.Result{
   142  			Requeue:      true,
   143  			RequeueAfter: 5 * time.Second,
   144  		}, nil
   145  	}
   146  	return reconcile.Result{}, nil
   147  }