github.com/operator-framework/operator-lifecycle-manager@v0.30.0/pkg/controller/operators/olm/apiservices.go (about) 1 package olm 2 3 import ( 4 "context" 5 "fmt" 6 7 hashutil "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/kubernetes/pkg/util/hash" 8 log "github.com/sirupsen/logrus" 9 appsv1 "k8s.io/api/apps/v1" 10 apierrors "k8s.io/apimachinery/pkg/api/errors" 11 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 12 "k8s.io/apimachinery/pkg/labels" 13 utilerrors "k8s.io/apimachinery/pkg/util/errors" 14 15 "github.com/operator-framework/api/pkg/operators/v1alpha1" 16 "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/certs" 17 olmerrors "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/errors" 18 "github.com/operator-framework/operator-lifecycle-manager/pkg/controller/install" 19 "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/ownerutil" 20 ) 21 22 const ( 23 // Name of packageserver API service 24 PackageserverName = "v1.packages.operators.coreos.com" 25 ) 26 27 // apiServiceResourceErrorActionable returns true if OLM can do something about any one 28 // of the apiService errors in errs; otherwise returns false 29 // 30 // This method can be used to determine if a CSV in a failed state due to APIService 31 // issues can resolve them by reinstalling 32 func (a *Operator) apiServiceResourceErrorActionable(err error) bool { 33 filtered := utilerrors.FilterOut(err, func(e error) bool { 34 _, unadoptable := e.(olmerrors.UnadoptableError) 35 return !unadoptable 36 }) 37 actionable := filtered == nil 38 39 return actionable 40 } 41 42 // checkAPIServiceResources checks if all expected generated resources for the given APIService exist 43 func (a *Operator) checkAPIServiceResources(csv *v1alpha1.ClusterServiceVersion, hashFunc certs.PEMHash) error { 44 logger := log.WithFields(log.Fields{ 45 "csv": csv.GetName(), 46 "namespace": csv.GetNamespace(), 47 }) 48 49 errs := []error{} 50 for _, desc := range csv.GetOwnedAPIServiceDescriptions() { 51 apiServiceName := desc.GetName() 52 logger := logger.WithFields(log.Fields{ 53 "apiservice": apiServiceName, 54 }) 55 56 apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(apiServiceName) 57 if err != nil { 58 logger.Warnf("could not retrieve generated APIService") 59 errs = append(errs, err) 60 continue 61 } 62 63 // Check if the APIService is adoptable 64 adoptable, err := install.IsAPIServiceAdoptable(a.lister, csv, apiService) 65 if err != nil { 66 logger.WithFields(log.Fields{"obj": "apiService", "labels": apiService.GetLabels()}).Errorf("adoption check failed - %v", err) 67 errs = append(errs, err) 68 return utilerrors.NewAggregate(errs) 69 } 70 71 if !adoptable { 72 logger.WithFields(log.Fields{"obj": "apiService", "labels": apiService.GetLabels()}).Errorf("adoption failed") 73 err := olmerrors.NewUnadoptableError("", apiServiceName) 74 logger.WithError(err).Warn("found unadoptable apiservice") 75 errs = append(errs, err) 76 return utilerrors.NewAggregate(errs) 77 } 78 79 serviceName := install.ServiceName(desc.DeploymentName) 80 service, err := a.lister.CoreV1().ServiceLister().Services(csv.GetNamespace()).Get(serviceName) 81 if err != nil { 82 logger.WithField("service", serviceName).Warnf("could not retrieve generated Service") 83 errs = append(errs, err) 84 continue 85 } 86 87 // Check if the APIService points to the correct service 88 if apiService.Spec.Service.Name != serviceName || apiService.Spec.Service.Namespace != csv.GetNamespace() { 89 logger.WithFields(log.Fields{"service": apiService.Spec.Service.Name, "serviceNamespace": apiService.Spec.Service.Namespace}).Warnf("APIService service reference mismatch") 90 errs = append(errs, fmt.Errorf("found APIService and service reference mismatch")) 91 continue 92 } 93 94 // Check if CA is Active 95 caBundle := apiService.Spec.CABundle 96 _, err = certs.PEMToCert(caBundle) 97 if err != nil { 98 logger.Warnf("could not convert APIService CA bundle to x509 cert") 99 errs = append(errs, err) 100 continue 101 } 102 103 // Check if serving cert is active 104 secretName := install.SecretName(serviceName) 105 secret, err := a.lister.CoreV1().SecretLister().Secrets(csv.GetNamespace()).Get(secretName) 106 if err != nil { 107 logger.WithField("secret", secretName).Warnf("could not retrieve generated Secret: %v", err) 108 errs = append(errs, err) 109 continue 110 } 111 _, err = certs.PEMToCert(secret.Data["tls.crt"]) 112 if err != nil { 113 logger.Warnf("could not convert serving cert to x509 cert") 114 errs = append(errs, err) 115 continue 116 } 117 118 // Check if CA hash matches expected 119 caHash := hashFunc(caBundle) 120 if hash, ok := secret.GetAnnotations()[install.OLMCAHashAnnotationKey]; !ok || hash != caHash { 121 logger.WithField("secret", secretName).Warnf("secret CA cert hash does not match expected") 122 errs = append(errs, fmt.Errorf("secret %s CA cert hash does not match expected", secretName)) 123 continue 124 } 125 126 // Ensure the existing Deployment has a matching CA hash annotation 127 deployment, err := a.lister.AppsV1().DeploymentLister().Deployments(csv.GetNamespace()).Get(desc.DeploymentName) 128 if apierrors.IsNotFound(err) || err != nil { 129 logger.WithField("deployment", desc.DeploymentName).Warnf("expected Deployment could not be retrieved") 130 errs = append(errs, err) 131 continue 132 } 133 if hash, ok := deployment.Spec.Template.GetAnnotations()[install.OLMCAHashAnnotationKey]; !ok || hash != caHash { 134 logger.WithField("deployment", desc.DeploymentName).Warnf("Deployment CA cert hash does not match expected") 135 errs = append(errs, fmt.Errorf("deployment %s CA cert hash does not match expected", desc.DeploymentName)) 136 continue 137 } 138 139 // Ensure the Deployment's ServiceAccount exists 140 serviceAccountName := deployment.Spec.Template.Spec.ServiceAccountName 141 if serviceAccountName == "" { 142 serviceAccountName = "default" 143 } 144 _, err = a.opClient.KubernetesInterface().CoreV1().ServiceAccounts(deployment.GetNamespace()).Get(context.TODO(), serviceAccountName, metav1.GetOptions{}) 145 if err != nil { 146 logger.WithError(err).WithField("serviceaccount", serviceAccountName).Warnf("could not retrieve ServiceAccount") 147 errs = append(errs, err) 148 } 149 150 if _, err := a.lister.RbacV1().RoleLister().Roles(secret.GetNamespace()).Get(secret.GetName()); err != nil { 151 logger.WithError(err).Warnf("could not retrieve role %s/%s", secret.GetNamespace(), secret.GetName()) 152 errs = append(errs, err) 153 } 154 if _, err := a.lister.RbacV1().RoleBindingLister().RoleBindings(secret.GetNamespace()).Get(secret.GetName()); err != nil { 155 logger.WithError(err).Warnf("could not retrieve role binding %s/%s", secret.GetNamespace(), secret.GetName()) 156 errs = append(errs, err) 157 } 158 if _, err := a.lister.RbacV1().ClusterRoleBindingLister().Get(install.AuthDelegatorClusterRoleBindingName(service.GetName())); err != nil { 159 logger.WithError(err).Warnf("could not retrieve auth delegator cluster role binding %s", install.AuthDelegatorClusterRoleBindingName(service.GetName())) 160 errs = append(errs, err) 161 } 162 if _, err := a.lister.RbacV1().RoleBindingLister().RoleBindings(install.KubeSystem).Get(install.AuthReaderRoleBindingName(service.GetName())); err != nil { 163 logger.WithError(err).Warnf("could not retrieve role binding %s/%s", install.KubeSystem, install.AuthReaderRoleBindingName(service.GetName())) 164 errs = append(errs, err) 165 } 166 } 167 168 return utilerrors.NewAggregate(errs) 169 } 170 171 func (a *Operator) areAPIServicesAvailable(csv *v1alpha1.ClusterServiceVersion) (bool, error) { 172 for _, desc := range csv.Spec.APIServiceDefinitions.Owned { 173 apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(desc.GetName()) 174 if apierrors.IsNotFound(err) { 175 a.logger.Debugf("APIRegistration APIService %s not found", desc.GetName()) 176 return false, nil 177 } 178 179 if err != nil { 180 return false, err 181 } 182 183 if !install.IsAPIServiceAvailable(apiService) { 184 a.logger.Debugf("APIService not available for %s", desc.GetName()) 185 return false, nil 186 } 187 188 if ok, err := a.isGVKRegistered(desc.Group, desc.Version, desc.Kind); !ok || err != nil { 189 a.logger.Debugf("%s.%s/%s not registered for %s", desc.Group, desc.Version, desc.Kind, desc.GetName()) 190 return false, err 191 } 192 } 193 194 return true, nil 195 } 196 197 // getAPIServiceCABundle returns the CA associated with an API service 198 func (a *Operator) getAPIServiceCABundle(csv *v1alpha1.ClusterServiceVersion, desc *v1alpha1.APIServiceDescription) ([]byte, error) { 199 apiServiceName := desc.GetName() 200 apiService, err := a.lister.APIRegistrationV1().APIServiceLister().Get(apiServiceName) 201 202 if err != nil { 203 return nil, fmt.Errorf("could not retrieve generated APIService: %v", err) 204 } 205 206 if len(apiService.Spec.CABundle) > 0 { 207 return apiService.Spec.CABundle, nil 208 } 209 210 return nil, fmt.Errorf("unable to find CA") 211 } 212 213 // getWebhookCABundle returns the CA associated with a webhook 214 func (a *Operator) getWebhookCABundle(csv *v1alpha1.ClusterServiceVersion, desc *v1alpha1.WebhookDescription) ([]byte, error) { 215 webhookLabels := ownerutil.OwnerLabel(csv, v1alpha1.ClusterServiceVersionKind) 216 webhookLabels[install.WebhookDescKey] = desc.GenerateName 217 webhookSelector := labels.SelectorFromSet(webhookLabels).String() 218 219 switch desc.Type { 220 case v1alpha1.MutatingAdmissionWebhook: 221 existingWebhooks, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().MutatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 222 if err != nil { 223 return nil, fmt.Errorf("could not retrieve generated MutatingWebhookConfiguration: %v", err) 224 } 225 226 if len(existingWebhooks.Items) > 0 { 227 return existingWebhooks.Items[0].Webhooks[0].ClientConfig.CABundle, nil 228 } 229 case v1alpha1.ValidatingAdmissionWebhook: 230 existingWebhooks, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().ValidatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 231 if err != nil { 232 return nil, fmt.Errorf("could not retrieve generated ValidatingWebhookConfiguration: %v", err) 233 } 234 235 if len(existingWebhooks.Items) > 0 { 236 return existingWebhooks.Items[0].Webhooks[0].ClientConfig.CABundle, nil 237 } 238 case v1alpha1.ConversionWebhook: 239 for _, conversionCRD := range desc.ConversionCRDs { 240 // check if CRD exists on cluster 241 crd, err := a.opClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), conversionCRD, metav1.GetOptions{}) 242 if err != nil { 243 continue 244 } 245 if crd.Spec.Conversion == nil || crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil || crd.Spec.Conversion.Webhook.ClientConfig.CABundle == nil { 246 continue 247 } 248 249 return crd.Spec.Conversion.Webhook.ClientConfig.CABundle, nil 250 } 251 } 252 253 return nil, fmt.Errorf("unable to find CA") 254 } 255 256 // updateDeploymentSpecsWithAPIServiceData transforms an install strategy to include information about apiservices 257 // it is used in generating hashes for deployment specs to know when something in the spec has changed, 258 // but duplicates a lot of installAPIServiceRequirements and should be refactored. 259 func (a *Operator) updateDeploymentSpecsWithAPIServiceData(csv *v1alpha1.ClusterServiceVersion, strategy install.Strategy) (install.Strategy, error) { 260 // Assume the strategy is for a deployment 261 strategyDetailsDeployment, ok := strategy.(*v1alpha1.StrategyDetailsDeployment) 262 if !ok { 263 return nil, fmt.Errorf("unsupported InstallStrategy type") 264 } 265 266 // Return early if there are no owned APIServices 267 if !csv.HasCAResources() { 268 return strategyDetailsDeployment, nil 269 } 270 271 depSpecs := make(map[string]appsv1.DeploymentSpec) 272 for _, sddSpec := range strategyDetailsDeployment.DeploymentSpecs { 273 depSpecs[sddSpec.Name] = sddSpec.Spec 274 } 275 276 for _, desc := range csv.Spec.APIServiceDefinitions.Owned { 277 caBundle, err := a.getAPIServiceCABundle(csv, &desc) 278 if err != nil { 279 return nil, fmt.Errorf("could not retrieve caBundle for owned APIServices %s: %v", fmt.Sprintf("%s.%s", desc.Version, desc.Group), err) 280 } 281 caHash := certs.PEMSHA256(caBundle) 282 283 depSpec, ok := depSpecs[desc.DeploymentName] 284 if !ok { 285 return nil, fmt.Errorf("strategyDetailsDeployment is missing deployment %s for owned APIServices %s", desc.DeploymentName, fmt.Sprintf("%s.%s", desc.Version, desc.Group)) 286 } 287 288 if depSpec.Template.Spec.ServiceAccountName == "" { 289 depSpec.Template.Spec.ServiceAccountName = "default" 290 } 291 292 // Update deployment with secret volume mount. 293 secret, err := a.lister.CoreV1().SecretLister().Secrets(csv.GetNamespace()).Get(install.SecretName(install.ServiceName(desc.DeploymentName))) 294 if err != nil { 295 return nil, fmt.Errorf("unable to get secret %s", install.SecretName(install.ServiceName(desc.DeploymentName))) 296 } 297 298 install.AddDefaultCertVolumeAndVolumeMounts(&depSpec, secret.GetName()) 299 install.SetCAAnnotation(&depSpec, caHash) 300 depSpecs[desc.DeploymentName] = depSpec 301 } 302 303 for _, desc := range csv.Spec.WebhookDefinitions { 304 caBundle, err := a.getWebhookCABundle(csv, &desc) 305 if err != nil { 306 return nil, fmt.Errorf("could not retrieve caBundle for WebhookDescription %s: %v", desc.GenerateName, err) 307 } 308 caHash := certs.PEMSHA256(caBundle) 309 310 depSpec, ok := depSpecs[desc.DeploymentName] 311 if !ok { 312 return nil, fmt.Errorf("strategyDetailsDeployment is missing deployment %s for WebhookDescription %s", desc.DeploymentName, desc.GenerateName) 313 } 314 315 if depSpec.Template.Spec.ServiceAccountName == "" { 316 depSpec.Template.Spec.ServiceAccountName = "default" 317 } 318 319 // Update deployment with secret volume mount. 320 secret, err := a.lister.CoreV1().SecretLister().Secrets(csv.GetNamespace()).Get(install.SecretName(install.ServiceName(desc.DeploymentName))) 321 if err != nil { 322 return nil, fmt.Errorf("unable to get secret %s", install.SecretName(install.ServiceName(desc.DeploymentName))) 323 } 324 install.AddDefaultCertVolumeAndVolumeMounts(&depSpec, secret.GetName()) 325 326 install.SetCAAnnotation(&depSpec, caHash) 327 depSpecs[desc.DeploymentName] = depSpec 328 } 329 330 // Replace all matching DeploymentSpecs in the strategy 331 for i, sddSpec := range strategyDetailsDeployment.DeploymentSpecs { 332 if depSpec, ok := depSpecs[sddSpec.Name]; ok { 333 strategyDetailsDeployment.DeploymentSpecs[i].Spec = depSpec 334 } 335 } 336 return strategyDetailsDeployment, nil 337 } 338 339 func (a *Operator) cleanUpRemovedWebhooks(csv *v1alpha1.ClusterServiceVersion) error { 340 webhookLabels := ownerutil.OwnerLabel(csv, v1alpha1.ClusterServiceVersionKind) 341 webhookSelector := labels.SelectorFromSet(webhookLabels).String() 342 343 csvWebhookGenerateNames := make(map[string]struct{}, len(csv.Spec.WebhookDefinitions)) 344 for _, webhook := range csv.Spec.WebhookDefinitions { 345 csvWebhookGenerateNames[webhook.GenerateName] = struct{}{} 346 } 347 348 // Delete unknown ValidatingWebhooksConfigurations owned by the CSV 349 validatingWebhookConfigurationList, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().ValidatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 350 if err != nil { 351 return err 352 } 353 for _, webhook := range validatingWebhookConfigurationList.Items { 354 webhookGenerateNameLabel, ok := webhook.GetLabels()[install.WebhookDescKey] 355 if !ok { 356 return fmt.Errorf("validatingWebhookConfiguration %s does not have WebhookDesc key", webhook.Name) 357 } 358 if _, ok := csvWebhookGenerateNames[webhookGenerateNameLabel]; !ok { 359 err = a.opClient.KubernetesInterface().AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(context.TODO(), webhook.Name, metav1.DeleteOptions{}) 360 if err != nil && apierrors.IsNotFound(err) { 361 return err 362 } 363 } 364 } 365 366 // Delete unknown MutatingWebhooksConfigurations owned by the CSV 367 mutatingWebhookConfigurationList, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().MutatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 368 if err != nil { 369 return err 370 } 371 for _, webhook := range mutatingWebhookConfigurationList.Items { 372 webhookGenerateNameLabel, ok := webhook.GetLabels()[install.WebhookDescKey] 373 if !ok { 374 return fmt.Errorf("mutatingWebhookConfiguration %s does not have WebhookDesc key", webhook.Name) 375 } 376 if _, ok := csvWebhookGenerateNames[webhookGenerateNameLabel]; !ok { 377 err = a.opClient.KubernetesInterface().AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.TODO(), webhook.Name, metav1.DeleteOptions{}) 378 if err != nil && apierrors.IsNotFound(err) { 379 return err 380 } 381 } 382 } 383 384 return nil 385 } 386 387 func (a *Operator) areWebhooksAvailable(csv *v1alpha1.ClusterServiceVersion) (bool, error) { 388 err := a.cleanUpRemovedWebhooks(csv) 389 if err != nil { 390 return false, err 391 } 392 for _, desc := range csv.Spec.WebhookDefinitions { 393 // Create Webhook Label Selector 394 webhookLabels := ownerutil.OwnerLabel(csv, v1alpha1.ClusterServiceVersionKind) 395 webhookLabels[install.WebhookDescKey] = desc.GenerateName 396 hash, err := hashutil.DeepHashObject(&desc) 397 if err != nil { 398 return false, err 399 } 400 webhookLabels[install.WebhookHashKey] = hash 401 webhookSelector := labels.SelectorFromSet(webhookLabels).String() 402 403 webhookCount := 0 404 switch desc.Type { 405 case v1alpha1.ValidatingAdmissionWebhook: 406 webhookList, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().ValidatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 407 if err != nil { 408 return false, err 409 } 410 webhookCount = len(webhookList.Items) 411 case v1alpha1.MutatingAdmissionWebhook: 412 webhookList, err := a.opClient.KubernetesInterface().AdmissionregistrationV1().MutatingWebhookConfigurations().List(context.TODO(), metav1.ListOptions{LabelSelector: webhookSelector}) 413 if err != nil { 414 return false, err 415 } 416 webhookCount = len(webhookList.Items) 417 case v1alpha1.ConversionWebhook: 418 for _, conversionCRD := range desc.ConversionCRDs { 419 // check if CRD exists on cluster 420 crd, err := a.opClient.ApiextensionsInterface().ApiextensionsV1().CustomResourceDefinitions().Get(context.TODO(), conversionCRD, metav1.GetOptions{}) 421 if err != nil { 422 log.Infof("CRD not found %v, error: %s", desc, err.Error()) 423 return false, err 424 } 425 426 if crd.Spec.Conversion == nil || crd.Spec.Conversion.Strategy != "Webhook" || crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil || crd.Spec.Conversion.Webhook.ClientConfig.CABundle == nil { 427 return false, fmt.Errorf("conversionWebhook not ready") 428 } 429 webhookCount++ 430 } 431 } 432 if webhookCount == 0 { 433 return false, nil 434 } 435 } 436 return true, nil 437 }