github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/kube/client.go (about) 1 /* 2 Copyright 2016 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package kube 18 19 import ( 20 "bytes" 21 "crypto/tls" 22 "crypto/x509" 23 "encoding/json" 24 "errors" 25 "fmt" 26 "io" 27 "io/ioutil" 28 "net/http" 29 "strconv" 30 "strings" 31 "time" 32 33 "github.com/sirupsen/logrus" 34 "k8s.io/api/core/v1" 35 "k8s.io/apimachinery/pkg/util/sets" 36 "sigs.k8s.io/yaml" 37 ) 38 39 const ( 40 // TestContainerName specifies the primary container name. 41 TestContainerName = "test" 42 43 inClusterBaseURL = "https://kubernetes.default" 44 maxRetries = 8 45 retryDelay = 2 * time.Second 46 requestTimeout = time.Minute 47 48 // EmptySelector selects everything 49 EmptySelector = "" 50 51 // DefaultClusterAlias specifies the default cluster key to schedule jobs. 52 DefaultClusterAlias = "default" 53 ) 54 55 // newClient is used to allow mocking out the behavior of 'NewClient' while testing. 56 var newClient = NewClient 57 58 // Logger can print debug messages 59 type Logger interface { 60 Debugf(s string, v ...interface{}) 61 } 62 63 // Client interacts with the Kubernetes api-server. 64 type Client struct { 65 // If logger is non-nil, log all method calls with it. 66 logger Logger 67 68 baseURL string 69 deckURL string 70 client *http.Client 71 token string 72 namespace string 73 fake bool 74 75 hiddenReposProvider func() []string 76 hiddenOnly bool 77 } 78 79 // SetHiddenReposProvider takes a continuation that fetches a list of orgs and repos for 80 // which PJs should not be returned. 81 // NOTE: This function is not thread safe and should be called before the client is in use. 82 func (c *Client) SetHiddenReposProvider(p func() []string, hiddenOnly bool) { 83 c.hiddenReposProvider = p 84 c.hiddenOnly = hiddenOnly 85 } 86 87 // Namespace returns a copy of the client pointing at the specified namespace. 88 func (c *Client) Namespace(ns string) *Client { 89 nc := *c 90 nc.namespace = ns 91 return &nc 92 } 93 94 func (c *Client) log(methodName string, args ...interface{}) { 95 if c.logger == nil { 96 return 97 } 98 var as []string 99 for _, arg := range args { 100 as = append(as, fmt.Sprintf("%v", arg)) 101 } 102 c.logger.Debugf("%s(%s)", methodName, strings.Join(as, ", ")) 103 } 104 105 // ConflictError is http 409. 106 type ConflictError struct { 107 e error 108 } 109 110 func (e ConflictError) Error() string { 111 return e.e.Error() 112 } 113 114 // NewConflictError returns an error with the embedded inner error 115 func NewConflictError(e error) ConflictError { 116 return ConflictError{e: e} 117 } 118 119 // UnprocessableEntityError happens when the apiserver returns http 422. 120 type UnprocessableEntityError struct { 121 e error 122 } 123 124 func (e UnprocessableEntityError) Error() string { 125 return e.e.Error() 126 } 127 128 // NewUnprocessableEntityError returns an error with the embedded inner error 129 func NewUnprocessableEntityError(e error) UnprocessableEntityError { 130 return UnprocessableEntityError{e: e} 131 } 132 133 // NotFoundError happens when the apiserver returns http 404 134 type NotFoundError struct { 135 e error 136 } 137 138 func (e NotFoundError) Error() string { 139 return e.e.Error() 140 } 141 142 // NewNotFoundError returns an error with the embedded inner error 143 func NewNotFoundError(e error) NotFoundError { 144 return NotFoundError{e: e} 145 } 146 147 type request struct { 148 method string 149 path string 150 deckPath string 151 query map[string]string 152 requestBody interface{} 153 } 154 155 func (c *Client) request(r *request, ret interface{}) error { 156 out, err := c.requestRetry(r) 157 if err != nil { 158 return err 159 } 160 if ret != nil { 161 if err := json.Unmarshal(out, ret); err != nil { 162 return err 163 } 164 } 165 return nil 166 } 167 168 func (c *Client) retry(r *request) (*http.Response, error) { 169 var resp *http.Response 170 var err error 171 backoff := retryDelay 172 for retries := 0; retries < maxRetries; retries++ { 173 resp, err = c.doRequest(r.method, r.deckPath, r.path, r.query, r.requestBody) 174 if err == nil { 175 if resp.StatusCode < 500 { 176 break 177 } 178 resp.Body.Close() 179 } 180 181 time.Sleep(backoff) 182 backoff *= 2 183 } 184 return resp, err 185 } 186 187 // Retry on transport failures. Does not retry on 500s. 188 func (c *Client) requestRetryStream(r *request) (io.ReadCloser, error) { 189 if c.fake && r.deckPath == "" { 190 return nil, nil 191 } 192 resp, err := c.retry(r) 193 if err != nil { 194 return nil, err 195 } 196 if resp.StatusCode == 409 { 197 return nil, NewConflictError(fmt.Errorf("body cannot be streamed")) 198 } else if resp.StatusCode < 200 || resp.StatusCode > 299 { 199 return nil, fmt.Errorf("response has status \"%s\"", resp.Status) 200 } 201 return resp.Body, nil 202 } 203 204 // Retry on transport failures. Does not retry on 500s. 205 func (c *Client) requestRetry(r *request) ([]byte, error) { 206 if c.fake && r.deckPath == "" { 207 return []byte("{}"), nil 208 } 209 resp, err := c.retry(r) 210 if err != nil { 211 return nil, err 212 } 213 214 defer resp.Body.Close() 215 rb, err := ioutil.ReadAll(resp.Body) 216 if err != nil { 217 return nil, err 218 } 219 if resp.StatusCode == 409 { 220 return nil, NewConflictError(fmt.Errorf("body: %s", string(rb))) 221 } else if resp.StatusCode == 422 { 222 return nil, NewUnprocessableEntityError(fmt.Errorf("body: %s", string(rb))) 223 } else if resp.StatusCode == 404 { 224 return nil, NewNotFoundError(fmt.Errorf("body: %s", string(rb))) 225 } else if resp.StatusCode < 200 || resp.StatusCode > 299 { 226 return nil, fmt.Errorf("response has status \"%s\" and body \"%s\"", resp.Status, string(rb)) 227 } 228 return rb, nil 229 } 230 231 func (c *Client) doRequest(method, deckPath, urlPath string, query map[string]string, body interface{}) (*http.Response, error) { 232 url := c.baseURL + urlPath 233 if c.deckURL != "" && deckPath != "" { 234 url = c.deckURL + deckPath 235 } 236 var buf io.Reader 237 if body != nil { 238 b, err := json.Marshal(body) 239 if err != nil { 240 return nil, err 241 } 242 buf = bytes.NewBuffer(b) 243 } 244 req, err := http.NewRequest(method, url, buf) 245 if err != nil { 246 return nil, err 247 } 248 if c.token != "" { 249 req.Header.Set("Authorization", "Bearer "+c.token) 250 } 251 if method == http.MethodPatch { 252 req.Header.Set("Content-Type", "application/strategic-merge-patch+json") 253 } else { 254 req.Header.Set("Content-Type", "application/json") 255 } 256 257 q := req.URL.Query() 258 for k, v := range query { 259 q.Add(k, v) 260 } 261 req.URL.RawQuery = q.Encode() 262 263 return c.client.Do(req) 264 } 265 266 // NewFakeClient creates a client that doesn't do anything. If you provide a 267 // deck URL then the client will hit that for the supported calls. 268 func NewFakeClient(deckURL string) *Client { 269 return &Client{ 270 namespace: "default", 271 deckURL: deckURL, 272 client: &http.Client{}, 273 fake: true, 274 } 275 } 276 277 // NewClientInCluster creates a Client that works from within a pod. 278 func NewClientInCluster(namespace string) (*Client, error) { 279 tokenFile := "/var/run/secrets/kubernetes.io/serviceaccount/token" 280 token, err := ioutil.ReadFile(tokenFile) 281 if err != nil { 282 return nil, err 283 } 284 285 rootCAFile := "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" 286 certData, err := ioutil.ReadFile(rootCAFile) 287 if err != nil { 288 return nil, err 289 } 290 291 cp := x509.NewCertPool() 292 cp.AppendCertsFromPEM(certData) 293 294 tr := &http.Transport{ 295 TLSClientConfig: &tls.Config{ 296 MinVersion: tls.VersionTLS12, 297 RootCAs: cp, 298 }, 299 } 300 return &Client{ 301 logger: logrus.WithField("client", "kube"), 302 baseURL: inClusterBaseURL, 303 client: &http.Client{Transport: tr, Timeout: requestTimeout}, 304 token: string(token), 305 namespace: namespace, 306 }, nil 307 } 308 309 // Cluster represents the information necessary to talk to a Kubernetes 310 // master endpoint. 311 // NOTE: if your cluster runs on GKE you can use the following command to get these credentials: 312 // gcloud --project <gcp_project> container clusters describe --zone <zone> <cluster_name> 313 type Cluster struct { 314 // The IP address of the cluster's master endpoint. 315 Endpoint string `json:"endpoint"` 316 // Base64-encoded public cert used by clients to authenticate to the 317 // cluster endpoint. 318 ClientCertificate []byte `json:"clientCertificate"` 319 // Base64-encoded private key used by clients.. 320 ClientKey []byte `json:"clientKey"` 321 // Base64-encoded public certificate that is the root of trust for the 322 // cluster. 323 ClusterCACertificate []byte `json:"clusterCaCertificate"` 324 } 325 326 // NewClientFromFile reads a Cluster object at clusterPath and returns an 327 // authenticated client using the keys within. 328 func NewClientFromFile(clusterPath, namespace string) (*Client, error) { 329 data, err := ioutil.ReadFile(clusterPath) 330 if err != nil { 331 return nil, err 332 } 333 var c Cluster 334 if err := yaml.Unmarshal(data, &c); err != nil { 335 return nil, err 336 } 337 return NewClient(&c, namespace) 338 } 339 340 // UnmarshalClusterMap reads a map[string]Cluster in yaml bytes. 341 func UnmarshalClusterMap(data []byte) (map[string]Cluster, error) { 342 var raw map[string]Cluster 343 if err := yaml.Unmarshal(data, &raw); err != nil { 344 // If we failed to unmarshal the multicluster format try the single Cluster format. 345 var singleConfig Cluster 346 if err := yaml.Unmarshal(data, &singleConfig); err != nil { 347 return nil, err 348 } 349 raw = map[string]Cluster{DefaultClusterAlias: singleConfig} 350 } 351 return raw, nil 352 } 353 354 // MarshalClusterMap writes c as yaml bytes. 355 func MarshalClusterMap(c map[string]Cluster) ([]byte, error) { 356 return yaml.Marshal(c) 357 } 358 359 // ClientMapFromFile reads the file at clustersPath and attempts to load a map of cluster aliases 360 // to authenticated clients to the respective clusters. 361 // The file at clustersPath is expected to be a yaml map from strings to Cluster structs OR it may 362 // simply be a single Cluster struct which will be assigned the alias $DefaultClusterAlias. 363 // If the file is an alias map, it must include the alias $DefaultClusterAlias. 364 func ClientMapFromFile(clustersPath, namespace string) (map[string]*Client, error) { 365 data, err := ioutil.ReadFile(clustersPath) 366 if err != nil { 367 return nil, fmt.Errorf("read error: %v", err) 368 } 369 raw, err := UnmarshalClusterMap(data) 370 if err != nil { 371 return nil, fmt.Errorf("unmarshal error: %v", err) 372 } 373 foundDefault := false 374 result := map[string]*Client{} 375 for alias, config := range raw { 376 client, err := newClient(&config, namespace) 377 if err != nil { 378 return nil, fmt.Errorf("failed to load config for build cluster alias %q in file %q: %v", alias, clustersPath, err) 379 } 380 result[alias] = client 381 if alias == DefaultClusterAlias { 382 foundDefault = true 383 } 384 } 385 if !foundDefault { 386 return nil, fmt.Errorf("failed to find the required %q alias in build cluster config %q", DefaultClusterAlias, clustersPath) 387 } 388 return result, nil 389 } 390 391 // NewClient returns an authenticated Client using the keys in the Cluster. 392 func NewClient(c *Cluster, namespace string) (*Client, error) { 393 // Relies on json encoding/decoding []byte as base64 394 // https://golang.org/pkg/encoding/json/#Marshal 395 cc := c.ClientCertificate 396 ck := c.ClientKey 397 ca := c.ClusterCACertificate 398 399 cert, err := tls.X509KeyPair(cc, ck) 400 if err != nil { 401 return nil, err 402 } 403 404 cp := x509.NewCertPool() 405 cp.AppendCertsFromPEM(ca) 406 407 tr := &http.Transport{ 408 TLSClientConfig: &tls.Config{ 409 MinVersion: tls.VersionTLS12, 410 Certificates: []tls.Certificate{cert}, 411 RootCAs: cp, 412 }, 413 } 414 return &Client{ 415 logger: logrus.WithField("client", "kube"), 416 baseURL: c.Endpoint, 417 client: &http.Client{Transport: tr, Timeout: requestTimeout}, 418 namespace: namespace, 419 }, nil 420 } 421 422 // GetPod is analogous to kubectl get pods/NAME namespace=client.namespace 423 func (c *Client) GetPod(name string) (Pod, error) { 424 c.log("GetPod", name) 425 var retPod Pod 426 err := c.request(&request{ 427 path: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name), 428 }, &retPod) 429 return retPod, err 430 } 431 432 // ListPods is analogous to kubectl get pods --selector=SELECTOR --namespace=client.namespace 433 func (c *Client) ListPods(selector string) ([]Pod, error) { 434 c.log("ListPods", selector) 435 var pl struct { 436 Items []Pod `json:"items"` 437 } 438 err := c.request(&request{ 439 path: fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace), 440 query: map[string]string{"labelSelector": selector}, 441 }, &pl) 442 return pl.Items, err 443 } 444 445 // DeletePod deletes the pod at name in the client's specified namespace. 446 // 447 // Analogous to kubectl delete pod --namespace=client.namespace 448 func (c *Client) DeletePod(name string) error { 449 c.log("DeletePod", name) 450 return c.request(&request{ 451 method: http.MethodDelete, 452 path: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", c.namespace, name), 453 }, nil) 454 } 455 456 // CreateProwJob creates a prowjob in the client's specified namespace. 457 // 458 // Analogous to kubectl create prowjob --namespace=client.namespace 459 func (c *Client) CreateProwJob(j ProwJob) (ProwJob, error) { 460 var representation string 461 if out, err := json.Marshal(j); err == nil { 462 representation = string(out[:]) 463 } else { 464 representation = fmt.Sprintf("%v", j) 465 } 466 c.log("CreateProwJob", representation) 467 var retJob ProwJob 468 err := c.request(&request{ 469 method: http.MethodPost, 470 path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace), 471 requestBody: &j, 472 }, &retJob) 473 return retJob, err 474 } 475 476 func (c *Client) getHiddenRepos() sets.String { 477 if c.hiddenReposProvider == nil { 478 return nil 479 } 480 return sets.NewString(c.hiddenReposProvider()...) 481 } 482 483 func shouldHide(pj *ProwJob, hiddenRepos sets.String, showHiddenOnly bool) bool { 484 if pj.Spec.Refs == nil { 485 // periodic jobs do not have refs and therefore cannot be 486 // hidden by the org/repo mechanism 487 return false 488 } 489 shouldHide := hiddenRepos.HasAny(fmt.Sprintf("%s/%s", pj.Spec.Refs.Org, pj.Spec.Refs.Repo), pj.Spec.Refs.Org) 490 if showHiddenOnly { 491 return !shouldHide 492 } 493 return shouldHide 494 } 495 496 // GetProwJob returns the prowjob at name in the client's specified namespace. 497 // 498 // Analogous to kubectl get prowjob/NAME --namespace=client.namespace 499 func (c *Client) GetProwJob(name string) (ProwJob, error) { 500 c.log("GetProwJob", name) 501 var pj ProwJob 502 err := c.request(&request{ 503 path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name), 504 }, &pj) 505 if err == nil && shouldHide(&pj, c.getHiddenRepos(), c.hiddenOnly) { 506 pj = ProwJob{} 507 // Revealing the existence of this prow job is ok because the pj name cannot be used to 508 // retrieve the pj itself. Furthermore, a timing attack could differentiate true 404s from 509 // 404s returned when a hidden pj is queried so returning a 404 wouldn't hide the pj's existence. 510 err = errors.New("403 ProwJob is hidden") 511 } 512 return pj, err 513 } 514 515 // ListProwJobs lists prowjobs using the specified labelSelector in the client's specified namespace. 516 // 517 // Analogous to kubectl get prowjobs --selector=SELECTOR --namespace=client.namespace 518 func (c *Client) ListProwJobs(selector string) ([]ProwJob, error) { 519 c.log("ListProwJobs", selector) 520 var jl struct { 521 Items []ProwJob `json:"items"` 522 } 523 err := c.request(&request{ 524 path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs", c.namespace), 525 deckPath: "/prowjobs.js", 526 query: map[string]string{"labelSelector": selector}, 527 }, &jl) 528 if err == nil { 529 hidden := c.getHiddenRepos() 530 var pjs []ProwJob 531 for _, pj := range jl.Items { 532 if !shouldHide(&pj, hidden, c.hiddenOnly) { 533 pjs = append(pjs, pj) 534 } 535 } 536 jl.Items = pjs 537 } 538 return jl.Items, err 539 } 540 541 // DeleteProwJob deletes the prowjob at name in the client's specified namespace. 542 // 543 // Analogous to kubectl delete prowjob/NAME --namespace=client.namespace 544 func (c *Client) DeleteProwJob(name string) error { 545 c.log("DeleteProwJob", name) 546 return c.request(&request{ 547 method: http.MethodDelete, 548 path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name), 549 }, nil) 550 } 551 552 // ReplaceProwJob will replace name with job in the client's specified namespace. 553 // 554 // Analogous to kubectl replace prowjobs/NAME --namespace=client.namespace 555 func (c *Client) ReplaceProwJob(name string, job ProwJob) (ProwJob, error) { 556 c.log("ReplaceProwJob", name, job) 557 var retJob ProwJob 558 err := c.request(&request{ 559 method: http.MethodPut, 560 path: fmt.Sprintf("/apis/prow.k8s.io/v1/namespaces/%s/prowjobs/%s", c.namespace, name), 561 requestBody: &job, 562 }, &retJob) 563 return retJob, err 564 } 565 566 // CreatePod creates a pod in the client's specified namespace. 567 // 568 // Analogous to kubectl create pod --namespace=client.namespace 569 func (c *Client) CreatePod(p v1.Pod) (Pod, error) { 570 c.log("CreatePod", p) 571 var retPod Pod 572 err := c.request(&request{ 573 method: http.MethodPost, 574 path: fmt.Sprintf("/api/v1/namespaces/%s/pods", c.namespace), 575 requestBody: &p, 576 }, &retPod) 577 return retPod, err 578 } 579 580 // GetLog returns the log of the default container in the specified pod, in the client's specified namespace. 581 // 582 // Analogous to kubectl logs pod --namespace=client.namespace 583 func (c *Client) GetLog(pod string) ([]byte, error) { 584 c.log("GetLog", pod) 585 return c.requestRetry(&request{ 586 path: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod), 587 }) 588 } 589 590 // GetLogTail returns the last n bytes of the log of the specified container in the specified pod, 591 // in the client's specified namespace. 592 // 593 // Analogous to kubectl logs pod --tail -1 --limit-bytes n -c container --namespace=client.namespace 594 func (c *Client) GetLogTail(pod, container string, n int64) ([]byte, error) { 595 c.log("GetLogTail", pod, n) 596 return c.requestRetry(&request{ 597 path: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod), 598 query: map[string]string{ // Because we want last n bytes, we fetch all lines and then limit to n bytes 599 "tailLines": "-1", 600 "container": container, 601 "limitBytes": strconv.FormatInt(n, 10), 602 }, 603 }) 604 } 605 606 // GetContainerLog returns the log of a container in the specified pod, in the client's specified namespace. 607 // 608 // Analogous to kubectl logs pod -c container --namespace=client.namespace 609 func (c *Client) GetContainerLog(pod, container string) ([]byte, error) { 610 c.log("GetContainerLog", pod) 611 return c.requestRetry(&request{ 612 path: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/log", c.namespace, pod), 613 query: map[string]string{"container": container}, 614 }) 615 } 616 617 // CreateConfigMap creates a configmap, in the client's specified namespace. 618 // 619 // Analogous to kubectl create configmap --namespace=client.namespace 620 func (c *Client) CreateConfigMap(content ConfigMap) (ConfigMap, error) { 621 c.log("CreateConfigMap") 622 var retConfigMap ConfigMap 623 err := c.request(&request{ 624 method: http.MethodPost, 625 path: fmt.Sprintf("/api/v1/namespaces/%s/configmaps", c.namespace), 626 requestBody: &content, 627 }, &retConfigMap) 628 629 return retConfigMap, err 630 } 631 632 // GetConfigMap gets the configmap identified, in the client's specified namespace. 633 // 634 // Analogous to kubectl get configmap --namespace=client.namespace 635 func (c *Client) GetConfigMap(name, namespace string) (ConfigMap, error) { 636 c.log("GetConfigMap", name) 637 if namespace == "" { 638 namespace = c.namespace 639 } 640 var retConfigMap ConfigMap 641 err := c.request(&request{ 642 path: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", namespace, name), 643 }, &retConfigMap) 644 645 return retConfigMap, err 646 } 647 648 // ReplaceConfigMap puts the configmap into name. 649 // 650 // Analogous to kubectl replace configmap 651 // 652 // If config.Namespace is empty, the client's specified namespace is used. 653 // Returns the content returned by the apiserver 654 func (c *Client) ReplaceConfigMap(name string, config ConfigMap) (ConfigMap, error) { 655 c.log("ReplaceConfigMap", name) 656 namespace := c.namespace 657 if config.Namespace != "" { 658 namespace = config.Namespace 659 } 660 var retConfigMap ConfigMap 661 err := c.request(&request{ 662 method: http.MethodPut, 663 path: fmt.Sprintf("/api/v1/namespaces/%s/configmaps/%s", namespace, name), 664 requestBody: &config, 665 }, &retConfigMap) 666 667 return retConfigMap, err 668 }