github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/prow/cmd/jenkins-operator/main.go (about) 1 /* 2 Copyright 2017 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package main 18 19 import ( 20 "crypto/tls" 21 "crypto/x509" 22 "errors" 23 "flag" 24 "fmt" 25 "io/ioutil" 26 "net/http" 27 "net/url" 28 "os" 29 "os/signal" 30 "syscall" 31 "time" 32 33 "github.com/NYTimes/gziphandler" 34 "github.com/prometheus/client_golang/prometheus/promhttp" 35 "github.com/sirupsen/logrus" 36 "k8s.io/apimachinery/pkg/labels" 37 38 "k8s.io/test-infra/prow/config" 39 "k8s.io/test-infra/prow/flagutil" 40 "k8s.io/test-infra/prow/github" 41 "k8s.io/test-infra/prow/jenkins" 42 "k8s.io/test-infra/prow/kube" 43 "k8s.io/test-infra/prow/logrusutil" 44 m "k8s.io/test-infra/prow/metrics" 45 ) 46 47 type options struct { 48 configPath string 49 selector string 50 totURL string 51 deckURL string 52 53 jenkinsURL string 54 jenkinsUserName string 55 jenkinsTokenFile string 56 jenkinsBearerTokenFile string 57 certFile string 58 keyFile string 59 caCertFile string 60 csrfProtect bool 61 62 githubEndpoint flagutil.Strings 63 githubTokenFile string 64 dryRun bool 65 } 66 67 func (o *options) Validate() error { 68 if o.jenkinsTokenFile == "" && o.jenkinsBearerTokenFile == "" { 69 return errors.New("either --jenkins-token-file or --jenkins-bearer-token-file must be set") 70 } else if o.jenkinsTokenFile != "" && o.jenkinsBearerTokenFile != "" { 71 return errors.New("only one of --jenkins-token-file or --jenkins-bearer-token-file can be set") 72 } 73 74 var transportSecretsProvided int 75 if o.certFile == "" { 76 transportSecretsProvided = transportSecretsProvided + 1 77 } 78 if o.keyFile == "" { 79 transportSecretsProvided = transportSecretsProvided + 1 80 } 81 if o.caCertFile == "" { 82 transportSecretsProvided = transportSecretsProvided + 1 83 } 84 if transportSecretsProvided != 0 && transportSecretsProvided != 3 { 85 return errors.New("either --cert-file, --key-file, and --ca-cert-file must all be provided or none of them must be provided") 86 } 87 return nil 88 } 89 90 func gatherOptions() options { 91 o := options{ 92 githubEndpoint: flagutil.NewStrings("https://api.github.com"), 93 } 94 flag.StringVar(&o.configPath, "config-path", "/etc/config/config.yaml", "Path to config.yaml.") 95 flag.StringVar(&o.selector, "label-selector", kube.EmptySelector, "Label selector to be applied in prowjobs. See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors for constructing a label selector.") 96 flag.StringVar(&o.totURL, "tot-url", "", "Tot URL") 97 flag.StringVar(&o.deckURL, "deck-url", "", "Deck URL for read-only access to the cluster.") 98 99 flag.StringVar(&o.jenkinsURL, "jenkins-url", "http://jenkins-proxy", "Jenkins URL") 100 flag.StringVar(&o.jenkinsUserName, "jenkins-user", "jenkins-trigger", "Jenkins username") 101 flag.StringVar(&o.jenkinsTokenFile, "jenkins-token-file", "", "Path to the file containing the Jenkins API token.") 102 flag.StringVar(&o.jenkinsBearerTokenFile, "jenkins-bearer-token-file", "", "Path to the file containing the Jenkins API bearer token.") 103 flag.StringVar(&o.certFile, "cert-file", "", "Path to a PEM-encoded certificate file.") 104 flag.StringVar(&o.keyFile, "key-file", "", "Path to a PEM-encoded key file.") 105 flag.StringVar(&o.caCertFile, "ca-cert-file", "", "Path to a PEM-encoded CA certificate file.") 106 flag.BoolVar(&o.csrfProtect, "csrf-protect", false, "Request a CSRF protection token from Jenkins that will be used in all subsequent requests to Jenkins.") 107 108 flag.Var(&o.githubEndpoint, "github-endpoint", "GitHub's API endpoint.") 109 flag.StringVar(&o.githubTokenFile, "github-token-file", "/etc/github/oauth", "Path to the file containing the GitHub OAuth token.") 110 flag.BoolVar(&o.dryRun, "dry-run", true, "Whether or not to make mutating API calls to GitHub/Kubernetes/Jenkins.") 111 flag.Parse() 112 return o 113 } 114 115 func main() { 116 o := gatherOptions() 117 if err := o.Validate(); err != nil { 118 logrus.Fatalf("Invalid options: %v", err) 119 } 120 logrus.SetFormatter( 121 logrusutil.NewDefaultFieldsFormatter(nil, logrus.Fields{"component": "jenkins-operator"}), 122 ) 123 124 if _, err := labels.Parse(o.selector); err != nil { 125 logrus.WithError(err).Fatal("Error parsing label selector.") 126 } 127 128 configAgent := &config.Agent{} 129 if err := configAgent.Start(o.configPath, ""); err != nil { 130 logrus.WithError(err).Fatal("Error starting config agent.") 131 } 132 133 kc, err := kube.NewClientInCluster(configAgent.Config().ProwJobNamespace) 134 if err != nil { 135 logrus.WithError(err).Fatal("Error getting kube client.") 136 } 137 138 ac := &jenkins.AuthConfig{ 139 CSRFProtect: o.csrfProtect, 140 } 141 142 var tokens []string 143 tokens = append(tokens, o.githubTokenFile) 144 145 if o.jenkinsTokenFile != "" { 146 tokens = append(tokens, o.jenkinsTokenFile) 147 } 148 149 if o.jenkinsBearerTokenFile != "" { 150 tokens = append(tokens, o.jenkinsBearerTokenFile) 151 } 152 153 // Start the secret agent. 154 secretAgent := &config.SecretAgent{} 155 if err := secretAgent.Start(tokens); err != nil { 156 logrus.WithError(err).Fatal("Error starting secrets agent.") 157 } 158 159 if o.jenkinsTokenFile != "" { 160 ac.Basic = &jenkins.BasicAuthConfig{ 161 User: o.jenkinsUserName, 162 GetToken: secretAgent.GetTokenGenerator(o.jenkinsTokenFile), 163 } 164 } else if o.jenkinsBearerTokenFile != "" { 165 ac.BearerToken = &jenkins.BearerTokenAuthConfig{ 166 GetToken: secretAgent.GetTokenGenerator(o.jenkinsBearerTokenFile), 167 } 168 } 169 var tlsConfig *tls.Config 170 if o.certFile != "" && o.keyFile != "" { 171 config, err := loadCerts(o.certFile, o.keyFile, o.caCertFile) 172 if err != nil { 173 logrus.WithError(err).Fatalf("Could not read certificate files.") 174 } 175 tlsConfig = config 176 } 177 metrics := jenkins.NewMetrics() 178 jc, err := jenkins.NewClient(o.jenkinsURL, o.dryRun, tlsConfig, ac, nil, metrics.ClientMetrics) 179 if err != nil { 180 logrus.WithError(err).Fatalf("Could not setup Jenkins client.") 181 } 182 183 // Check if github endpoint has a valid url. 184 for _, ep := range o.githubEndpoint.Strings() { 185 _, err = url.ParseRequestURI(ep) 186 if err != nil { 187 logrus.WithError(err).Fatalf("Invalid --endpoint URL %q.", ep) 188 } 189 } 190 191 var ghc *github.Client 192 if o.dryRun { 193 ghc = github.NewDryRunClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...) 194 kc = kube.NewFakeClient(o.deckURL) 195 } else { 196 ghc = github.NewClient(secretAgent.GetTokenGenerator(o.githubTokenFile), o.githubEndpoint.Strings()...) 197 } 198 199 c, err := jenkins.NewController(kc, jc, ghc, nil, configAgent, o.totURL, o.selector) 200 if err != nil { 201 logrus.WithError(err).Fatal("Failed to instantiate Jenkins controller.") 202 } 203 204 // Push metrics to the configured prometheus pushgateway endpoint. 205 pushGateway := configAgent.Config().PushGateway 206 if pushGateway.Endpoint != "" { 207 go m.PushMetrics("jenkins-operator", pushGateway.Endpoint, pushGateway.Interval) 208 } 209 // Serve Jenkins logs here and proxy deck to use this endpoint 210 // instead of baking agent-specific logic in deck. This func also 211 // serves prometheus metrics. 212 go serve(jc) 213 // gather metrics for the jobs handled by the jenkins controller. 214 go gather(c) 215 216 tick := time.Tick(30 * time.Second) 217 sig := make(chan os.Signal, 1) 218 signal.Notify(sig, os.Interrupt, syscall.SIGTERM) 219 220 for { 221 select { 222 case <-tick: 223 start := time.Now() 224 if err := c.Sync(); err != nil { 225 logrus.WithError(err).Error("Error syncing.") 226 } 227 duration := time.Since(start) 228 logrus.WithField("duration", fmt.Sprintf("%v", duration)).Info("Synced") 229 metrics.ResyncPeriod.Observe(duration.Seconds()) 230 case <-sig: 231 logrus.Info("Jenkins operator is shutting down...") 232 return 233 } 234 } 235 } 236 237 func loadCerts(certFile, keyFile, caCertFile string) (*tls.Config, error) { 238 cert, err := tls.LoadX509KeyPair(certFile, keyFile) 239 if err != nil { 240 return nil, err 241 } 242 243 tlsConfig := &tls.Config{ 244 Certificates: []tls.Certificate{cert}, 245 } 246 247 if caCertFile != "" { 248 caCert, err := ioutil.ReadFile(caCertFile) 249 if err != nil { 250 return nil, err 251 } 252 caCertPool := x509.NewCertPool() 253 caCertPool.AppendCertsFromPEM(caCert) 254 tlsConfig.RootCAs = caCertPool 255 } 256 257 tlsConfig.BuildNameToCertificate() 258 return tlsConfig, nil 259 } 260 261 // serve starts a http server and serves Jenkins logs 262 // and prometheus metrics. Meant to be called inside 263 // a goroutine. 264 func serve(jc *jenkins.Client) { 265 http.Handle("/", gziphandler.GzipHandler(handleLog(jc))) 266 http.Handle("/metrics", promhttp.Handler()) 267 logrus.WithError(http.ListenAndServe(":8080", nil)).Fatal("ListenAndServe returned.") 268 } 269 270 // gather metrics from the jenkins controller. 271 // Meant to be called inside a goroutine. 272 func gather(c *jenkins.Controller) { 273 tick := time.Tick(30 * time.Second) 274 sig := make(chan os.Signal, 1) 275 signal.Notify(sig, os.Interrupt, syscall.SIGTERM) 276 277 for { 278 select { 279 case <-tick: 280 start := time.Now() 281 c.SyncMetrics() 282 logrus.WithField("metrics-duration", fmt.Sprintf("%v", time.Since(start))).Debug("Metrics synced") 283 case <-sig: 284 logrus.Debug("Jenkins operator gatherer is shutting down...") 285 return 286 } 287 } 288 }