github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/integration/e2e/service.go (about) 1 package e2e 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "io/ioutil" 8 "net" 9 "os/exec" 10 "regexp" 11 "strconv" 12 "strings" 13 "time" 14 15 "github.com/go-kit/log" 16 "github.com/grafana/dskit/backoff" 17 "github.com/pkg/errors" 18 "github.com/prometheus/common/expfmt" 19 "github.com/thanos-io/thanos/pkg/runutil" 20 ) 21 22 var ( 23 dockerIPv4PortPattern = regexp.MustCompile(`^\d+\.\d+\.\d+\.\d+:(\d+)$`) 24 errMissingMetric = errors.New("metric not found") 25 ) 26 27 // ConcreteService represents microservice with optional ports which will be discoverable from docker 28 // with <name>:<port>. For connecting from test, use `Endpoint` method. 29 // 30 // ConcreteService can be reused (started and stopped many time), but it can represent only one running container 31 // at the time. 32 type ConcreteService struct { 33 name string 34 image string 35 networkPorts []int 36 env map[string]string 37 user string 38 command *Command 39 readiness ReadinessProbe 40 41 // Maps container ports to dynamically binded local ports. 42 networkPortsContainerToLocal map[int]int 43 44 // Generic retry backoff. 45 retryBackoff *backoff.Backoff 46 47 // docker NetworkName used to start this container. 48 // If empty it means service is stopped. 49 usedNetworkName string 50 } 51 52 func NewConcreteService( 53 name string, 54 image string, 55 command *Command, 56 readiness ReadinessProbe, 57 networkPorts ...int, 58 ) *ConcreteService { 59 return &ConcreteService{ 60 name: name, 61 image: image, 62 networkPorts: networkPorts, 63 command: command, 64 networkPortsContainerToLocal: map[int]int{}, 65 readiness: readiness, 66 retryBackoff: backoff.New(context.Background(), backoff.Config{ 67 MinBackoff: 300 * time.Millisecond, 68 MaxBackoff: 600 * time.Millisecond, 69 MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯ 70 }), 71 } 72 } 73 74 func (s *ConcreteService) isExpectedRunning() bool { 75 return s.usedNetworkName != "" 76 } 77 78 func (s *ConcreteService) Name() string { return s.name } 79 80 // Less often used options. 81 82 func (s *ConcreteService) SetBackoff(cfg backoff.Config) { 83 s.retryBackoff = backoff.New(context.Background(), cfg) 84 } 85 86 func (s *ConcreteService) SetEnvVars(env map[string]string) { 87 s.env = env 88 } 89 90 func (s *ConcreteService) SetUser(user string) { 91 s.user = user 92 } 93 94 func (s *ConcreteService) Start(networkName, sharedDir string) (err error) { 95 // In case of any error, if the container was already created, we 96 // have to cleanup removing it. We ignore the error of the "docker rm" 97 // because we don't know if the container was created or not. 98 defer func() { 99 if err != nil { 100 _, _ = RunCommandAndGetOutput("docker", "rm", "--force", s.name) 101 } 102 }() 103 104 cmd := exec.Command("docker", s.buildDockerRunArgs(networkName, sharedDir)...) 105 cmd.Stdout = &LinePrefixLogger{prefix: s.name + ": ", logger: logger} 106 cmd.Stderr = &LinePrefixLogger{prefix: s.name + ": ", logger: logger} 107 if err = cmd.Start(); err != nil { 108 return err 109 } 110 s.usedNetworkName = networkName 111 112 // Wait until the container has been started. 113 if err = s.WaitForRunning(); err != nil { 114 return err 115 } 116 117 // Get the dynamic local ports mapped to the container. 118 for _, containerPort := range s.networkPorts { 119 var out []byte 120 121 out, err = RunCommandAndGetOutput("docker", "port", s.containerName(), strconv.Itoa(containerPort)) 122 if err != nil { 123 // Catch init errors. 124 if werr := s.WaitForRunning(); werr != nil { 125 return errors.Wrapf(werr, "failed to get mapping for port as container %s exited: %v", s.containerName(), err) 126 } 127 return errors.Wrapf(err, "unable to get mapping for port %d; service: %s; output: %q", containerPort, s.name, out) 128 } 129 130 localPort, err := parseDockerIPv4Port(string(out)) 131 if err != nil { 132 return errors.Wrapf(err, "unable to get mapping for port %d (output: %s); service: %s", containerPort, string(out), s.name) 133 } 134 135 s.networkPortsContainerToLocal[containerPort] = localPort 136 } 137 138 logger.Log("Ports for container:", s.containerName(), "Mapping:", s.networkPortsContainerToLocal) 139 return nil 140 } 141 142 func (s *ConcreteService) Stop() error { 143 if !s.isExpectedRunning() { 144 return nil 145 } 146 147 logger.Log("Stopping", s.name) 148 149 if out, err := RunCommandAndGetOutput("docker", "stop", "--time=30", s.containerName()); err != nil { 150 logger.Log(string(out)) 151 return err 152 } 153 s.usedNetworkName = "" 154 155 return nil 156 } 157 158 func (s *ConcreteService) Kill() error { 159 if !s.isExpectedRunning() { 160 return nil 161 } 162 163 logger.Log("Killing", s.name) 164 165 if out, err := RunCommandAndGetOutput("docker", "kill", s.containerName()); err != nil { 166 logger.Log(string(out)) 167 return err 168 } 169 170 // Wait until the container actually stopped. However, this could fail if 171 // the container already exited, so we just ignore the error. 172 _, _ = RunCommandAndGetOutput("docker", "wait", s.containerName()) 173 174 s.usedNetworkName = "" 175 176 return nil 177 } 178 179 // Endpoint returns external (from host perspective) service endpoint (host:port) for given internal port. 180 // External means that it will be accessible only from host, but not from docker containers. 181 // 182 // If your service is not running, this method returns incorrect `stopped` endpoint. 183 func (s *ConcreteService) Endpoint(port int) string { 184 if !s.isExpectedRunning() { 185 return "stopped" 186 } 187 188 // Map the container port to the local port. 189 localPort, ok := s.networkPortsContainerToLocal[port] 190 if !ok { 191 return "" 192 } 193 194 // Do not use "localhost" cause it doesn't work with the AWS DynamoDB client. 195 return fmt.Sprintf("127.0.0.1:%d", localPort) 196 } 197 198 // NetworkEndpoint returns internal service endpoint (host:port) for given internal port. 199 // Internal means that it will be accessible only from docker containers within the network that this 200 // service is running in. If you configure your local resolver with docker DNS namespace you can access it from host 201 // as well. Use `Endpoint` for host access. 202 // 203 // If your service is not running, use `NetworkEndpointFor` instead. 204 func (s *ConcreteService) NetworkEndpoint(port int) string { 205 if s.usedNetworkName == "" { 206 return "stopped" 207 } 208 return s.NetworkEndpointFor(s.usedNetworkName, port) 209 } 210 211 // NetworkEndpointFor returns internal service endpoint (host:port) for given internal port and network. 212 // Internal means that it will be accessible only from docker containers within the given network. If you configure 213 // your local resolver with docker DNS namespace you can access it from host as well. 214 // 215 // This method return correct endpoint for the service in any state. 216 func (s *ConcreteService) NetworkEndpointFor(networkName string, port int) string { 217 return fmt.Sprintf("%s:%d", NetworkContainerHost(networkName, s.name), port) 218 } 219 220 func (s *ConcreteService) SetReadinessProbe(probe ReadinessProbe) { 221 s.readiness = probe 222 } 223 224 func (s *ConcreteService) Ready() error { 225 if !s.isExpectedRunning() { 226 return fmt.Errorf("service %s is stopped", s.Name()) 227 } 228 229 // Ensure the service has a readiness probe configure. 230 if s.readiness == nil { 231 return nil 232 } 233 234 return s.readiness.Ready(s) 235 } 236 237 func (s *ConcreteService) containerName() string { 238 return NetworkContainerHost(s.usedNetworkName, s.name) 239 } 240 241 func (s *ConcreteService) WaitForRunning() (err error) { 242 if !s.isExpectedRunning() { 243 return fmt.Errorf("service %s is stopped", s.Name()) 244 } 245 246 for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 247 // Enforce a timeout on the command execution because we've seen some flaky tests 248 // stuck here. 249 250 var out []byte 251 out, err = RunCommandWithTimeoutAndGetOutput(5*time.Second, "docker", "inspect", "--format={{json .State.Running}}", s.containerName()) 252 if err != nil { 253 s.retryBackoff.Wait() 254 continue 255 } 256 257 if out == nil { 258 err = fmt.Errorf("nil output") 259 s.retryBackoff.Wait() 260 continue 261 } 262 263 str := strings.TrimSpace(string(out)) 264 if str != "true" { 265 err = fmt.Errorf("unexpected output: %q", str) 266 s.retryBackoff.Wait() 267 continue 268 } 269 270 return nil 271 } 272 273 return fmt.Errorf("docker container %s failed to start: %v", s.name, err) 274 } 275 276 func (s *ConcreteService) WaitReady() (err error) { 277 if !s.isExpectedRunning() { 278 return fmt.Errorf("service %s is stopped", s.Name()) 279 } 280 281 for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 282 err = s.Ready() 283 if err == nil { 284 return nil 285 } 286 287 s.retryBackoff.Wait() 288 } 289 290 return fmt.Errorf("the service %s is not ready; err: %v", s.name, err) 291 } 292 293 func (s *ConcreteService) buildDockerRunArgs(networkName, sharedDir string) []string { 294 args := []string{"run", "--rm", "--net=" + networkName, "--name=" + networkName + "-" + s.name, "--hostname=" + s.name} 295 296 // For Drone CI users, expire the container after 6 hours using drone-gc 297 args = append(args, "--label", fmt.Sprintf("io.drone.expires=%d", time.Now().Add(6*time.Hour).Unix())) 298 299 // Mount the shared/ directory into the container 300 args = append(args, "-v", fmt.Sprintf("%s:%s:z", sharedDir, ContainerSharedDir)) 301 302 // Environment variables 303 for name, value := range s.env { 304 args = append(args, "-e", name+"="+value) 305 } 306 307 if s.user != "" { 308 args = append(args, "--user", s.user) 309 } 310 311 // Published ports 312 for _, port := range s.networkPorts { 313 args = append(args, "-p", strconv.Itoa(port)) 314 } 315 316 // Disable entrypoint if required 317 if s.command != nil && s.command.entrypointDisabled { 318 args = append(args, "--entrypoint", "") 319 } 320 321 args = append(args, s.image) 322 323 if s.command != nil { 324 args = append(args, s.command.cmd) 325 args = append(args, s.command.args...) 326 } 327 328 return args 329 } 330 331 // Exec runs the provided against a the docker container specified by this 332 // service. It returns the stdout, stderr, and error response from attempting 333 // to run the command. 334 func (s *ConcreteService) Exec(command *Command) (string, string, error) { 335 args := []string{"exec", s.containerName()} 336 args = append(args, command.cmd) 337 args = append(args, command.args...) 338 339 cmd := exec.Command("docker", args...) 340 var stdout bytes.Buffer 341 cmd.Stdout = &stdout 342 343 var stderr bytes.Buffer 344 cmd.Stderr = &stderr 345 346 err := cmd.Run() 347 348 return stdout.String(), stderr.String(), err 349 } 350 351 // NetworkContainerHost return the hostname of the container within the network. This is 352 // the address a container should use to connect to other containers. 353 func NetworkContainerHost(networkName, containerName string) string { 354 return fmt.Sprintf("%s-%s", networkName, containerName) 355 } 356 357 // NetworkContainerHostPort return the host:port address of a container within the network. 358 func NetworkContainerHostPort(networkName, containerName string, port int) string { 359 return fmt.Sprintf("%s-%s:%d", networkName, containerName, port) 360 } 361 362 type Command struct { 363 cmd string 364 args []string 365 entrypointDisabled bool 366 } 367 368 func NewCommand(cmd string, args ...string) *Command { 369 return &Command{ 370 cmd: cmd, 371 args: args, 372 } 373 } 374 375 func NewCommandWithoutEntrypoint(cmd string, args ...string) *Command { 376 return &Command{ 377 cmd: cmd, 378 args: args, 379 entrypointDisabled: true, 380 } 381 } 382 383 type ReadinessProbe interface { 384 Ready(service *ConcreteService) (err error) 385 } 386 387 // HTTPReadinessProbe checks readiness by making HTTP call and checking for expected HTTP status code 388 type HTTPReadinessProbe struct { 389 port int 390 path string 391 expectedStatusRangeStart int 392 expectedStatusRangeEnd int 393 expectedContent []string 394 } 395 396 func NewHTTPReadinessProbe(port int, path string, expectedStatusRangeStart, expectedStatusRangeEnd int, expectedContent ...string) *HTTPReadinessProbe { 397 return &HTTPReadinessProbe{ 398 port: port, 399 path: path, 400 expectedStatusRangeStart: expectedStatusRangeStart, 401 expectedStatusRangeEnd: expectedStatusRangeEnd, 402 expectedContent: expectedContent, 403 } 404 } 405 406 func (p *HTTPReadinessProbe) Ready(service *ConcreteService) (err error) { 407 endpoint := service.Endpoint(p.port) 408 if endpoint == "" { 409 return fmt.Errorf("cannot get service endpoint for port %d", p.port) 410 } else if endpoint == "stopped" { 411 return errors.New("service has stopped") 412 } 413 414 res, err := GetRequest("http://" + endpoint + p.path) 415 if err != nil { 416 return err 417 } 418 419 defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "response readiness") 420 body, _ := ioutil.ReadAll(res.Body) 421 422 if res.StatusCode < p.expectedStatusRangeStart || res.StatusCode > p.expectedStatusRangeEnd { 423 return fmt.Errorf("expected code in range: [%v, %v], got status code: %v and body: %v", p.expectedStatusRangeStart, p.expectedStatusRangeEnd, res.StatusCode, string(body)) 424 } 425 426 for _, expected := range p.expectedContent { 427 if !strings.Contains(string(body), expected) { 428 return fmt.Errorf("expected body containing %s, got: %v", expected, string(body)) 429 } 430 } 431 432 return nil 433 } 434 435 // TCPReadinessProbe checks readiness by ensure a TCP connection can be established. 436 type TCPReadinessProbe struct { 437 port int 438 } 439 440 func NewTCPReadinessProbe(port int) *TCPReadinessProbe { 441 return &TCPReadinessProbe{ 442 port: port, 443 } 444 } 445 446 func (p *TCPReadinessProbe) Ready(service *ConcreteService) (err error) { 447 endpoint := service.Endpoint(p.port) 448 if endpoint == "" { 449 return fmt.Errorf("cannot get service endpoint for port %d", p.port) 450 } else if endpoint == "stopped" { 451 return errors.New("service has stopped") 452 } 453 454 conn, err := net.DialTimeout("tcp", endpoint, time.Second) 455 if err != nil { 456 return err 457 } 458 459 return conn.Close() 460 } 461 462 // CmdReadinessProbe checks readiness by `Exec`ing a command (within container) which returns 0 to consider status being ready 463 type CmdReadinessProbe struct { 464 cmd *Command 465 } 466 467 func NewCmdReadinessProbe(cmd *Command) *CmdReadinessProbe { 468 return &CmdReadinessProbe{cmd: cmd} 469 } 470 471 func (p *CmdReadinessProbe) Ready(service *ConcreteService) error { 472 _, _, err := service.Exec(p.cmd) 473 return err 474 } 475 476 type LinePrefixLogger struct { 477 prefix string 478 logger log.Logger 479 } 480 481 func (w *LinePrefixLogger) Write(p []byte) (n int, err error) { 482 for _, line := range strings.Split(string(p), "\n") { 483 // Skip empty lines 484 line = strings.TrimSpace(line) 485 if line == "" { 486 continue 487 } 488 489 // Write the prefix + line to the wrapped writer 490 if err := w.logger.Log(w.prefix + line); err != nil { 491 return 0, err 492 } 493 } 494 495 return len(p), nil 496 } 497 498 // HTTPService represents opinionated microservice with at least HTTP port that as mandatory requirement, 499 // serves metrics. 500 type HTTPService struct { 501 *ConcreteService 502 503 httpPort int 504 } 505 506 func NewHTTPService( 507 name string, 508 image string, 509 command *Command, 510 readiness ReadinessProbe, 511 httpPort int, 512 otherPorts ...int, 513 ) *HTTPService { 514 return &HTTPService{ 515 ConcreteService: NewConcreteService(name, image, command, readiness, append(otherPorts, httpPort)...), 516 httpPort: httpPort, 517 } 518 } 519 520 func (s *HTTPService) Metrics() (_ string, err error) { 521 // Map the container port to the local port 522 localPort := s.networkPortsContainerToLocal[s.httpPort] 523 524 // Fetch metrics. 525 res, err := GetRequest(fmt.Sprintf("http://localhost:%d/metrics", localPort)) 526 if err != nil { 527 return "", err 528 } 529 530 // Check the status code. 531 if res.StatusCode < 200 || res.StatusCode >= 300 { 532 return "", fmt.Errorf("unexpected status code %d while fetching metrics", res.StatusCode) 533 } 534 535 defer runutil.ExhaustCloseWithErrCapture(&err, res.Body, "metrics response") 536 body, err := ioutil.ReadAll(res.Body) 537 538 return string(body), err 539 } 540 541 func (s *HTTPService) HTTPPort() int { 542 return s.httpPort 543 } 544 545 func (s *HTTPService) HTTPEndpoint() string { 546 return s.Endpoint(s.httpPort) 547 } 548 549 func (s *HTTPService) NetworkHTTPEndpoint() string { 550 return s.NetworkEndpoint(s.httpPort) 551 } 552 553 func (s *HTTPService) NetworkHTTPEndpointFor(networkName string) string { 554 return s.NetworkEndpointFor(networkName, s.httpPort) 555 } 556 557 // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true 558 // when passed to given isExpected(...). 559 func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error { 560 return s.WaitSumMetricsWithOptions(isExpected, metricNames) 561 } 562 563 func (s *HTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error { 564 var ( 565 sums []float64 566 err error 567 options = buildMetricsOptions(opts) 568 ) 569 570 for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 571 sums, err = s.SumMetrics(metricNames, opts...) 572 if options.WaitMissingMetrics && errors.Is(err, errMissingMetric) { 573 continue 574 } 575 if err != nil { 576 return err 577 } 578 579 if isExpected(sums...) { 580 return nil 581 } 582 583 s.retryBackoff.Wait() 584 } 585 586 return fmt.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums) 587 } 588 589 // SumMetrics returns the sum of the values of each given metric names. 590 func (s *HTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) { 591 options := buildMetricsOptions(opts) 592 sums := make([]float64, len(metricNames)) 593 594 metrics, err := s.Metrics() 595 if err != nil { 596 return nil, err 597 } 598 599 var tp expfmt.TextParser 600 families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) 601 if err != nil { 602 return nil, err 603 } 604 605 for i, m := range metricNames { 606 sums[i] = 0.0 607 608 // Get the metric family. 609 mf, ok := families[m] 610 if !ok { 611 if options.SkipMissingMetrics { 612 continue 613 } 614 615 return nil, errors.Wrapf(errMissingMetric, "metric=%s service=%s", m, s.name) 616 } 617 618 // Filter metrics. 619 metrics := filterMetrics(mf.GetMetric(), options) 620 if len(metrics) == 0 { 621 if options.SkipMissingMetrics { 622 continue 623 } 624 625 return nil, errors.Wrapf(errMissingMetric, "metric=%s service=%s", m, s.name) 626 } 627 628 sums[i] = SumValues(getValues(metrics, options)) 629 } 630 631 return sums, nil 632 } 633 634 // WaitRemovedMetric waits until a metric disappear from the list of metrics exported by the service. 635 func (s *HTTPService) WaitRemovedMetric(metricName string, opts ...MetricsOption) error { 636 options := buildMetricsOptions(opts) 637 638 for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 639 // Fetch metrics. 640 metrics, err := s.Metrics() 641 if err != nil { 642 return err 643 } 644 645 // Parse metrics. 646 var tp expfmt.TextParser 647 families, err := tp.TextToMetricFamilies(strings.NewReader(metrics)) 648 if err != nil { 649 return err 650 } 651 652 // Get the metric family. 653 mf, ok := families[metricName] 654 if !ok { 655 return nil 656 } 657 658 // Filter metrics. 659 if len(filterMetrics(mf.GetMetric(), options)) == 0 { 660 return nil 661 } 662 663 s.retryBackoff.Wait() 664 } 665 666 return fmt.Errorf("the metric %s is still exported by %s", metricName, s.name) 667 } 668 669 // parseDockerIPv4Port parses the input string which is expected to be the output of "docker port" 670 // command and returns the first IPv4 port found. 671 func parseDockerIPv4Port(out string) (int, error) { 672 // The "docker port" output may be multiple lines if both IPv4 and IPv6 are supported, 673 // so we need to parse each line. 674 for _, line := range strings.Split(out, "\n") { 675 matches := dockerIPv4PortPattern.FindStringSubmatch(strings.TrimSpace(line)) 676 if len(matches) != 2 { 677 continue 678 } 679 680 port, err := strconv.Atoi(matches[1]) 681 if err != nil { 682 continue 683 } 684 685 return port, nil 686 } 687 688 // We've not been able to parse the output format. 689 return 0, errors.New("unknown output format") 690 }