github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/cloud/kubernetes/performance/cockroachdb-statefulset-insecure.yaml (about)

     1  # This configuration file sets up an insecure StatefulSet running CockroachDB with
     2  # tweaks to make it more performant than our default configuration files. All
     3  # changes from the default insecure configuration have been marked with a comment
     4  # starting with "NOTE" or "TODO".
     5  #
     6  # Beware that this configuration is quite insecure. By default, it will make
     7  # CockroachDB accesible on port 26257 on your Kubernetes nodes' network
     8  # interfaces, meaning that if your nodes are reachable from the Internet, then
     9  # this CockroachDB cluster will be too. To disable this behavior, remove the 
    10  # `hostNetwork` configuration field below.
    11  #
    12  # To use this file, customize all the parts labeled "TODO" before running:
    13  #   kubectl create -f cockroachdb-statefulset-insecure.yaml
    14  #
    15  # You will then have to initialize the cluster as described in the parent
    16  # directory's README.md file.
    17  #
    18  # If you don't see any pods being created, it's possible that your cluster was
    19  # not able to meet the resource requests asked for, whether it was the amount
    20  # of CPU, memory, or disk or the disk type. To find information about why pods
    21  # haven't been created, you can run:
    22  #   kubectl get events
    23  #
    24  # For more information on improving CockroachDB performance in Kubernetes, see
    25  # our docs:
    26  # https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html
    27  apiVersion: v1
    28  kind: Service
    29  metadata:
    30    # This service is meant to be used by clients of the database. It exposes a ClusterIP that will
    31    # automatically load balance connections to the different database pods.
    32    name: cockroachdb-public
    33    labels:
    34      app: cockroachdb
    35  spec:
    36    ports:
    37    # The main port, served by gRPC, serves Postgres-flavor SQL, internode
    38    # traffic and the cli.
    39    - port: 26257
    40      targetPort: 26257
    41      name: grpc
    42    # The secondary port serves the UI as well as health and debug endpoints.
    43    - port: 8080
    44      targetPort: 8080
    45      name: http
    46    selector:
    47      app: cockroachdb
    48  ---
    49  apiVersion: v1
    50  kind: Service
    51  metadata:
    52    # This service only exists to create DNS entries for each pod in the stateful
    53    # set such that they can resolve each other's IP addresses. It does not
    54    # create a load-balanced ClusterIP and should not be used directly by clients
    55    # in most circumstances.
    56    name: cockroachdb
    57    labels:
    58      app: cockroachdb
    59    annotations:
    60      # Use this annotation in addition to the actual publishNotReadyAddresses
    61      # field below because the annotation will stop being respected soon but the
    62      # field is broken in some versions of Kubernetes:
    63      # https://github.com/kubernetes/kubernetes/issues/58662
    64      service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
    65      # Enable automatic monitoring of all instances when Prometheus is running in the cluster.
    66      prometheus.io/scrape: "true"
    67      prometheus.io/path: "_status/vars"
    68      prometheus.io/port: "8080"
    69  spec:
    70    ports:
    71    - port: 26257
    72      targetPort: 26257
    73      name: grpc
    74    - port: 8080
    75      targetPort: 8080
    76      name: http
    77    # We want all pods in the StatefulSet to have their addresses published for
    78    # the sake of the other CockroachDB pods even before they're ready, since they
    79    # have to be able to talk to each other in order to become ready.
    80    publishNotReadyAddresses: true
    81    clusterIP: None
    82    selector:
    83      app: cockroachdb
    84  ---
    85  apiVersion: policy/v1beta1
    86  kind: PodDisruptionBudget
    87  metadata:
    88    name: cockroachdb-budget
    89    labels:
    90      app: cockroachdb
    91  spec:
    92    selector:
    93      matchLabels:
    94        app: cockroachdb
    95    maxUnavailable: 1
    96  ---
    97  apiVersion: apps/v1
    98  kind: StatefulSet
    99  metadata:
   100    name: cockroachdb
   101  spec:
   102    serviceName: "cockroachdb"
   103    replicas: 3
   104    selector:
   105      matchLabels:
   106        app: cockroachdb
   107    template:
   108      metadata:
   109        labels:
   110          app: cockroachdb
   111      spec:
   112        # NOTE: Running with `hostNetwork: true` means that CockroachDB will use
   113        # the host machines' IP address and hostname, and that nothing else on
   114        # the machines will be able to use the same ports. This means that only 1
   115        # CockroachDB pod will ever be schedulable on the same machine, because
   116        # otherwise their ports would conflict.
   117        #
   118        # If your client pods generate a lot of network traffic to and from the
   119        # CockroachDB cluster, you may see a benefit to doing the same thing in
   120        # their configurations.
   121        hostNetwork: true
   122        dnsPolicy: ClusterFirstWithHostNet
   123        # NOTE: If you are running clients that generate heavy load, you may find
   124        # it useful to copy this anti-affinity policy into the client pods'
   125        # configurations as well to avoid running them on the same machines as
   126        # CockroachDB and interfering with each other's performance.
   127        affinity:
   128          podAntiAffinity:
   129            preferredDuringSchedulingIgnoredDuringExecution:
   130            - weight: 100
   131              podAffinityTerm:
   132                labelSelector:
   133                  matchExpressions:
   134                  - key: app
   135                    operator: In
   136                    values:
   137                    - cockroachdb
   138                topologyKey: kubernetes.io/hostname
   139        containers:
   140        - name: cockroachdb
   141          # NOTE: Always use the most recent version of CockroachDB for the best
   142          # performance and reliability.
   143          image: cockroachdb/cockroach:v20.1.1
   144          imagePullPolicy: IfNotPresent
   145          # TODO: Change these to appropriate values for the hardware that you're running. You can see
   146          # the amount of allocatable resources on each of your Kubernetes nodes by running:
   147          #   kubectl describe nodes
   148          resources:
   149            requests:
   150              cpu: "16"
   151              memory: "8Gi"
   152            limits:
   153              # NOTE: Unless you have enabled the non-default Static CPU Management Policy
   154              # and are using an integer number of CPUs, we don't recommend setting a CPU limit.
   155              # See:
   156              #   https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy
   157              #   https://github.com/kubernetes/kubernetes/issues/51135
   158              #cpu: "16"
   159              memory: "8Gi"
   160          ports:
   161          - containerPort: 26257
   162            name: grpc
   163          - containerPort: 8080
   164            name: http
   165          livenessProbe:
   166            httpGet:
   167              path: "/health"
   168              port: http
   169            initialDelaySeconds: 30
   170            periodSeconds: 5
   171          readinessProbe:
   172            httpGet:
   173              path: "/health?ready=1"
   174              port: http
   175            initialDelaySeconds: 10
   176            periodSeconds: 5
   177            failureThreshold: 2
   178          volumeMounts:
   179          - name: datadir
   180            mountPath: /cockroach/cockroach-data
   181          env:
   182          - name: COCKROACH_CHANNEL
   183            value: kubernetes-insecure
   184          command:
   185            - "/bin/bash"
   186            - "-ecx"
   187            # The use of qualified `hostname -f` is crucial:
   188            # Other nodes aren't able to look up the unqualified hostname.
   189            - "exec /cockroach/cockroach start --logtostderr --insecure --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
   190        # No pre-stop hook is required, a SIGTERM plus some time is all that's
   191        # needed for graceful shutdown of a node.
   192        terminationGracePeriodSeconds: 60
   193        volumes:
   194        - name: datadir
   195          persistentVolumeClaim:
   196            claimName: datadir
   197    podManagementPolicy: Parallel
   198    updateStrategy:
   199      type: RollingUpdate
   200    volumeClaimTemplates:
   201    - metadata:
   202        name: datadir
   203      spec:
   204        accessModes:
   205          - "ReadWriteOnce"
   206        # TODO: This specifically asks for a storage class with the name "ssd". A
   207        # storage class of this name doesn't exist by default. See our docs for
   208        # more information on how to create an optimized storage class for use here:
   209        # https://www.cockroachlabs.com/docs/stable/orchestrate-cockroachdb-with-kubernetes-performance.html#disk-type
   210        storageClassName: ssd
   211        resources:
   212          requests:
   213            # TODO: This asks for a fairly large disk by default because on
   214            # certain popular clouds there is a direct correlation between disk
   215            # size and the IOPS provisioned to the disk. Change this as necessary
   216            # to suit your needs, but be aware that smaller disks will typically
   217            # mean worse performance.
   218            storage: 1024Gi