github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/cloud/kubernetes/performance/cockroachdb-statefulset-secure.yaml (about)

     1  # This configuration file sets up a secure StatefulSet running CockroachDB with
     2  # tweaks to make it more performant than our default configuration files. All
     3  # changes from the default secure configuration have been marked with a comment
     4  # starting with "NOTE" or "TODO".
     5  #
     6  # To use it, customize all the parts of the file labeled "TODO" before running:
     7  #   kubectl create -f cockroachdb-statefulset-secure.yaml
     8  #
     9  # You will then have to approve certificate-signing requests and initialize the
    10  # cluster as described in the parent directory's README.md file.
    11  #
    12  # If you don't see any pods being created, it's possible that your cluster was
    13  # not able to meet the resource requests asked for, whether it was the amount
    14  # of CPU, memory, or disk or the disk type. To find information about why pods
    15  # haven't been created, you can run:
    16  #   kubectl get events
    17  #
    18  # For more information on improving CockroachDB performance in Kubernetes, see
    19  # our docs:
    20  # https://www.cockroachlabs.com/docs/stable/kubernetes-performance.html
    21  apiVersion: v1
    22  kind: ServiceAccount
    23  metadata:
    24    name: cockroachdb
    25    labels:
    26      app: cockroachdb
    27  ---
    28  apiVersion: rbac.authorization.k8s.io/v1beta1
    29  kind: Role
    30  metadata:
    31    name: cockroachdb
    32    labels:
    33      app: cockroachdb
    34  rules:
    35  - apiGroups:
    36    - ""
    37    resources:
    38    - secrets
    39    verbs:
    40    - create
    41    - get
    42  ---
    43  apiVersion: rbac.authorization.k8s.io/v1beta1
    44  kind: ClusterRole
    45  metadata:
    46    name: cockroachdb
    47    labels:
    48      app: cockroachdb
    49  rules:
    50  - apiGroups:
    51    - certificates.k8s.io
    52    resources:
    53    - certificatesigningrequests
    54    verbs:
    55    - create
    56    - get
    57    - watch
    58  ---
    59  apiVersion: rbac.authorization.k8s.io/v1beta1
    60  kind: RoleBinding
    61  metadata:
    62    name: cockroachdb
    63    labels:
    64      app: cockroachdb
    65  roleRef:
    66    apiGroup: rbac.authorization.k8s.io
    67    kind: Role
    68    name: cockroachdb
    69  subjects:
    70  - kind: ServiceAccount
    71    name: cockroachdb
    72    namespace: default
    73  ---
    74  apiVersion: rbac.authorization.k8s.io/v1beta1
    75  kind: ClusterRoleBinding
    76  metadata:
    77    name: cockroachdb
    78    labels:
    79      app: cockroachdb
    80  roleRef:
    81    apiGroup: rbac.authorization.k8s.io
    82    kind: ClusterRole
    83    name: cockroachdb
    84  subjects:
    85  - kind: ServiceAccount
    86    name: cockroachdb
    87    namespace: default
    88  ---
    89  apiVersion: v1
    90  kind: Service
    91  metadata:
    92    # This service is meant to be used by clients of the database. It exposes a ClusterIP that will
    93    # automatically load balance connections to the different database pods.
    94    name: cockroachdb-public
    95    labels:
    96      app: cockroachdb
    97  spec:
    98    ports:
    99    # The main port, served by gRPC, serves Postgres-flavor SQL, internode
   100    # traffic and the cli.
   101    - port: 26257
   102      targetPort: 26257
   103      name: grpc
   104    # The secondary port serves the UI as well as health and debug endpoints.
   105    - port: 8080
   106      targetPort: 8080
   107      name: http
   108    selector:
   109      app: cockroachdb
   110  ---
   111  apiVersion: v1
   112  kind: Service
   113  metadata:
   114    # This service only exists to create DNS entries for each pod in the stateful
   115    # set such that they can resolve each other's IP addresses. It does not
   116    # create a load-balanced ClusterIP and should not be used directly by clients
   117    # in most circumstances.
   118    name: cockroachdb
   119    labels:
   120      app: cockroachdb
   121    annotations:
   122      # Use this annotation in addition to the actual publishNotReadyAddresses
   123      # field below because the annotation will stop being respected soon but the
   124      # field is broken in some versions of Kubernetes:
   125      # https://github.com/kubernetes/kubernetes/issues/58662
   126      service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
   127      # Enable automatic monitoring of all instances when Prometheus is running in the cluster.
   128      prometheus.io/scrape: "true"
   129      prometheus.io/path: "_status/vars"
   130      prometheus.io/port: "8080"
   131  spec:
   132    ports:
   133    - port: 26257
   134      targetPort: 26257
   135      name: grpc
   136    - port: 8080
   137      targetPort: 8080
   138      name: http
   139    # We want all pods in the StatefulSet to have their addresses published for
   140    # the sake of the other CockroachDB pods even before they're ready, since they
   141    # have to be able to talk to each other in order to become ready.
   142    publishNotReadyAddresses: true
   143    clusterIP: None
   144    selector:
   145      app: cockroachdb
   146  ---
   147  apiVersion: policy/v1beta1
   148  kind: PodDisruptionBudget
   149  metadata:
   150    name: cockroachdb-budget
   151    labels:
   152      app: cockroachdb
   153  spec:
   154    selector:
   155      matchLabels:
   156        app: cockroachdb
   157    maxUnavailable: 1
   158  ---
   159  apiVersion: apps/v1
   160  kind: StatefulSet
   161  metadata:
   162    name: cockroachdb
   163  spec:
   164    serviceName: "cockroachdb"
   165    replicas: 3
   166    selector:
   167      matchLabels:
   168        app: cockroachdb
   169    template:
   170      metadata:
   171        labels:
   172          app: cockroachdb
   173      spec:
   174        serviceAccountName: cockroachdb
   175        # NOTE: Running with `hostNetwork: true` means that CockroachDB will use
   176        # the host machines' IP address and hostname, and that nothing else on
   177        # the machines will be able to use the same ports. This means that only 1
   178        # CockroachDB pod will ever be schedulable on the same machine, because
   179        # otherwise their ports would conflict.
   180        #
   181        # If your client pods generate a lot of network traffic to and from the
   182        # CockroachDB cluster, you may see a benefit to doing the same thing in
   183        # their configurations.
   184        hostNetwork: true
   185        dnsPolicy: ClusterFirstWithHostNet
   186        # Init containers are run only once in the lifetime of a pod, before
   187        # it's started up for the first time. It has to exit successfully
   188        # before the pod's main containers are allowed to start.
   189        initContainers:
   190        # The init-certs container sends a certificate signing request to the
   191        # kubernetes cluster.
   192        # You can see pending requests using: kubectl get csr
   193        # CSRs can be approved using:         kubectl certificate approve <csr name>
   194        #
   195        # All addresses used to contact a node must be specified in the --addresses arg.
   196        #
   197        # In addition to the node certificate and key, the init-certs entrypoint will symlink
   198        # the cluster CA to the certs directory.
   199        - name: init-certs
   200          image: cockroachdb/cockroach-k8s-request-cert:0.4
   201          imagePullPolicy: IfNotPresent
   202          command:
   203          - "/bin/ash"
   204          - "-ecx"
   205          - "/request-cert -namespace=${POD_NAMESPACE} -certs-dir=/cockroach-certs -type=node -addresses=localhost,127.0.0.1,$(hostname -f),$(hostname -f|cut -f 1-2 -d '.'),cockroachdb-public,cockroachdb-public.$(hostname -f|cut -f 3- -d '.'),cockroachdb-public.$(hostname -f|cut -f 3-4 -d '.'),cockroachdb-public.$(hostname -f|cut -f 3 -d '.') -symlink-ca-from=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
   206          env:
   207          - name: POD_NAMESPACE
   208            valueFrom:
   209              fieldRef:
   210                fieldPath: metadata.namespace
   211          volumeMounts:
   212          - name: certs
   213            mountPath: /cockroach-certs
   214        # NOTE: If you are running clients that generate heavy load, you may find
   215        # it useful to copy this anti-affinity policy into the client pods'
   216        # configurations as well to avoid running them on the same machines as
   217        # CockroachDB and interfering with each other's performance.
   218        affinity:
   219          podAntiAffinity:
   220            preferredDuringSchedulingIgnoredDuringExecution:
   221            - weight: 100
   222              podAffinityTerm:
   223                labelSelector:
   224                  matchExpressions:
   225                  - key: app
   226                    operator: In
   227                    values:
   228                    - cockroachdb
   229                topologyKey: kubernetes.io/hostname
   230        containers:
   231        - name: cockroachdb
   232          # NOTE: Always use the most recent version of CockroachDB for the best
   233          # performance and reliability.
   234          image: cockroachdb/cockroach:v20.1.1
   235          imagePullPolicy: IfNotPresent
   236          # TODO: Change these to appropriate values for the hardware that you're running. You can see
   237          # the amount of allocatable resources on each of your Kubernetes nodes by running:
   238          #   kubectl describe nodes
   239          resources:
   240            requests:
   241              cpu: "16"
   242              memory: "8Gi"
   243            limits:
   244              # NOTE: Unless you have enabled the non-default Static CPU Management Policy
   245              # and are using an integer number of CPUs, we don't recommend setting a CPU limit.
   246              # See:
   247              #   https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#static-policy
   248              #   https://github.com/kubernetes/kubernetes/issues/51135
   249              #cpu: "16"
   250              memory: "8Gi"
   251          ports:
   252          - containerPort: 26257
   253            name: grpc
   254          - containerPort: 8080
   255            name: http
   256          livenessProbe:
   257            httpGet:
   258              path: "/health"
   259              port: http
   260              scheme: HTTPS
   261            initialDelaySeconds: 30
   262            periodSeconds: 5
   263          readinessProbe:
   264            httpGet:
   265              path: "/health?ready=1"
   266              port: http
   267              scheme: HTTPS
   268            initialDelaySeconds: 10
   269            periodSeconds: 5
   270            failureThreshold: 2
   271          volumeMounts:
   272          - name: datadir
   273            mountPath: /cockroach/cockroach-data
   274          - name: certs
   275            mountPath: /cockroach/cockroach-certs
   276          env:
   277          - name: COCKROACH_CHANNEL
   278            value: kubernetes-secure
   279          command:
   280            - "/bin/bash"
   281            - "-ecx"
   282            # The use of qualified `hostname -f` is crucial:
   283            # Other nodes aren't able to look up the unqualified hostname.
   284            - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%"
   285        # No pre-stop hook is required, a SIGTERM plus some time is all that's
   286        # needed for graceful shutdown of a node.
   287        terminationGracePeriodSeconds: 60
   288        volumes:
   289        - name: datadir
   290          persistentVolumeClaim:
   291            claimName: datadir
   292        - name: certs
   293          emptyDir: {}
   294    podManagementPolicy: Parallel
   295    updateStrategy:
   296      type: RollingUpdate
   297    volumeClaimTemplates:
   298    - metadata:
   299        name: datadir
   300      spec:
   301        accessModes:
   302          - "ReadWriteOnce"
   303        # TODO: This specifically asks for a storage class with the name "ssd". A
   304        # storage class of this name doesn't exist by default. See our docs for
   305        # more information on how to create an optimized storage class for use here:
   306        # https://www.cockroachlabs.com/docs/stable/orchestrate-cockroachdb-with-kubernetes-performance.html#disk-type
   307        storageClassName: ssd
   308        resources:
   309          requests:
   310            # TODO: This asks for a fairly large disk by default because on
   311            # certain popular clouds there is a direct correlation between disk
   312            # size and the IOPS provisioned to the disk. Change this as necessary
   313            # to suit your needs, but be aware that smaller disks will typically
   314            # mean worse performance.
   315            storage: 1024Gi