github.com/1aal/kubeblocks@v0.0.0-20231107070852-e1c03e598921/deploy/orioledb/templates/scripts.yaml (about) 1 apiVersion: v1 2 kind: ConfigMap 3 metadata: 4 name: orioledb-scripts 5 labels: 6 {{- include "orioledb.labels" . | nindent 4 }} 7 data: 8 {{/* init_container.sh: |*/}} 9 {{/* #!/bin/bash*/}} 10 {{/* set -o errexit*/}} 11 {{/* set -e*/}} 12 {{/* #mkdir -p /home/postgres/pgdata/conf*/}} 13 {{/* chmod 750 -R /home/postgres/pgdata*/}} 14 {{/* mkdir -p /var/lib/postgresql/conf*/}} 15 {{/* cp /home/postgres/conf/postgresql.conf /var/lib/postgresql/conf*/}} 16 {{/* chmod 777 -R /var/lib/postgresql/conf*/}} 17 {{/* cp /home/postgres/conf/postgresql.conf /home/postgres/pgdata/conf*/}} 18 {{/* chmod 777 /home/postgres/pgdata/conf/postgresql.conf*/}} 19 generate_patroni_yaml.py: | 20 #!/usr/bin/env python3 21 # -*- coding: utf-8 -*- 22 import os 23 import sys 24 import yaml 25 def write_file(config, filename, overwrite): 26 if not overwrite and os.path.exists(filename): 27 pass 28 else: 29 with open(filename, 'w') as f: 30 f.write(config) 31 def read_file_lines(file): 32 ret = [] 33 for line in file.readlines(): 34 line = line.strip() 35 if line and not line.startswith('#'): 36 ret.append(line) 37 return ret 38 def postgresql_conf_to_dict(file_path): 39 with open(file_path, 'r') as f: 40 content = f.read() 41 lines = content.splitlines() 42 result = {} 43 for line in lines: 44 if line.startswith('#'): 45 continue 46 if '=' not in line: 47 continue 48 key, value = line.split('=', 1) 49 result[key.strip()] = value.strip().strip("'") 50 return result 51 def main(filename): 52 restore_dir = os.environ.get('RESTORE_DATA_DIR', '') 53 local_config = yaml.safe_load( 54 os.environ.get('SPILO_CONFIGURATION', os.environ.get('PATRONI_CONFIGURATION', ''))) or {} 55 podip = os.environ.get('POD_IP') 56 # scope 57 local_config['scope'] = os.environ.get('SCOPE') 58 # name 59 local_config['name'] = os.environ.get('KB_POD_NAME') 60 # etcd3 61 local_config['etcd3'] = { 62 'host':os.environ.get('PATRONI_ETCD3_HOST') 63 } 64 # restapi 65 local_config['restapi'] = { 66 'listen': f'{podip}:8008', 67 'connect_address': f'{podip}:8008', 68 } 69 # postgresql 70 if not 'postgresql' in local_config: 71 local_config['postgresql'] = {} 72 postgresql = local_config['postgresql'] 73 postgresql['data_dir'] = os.environ.get('PGDATA') 74 postgresql['config_dir'] = '/home/postgres/pgdata/conf' 75 postgresql['custom_conf'] = '/home/postgres/conf/postgresql.conf' 76 postgresql['authentication'] = {} 77 postgresql['listen'] = '0.0.0.0:5432' 78 postgresql['connect_address'] = f'{podip}:5432' 79 authentication = postgresql['authentication'] 80 authentication['superuser'] = {"username": os.environ.get('PGUSER_SUPERUSER'), 81 'password': os.environ.get('PGPASSWORD_SUPERUSER')} 82 authentication['replication'] = {'username':os.environ.get('PGUSER_SUPERUSER'), 83 'password':os.environ.get('POSTGRES_PASSWORD')} 84 85 # add pg_hba.conf 86 with open('/home/postgres/conf/pg_hba.conf', 'r') as f: 87 lines = read_file_lines(f) 88 if lines: 89 postgresql['pg_hba'] = lines 90 91 if restore_dir and os.path.isfile(os.path.join(restore_dir, 'kb_restore.signal')): 92 if not 'bootstrap' in local_config: 93 local_config['bootstrap'] = {} 94 with open('/home/postgres/conf/kb_restore.conf', 'r') as f: 95 local_config['bootstrap'].update(yaml.safe_load(f)) 96 97 local_config['bootstrap'] = {} 98 bootstrap = local_config['bootstrap'] 99 bootstrap['dcs'] = { 100 'postgresql':{ 101 'parameters':{'listen_addresses':"0.0.0.0","port":5432} 102 } 103 } 104 # point in time recovery(PITR) 105 data_dir = os.environ.get('PGDATA', '') 106 if os.path.isfile("/home/postgres/pgdata/conf/recovery.conf"): 107 with open('/home/postgres/conf/kb_pitr.conf', 'r') as f: 108 pitr_config = yaml.safe_load(f) 109 re_config = postgresql_conf_to_dict("/home/postgres/pgdata/conf/recovery.conf") 110 pitr_config[pitr_config['method']]['recovery_conf'].update(re_config) 111 local_config['bootstrap'].update(pitr_config) 112 # bootstrap 113 {{/* if 'bootstrap' not in local_config.keys():*/}} 114 {{/* local_config['bootstrap'] = {*/}} 115 {{/* }*/}} 116 write_file(yaml.dump(local_config, default_flow_style=False), filename, True) 117 if __name__ == '__main__': 118 main(sys.argv[1]) 119 init.sql: | 120 CREATE EXTENSION pg_stat_statements; 121 setup.sh: | 122 if [ -d "/home/postgres/pgdata/conf" ]; then 123 chmod 750 /home/postgres/pgdata/pgroot/data 124 else 125 mkdir -p /home/postgres/pgdata/conf 126 chmod 777 -R /home/postgres/pgdata 127 cp /home/postgres/conf/postgresql.conf /home/postgres/pgdata/conf 128 chmod 777 -R /home/postgres/pgdata/conf 129 fi 130 source /dependency/conf 131 132 {{/* chmod 750 -R /home/postgres/pgdata*/}} 133 {{/* mkdir -p /var/lib/postgresql/conf*/}} 134 #cp /home/postgres/conf/postgresql.conf /var/lib/postgresql/conf 135 #chmod 777 -R /var/lib/postgresql/conf 136 #su - postgres -c "" 137 IFS='-' read -ra parts <<< "$KB_POD_NAME" 138 if [ ${parts[-1]} != '0' ]; then 139 MAX_RETRIES=30 140 WAIT_INTERVAL=5 141 retries=0 142 while [ $retries -lt $MAX_RETRIES ]; do 143 pg_isready -h $KB_ORIOLEDB_0_HOSTNAME 144 if [ $? -eq 0 ]; then 145 echo "PostgreSQL is ready!" 146 break 147 else 148 echo "PostgreSQL is not ready yet. Retrying in $WAIT_INTERVAL seconds..." 149 sleep $WAIT_INTERVAL 150 retries=$((retries + 1)) 151 fi 152 done 153 else 154 cp /kb-scripts/init.sql /docker-entrypoint-initdb.d 155 bash /usr/local/bin/docker-entrypoint.sh postgres 156 fi 157 #sleep 10000 158 python3 /kb-scripts/generate_patroni_yaml.py /var/lib/postgresql/tmp_patroni.yaml 159 {{/* python3 /kb-scripts/generate_patroni_yaml.py /var/lib/postgresql/user2.yaml*/}} 160 {{/* #python3 /kb-scripts/generate_patroni_yaml.py /var/lib/postgresql/user2.yaml*/}} 161 chmod 777 /var/lib/postgresql/tmp_patroni.yaml 162 #sleep 10000 163 su - postgres -c "patroni /var/lib/postgresql/tmp_patroni.yaml" 164 165 166 {{/* target_data_dir=/home/postgres/pgdata*/}} 167 {{/* ln -s $target_data_dir $PGDATA*/}} 168 {{/* su - postgres*/}} 169 {{/* postgres --config-file /var/lib/postgresql/data/pgdata/conf*/}} 170 {{/* sudo systemctl stop postgresql*/}} 171 {{/* #!/bin/bash*/}} 172 {{/* set -o errexit*/}} 173 {{/* set -ex*/}} 174 175 {{/* # Waiting for primary pod information from the DownwardAPI annotation to be available, with a maximum of 5 attempts*/}} 176 {{/* attempt=1*/}} 177 {{/* max_attempts=5*/}} 178 {{/* while [ $attempt -le $max_attempts ] && [ -z "$(cat /kb-podinfo/primary-pod)" ]; do*/}} 179 {{/* sleep 5*/}} 180 {{/* attempt=$((attempt + 1))*/}} 181 {{/* done*/}} 182 183 {{/* primary=$(cat /kb-podinfo/primary-pod)*/}} 184 {{/* echo "DownwardAPI get primary=$primary" >> /home/postgres/pgdata/.kb_set_up.log*/}} 185 {{/* echo "KB_POD_NAME=$KB_POD_NAME" >> /home/postgres/pgdata/.kb_set_up.log*/}} 186 {{/* if [ -z "$primary" ]; then*/}} 187 {{/* echo "Primary pod information not available. Exiting..."*/}} 188 {{/* exit 1*/}} 189 {{/* fi*/}} 190 191 {{/* # usage: retry <command>*/}} 192 {{/* # e.g. retry pg_isready -U postgres -h $primary_fqdn -p 5432*/}} 193 {{/* function retry {*/}} 194 {{/* local max_attempts=10*/}} 195 {{/* local attempt=1*/}} 196 {{/* until "$@" || [ $attempt -eq $max_attempts ]; do*/}} 197 {{/* echo "Command '$*' failed. Attempt $attempt of $max_attempts. Retrying in 5 seconds..."*/}} 198 {{/* attempt=$((attempt + 1))*/}} 199 {{/* sleep 5*/}} 200 {{/* done*/}} 201 {{/* if [ $attempt -eq $max_attempts ]; then*/}} 202 {{/* echo "Command '$*' failed after $max_attempts attempts. Exiting..."*/}} 203 {{/* exit 1*/}} 204 {{/* fi*/}} 205 {{/* }*/}} 206 {{/* if [ "$primary" != "$KB_POD_NAME" ]; then*/}} 207 {{/* primary_fqdn="$primary.$KB_CLUSTER_NAME-$KB_COMP_NAME-headless.$KB_NAMESPACE.svc"*/}} 208 {{/* echo "primary_fqdn=$primary_fqdn" >> /home/postgres/pgdata/.kb_set_up.log*/}} 209 {{/* # waiting for the primary to be ready, if the wait time exceeds the maximum number of retries, then the script will fail and exit.*/}} 210 {{/* retry pg_isready -U {{ default "postgres" | quote }} -h $primary_fqdn -p 5432*/}} 211 {{/* fi*/}} 212 213 {{/* if [ -f ${RESTORE_DATA_DIR}/kb_restore.signal ]; then*/}} 214 {{/* chown -R postgres ${RESTORE_DATA_DIR}*/}} 215 {{/* fi*/}} 216 {{/* python3 /kb-scripts/generate_patroni_yaml.py tmp_patroni.yaml*/}} 217 {{/* export SPILO_CONFIGURATION=$(cat tmp_patroni.yaml)*/}} 218 {{/* exec /launch.sh init*/}} 219 pgbouncer_setup.sh: | 220 #!/bin/bash 221 set -o errexit 222 set -ex 223 mkdir -p /opt/bitnami/pgbouncer/conf/ /opt/bitnami/pgbouncer/logs/ /opt/bitnami/pgbouncer/tmp/ 224 cp /home/pgbouncer/conf/pgbouncer.ini /opt/bitnami/pgbouncer/conf/ 225 echo "\"$POSTGRESQL_USERNAME\" \"$POSTGRESQL_PASSWORD\"" > /opt/bitnami/pgbouncer/conf/userlist.txt 226 echo -e "\\n[databases]" >> /opt/bitnami/pgbouncer/conf/pgbouncer.ini 227 echo "postgres=host=$KB_POD_IP port=5432 dbname=postgres" >> /opt/bitnami/pgbouncer/conf/pgbouncer.ini 228 chmod 777 /opt/bitnami/pgbouncer/conf/pgbouncer.ini 229 chmod 777 /opt/bitnami/pgbouncer/conf/userlist.txt 230 useradd pgbouncer 231 chown -R pgbouncer:pgbouncer /opt/bitnami/pgbouncer/conf/ /opt/bitnami/pgbouncer/logs/ /opt/bitnami/pgbouncer/tmp/ 232 /opt/bitnami/scripts/pgbouncer/run.sh 233 backup-log-collector.sh: | 234 #!/bin/bash 235 set -o errexit 236 set -o nounset 237 SHOW_START_TIME=$1 238 LOG_START_TIME="" 239 LOG_STOP_TIME="" 240 if [ "$SHOW_START_TIME" == "false" ]; then 241 latest_done_wal=$(ls -t ${PGDATA}/pg_wal/archive_status/|grep ".done"|head -n 1) 242 if [ "${latest_done_wal}" != "" ]; then 243 LOG_STOP_TIME=$(pg_waldump ${latest_done_wal%.*} --rmgr=Transaction 2>/dev/null |tail -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') 244 fi 245 [[ "${LOG_STOP_TIME}" != "" ]] && printf "{\"stopTime\": \"$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ')\"}" || printf "{}" 246 else 247 LOG_START_TIME=$(pg_waldump $(ls -Ftr $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'|head -n 1) --rmgr=Transaction 2>/dev/null |head -n 1|awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') 248 for i in $(ls -Ft $PGDATA/pg_wal/ | grep '[[:xdigit:]]$\|.partial$'); do LOG_STOP_TIME=$(pg_waldump $i --rmgr=Transaction 2>/dev/null|tail -n 1); [[ "$LOG_STOP_TIME" != "" ]] && break; done 249 LOG_STOP_TIME=$(echo $LOG_STOP_TIME |awk -F ' COMMIT ' '{print $2}'|awk -F ';' '{print $1}') 250 if [ "${LOG_START_TIME}" == "" ]; then LOG_START_TIME=${LOG_STOP_TIME}; fi 251 LOG_START_TIME=$(date -d "$LOG_START_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') 252 LOG_STOP_TIME=$(date -d "$LOG_STOP_TIME" -u '+%Y-%m-%dT%H:%M:%SZ') 253 printf "{\"startTime\": \"$LOG_START_TIME\" ,\"stopTime\": \"$LOG_STOP_TIME\"}" 254 fi 255 filesize-collector.sh: | 256 #!/bin/bash 257 set -e; 258 function getProperty() { 259 file=$1; key=$2; 260 echo $(grep "${key}: " ${file} | awk -F ': ' '{print $2}') 261 } 262 filename=$1 263 fileinfo=${PGDATA}/${filename} 264 if [ -f ${fileinfo} ]; then 265 TOTAL_SIZE=$(getProperty ${fileinfo} "TOTAL SIZE") 266 rm -f ${fileinfo} 267 printf "{\"totalSize\":\"${TOTAL_SIZE}\",\"manifests\":{\"backupTool\":{\"uploadTotalSize\":\"${TOTAL_SIZE}\"}}}" 268 else 269 printf "{}" 270 fi