github.com/niedbalski/juju@v0.0.0-20190215020005-8ff100488e47/acceptancetests/repository/xenial/mysql/hooks/config-changed (about) 1 #!/usr/bin/python 2 3 from subprocess import check_output,check_call, CalledProcessError, Popen, PIPE 4 import tempfile 5 import json 6 import re 7 import hashlib 8 import os 9 import sys 10 import platform 11 from string import upper 12 13 num_re = re.compile('^[0-9]+$') 14 15 # There should be a library for this 16 def human_to_bytes(human): 17 if num_re.match(human): 18 return human 19 factors = { 'K' : 1024 , 'M' : 1048576, 'G' : 1073741824, 'T' : 1099511627776 } 20 modifier=human[-1] 21 if modifier in factors: 22 return int(human[:-1]) * factors[modifier] 23 if modifier == '%': 24 total_ram = human_to_bytes(get_memtotal()) 25 26 if IS_32BIT_SYSTEM and total_ram > SYS_MEM_LIMIT: 27 total_ram = SYS_MEM_LIMIT 28 29 factor = int(human[:-1]) * 0.01 30 pctram = total_ram * factor 31 return int(pctram - (pctram % PAGE_SIZE)) 32 raise ValueError("Can only convert K,M,G, or T") 33 34 35 # Going for the biggest page size to avoid wasted bytes. InnoDB page size is 36 # 16MB 37 PAGE_SIZE = 16*1024*1024 38 try: 39 IS_32BIT_SYSTEM = sys.maxsize < 2**32 40 except OverflowError: 41 IS_32BIT_SYSTEM = True 42 43 if platform.machine() in ['armv7l']: 44 SYS_MEM_LIMIT = human_to_bytes('2700M') # experimentally determined 45 else: 46 SYS_MEM_LIMIT = human_to_bytes('4G') 47 48 if IS_32BIT_SYSTEM: 49 check_call(['juju-log','-l','INFO','32bit system restrictions in play']) 50 51 configs=json.loads(check_output(['config-get','--format=json'])) 52 53 def get_memtotal(): 54 with open('/proc/meminfo') as meminfo_file: 55 meminfo = {} 56 for line in meminfo_file: 57 (key, mem) = line.split(':', 2) 58 if key == 'MemTotal': 59 (mtot, modifier) = mem.strip().split(' ') 60 return '%s%s' % (mtot, upper(modifier[0])) 61 62 63 # There is preliminary code for mariadb, but switching 64 # from mariadb -> mysql fails badly, so it is disabled for now. 65 valid_flavors = ['distro','percona'] 66 if configs['flavor'] not in valid_flavors: 67 check_call(['juju-log','-l', 68 'ERROR', 69 'Invalid flavor, must be one of %s' % ','.join(valid_flavors)]) 70 sys.exit(1) 71 72 remove_pkgs=[] 73 if configs['flavor'] == 'distro': 74 apt_sources = [] 75 package = 'mysql-server' 76 elif configs['flavor'] == 'percona': 77 apt_sources = ['repo.percona.com/apt'] 78 package = 'percona-server-server' 79 remove_pkgs = ['mysql-client-core-5.5','mysql-server-core-5.5'] 80 elif configs['flavor'] == 'mariadb': 81 apt_sources = ['ftp.osuosl.org/pub/mariadb/repo/5.5/ubuntu'] 82 package = 'mariadb-server' 83 84 series = check_output(['lsb_release','-cs']) 85 86 for source in apt_sources: 87 server = source.split('/')[0] 88 if os.path.exists('keys/%s' % server): 89 check_call(['apt-key','add','keys/%s' % server]) 90 else: 91 check_call(['juju-log','-l','ERROR', 92 'No key for %s' % (server)]) 93 sys.exit(1) 94 check_call(['add-apt-repository','-y','deb http://%s %s main' % (source, series)]) 95 check_call(['apt-get','update']) 96 97 with open('/var/lib/mysql/mysql.passwd','r') as rpw: 98 root_pass = rpw.read() 99 100 dconf = Popen(['debconf-set-selections'], stdin=PIPE) 101 dconf.stdin.write("%s %s/root_password password %s\n" % (package, package, root_pass)) 102 dconf.stdin.write("%s %s/root_password_again password %s\n" % (package, package, root_pass)) 103 dconf.communicate() 104 dconf.wait() 105 106 if len(remove_pkgs): 107 check_call(['apt-get','-y','remove'] + remove_pkgs) 108 check_call(['apt-get','-y','install','-qq',package]) 109 110 # smart-calc stuff in the configs 111 dataset_bytes = human_to_bytes(configs['dataset-size']) 112 113 check_call(['juju-log','-l','INFO','dataset size in bytes: %d' % dataset_bytes]) 114 115 if configs['query-cache-size'] == -1 and configs['query-cache-type'] in ['ON','DEMAND']: 116 qcache_bytes = (dataset_bytes * 0.20) 117 qcache_bytes = int(qcache_bytes - (qcache_bytes % PAGE_SIZE)) 118 configs['query-cache-size'] = qcache_bytes 119 dataset_bytes -= qcache_bytes 120 121 # 5.5 allows the words, but not 5.1 122 if configs['query-cache-type'] == 'ON': 123 configs['query-cache-type']=1 124 elif configs['query-cache-type'] == 'DEMAND': 125 configs['query-cache-type']=2 126 else: 127 configs['query-cache-type']=0 128 129 preferred_engines=configs['preferred-storage-engine'].split(',') 130 chunk_size = int(dataset_bytes / len(preferred_engines)) 131 configs['innodb-flush-log-at-trx-commit']=1 132 configs['sync-binlog']=1 133 if 'InnoDB' in preferred_engines: 134 configs['innodb-buffer-pool-size'] = chunk_size 135 if configs['tuning-level'] == 'fast': 136 configs['innodb-flush-log-at-trx-commit']=2 137 else: 138 configs['innodb-buffer-pool-size'] = 0 139 140 configs['default-storage-engine'] = preferred_engines[0] 141 142 if 'MyISAM' in preferred_engines: 143 configs['key-buffer'] = chunk_size 144 else: 145 # Need a bit for auto lookups always 146 configs['key-buffer'] = human_to_bytes('8M') 147 148 if configs['tuning-level'] == 'fast': 149 configs['sync-binlog']=0 150 151 if configs['max-connections'] == -1: 152 configs['max-connections'] = '# max_connections = ?' 153 else: 154 configs['max-connections'] = 'max_connections = %s' % configs['max-connections'] 155 156 template=""" 157 ###################################### 158 # 159 # 160 # 161 # This file generated by the juju MySQL charm! 162 # 163 # Local changes will not be preserved! 164 # 165 # 166 # 167 ###################################### 168 # 169 # The MySQL database server configuration file. 170 # 171 # You can copy this to one of: 172 # - "/etc/mysql/my.cnf" to set global options, 173 # - "~/.my.cnf" to set user-specific options. 174 # 175 # One can use all long options that the program supports. 176 # Run program with --help to get a list of available options and with 177 # --print-defaults to see which it would actually understand and use. 178 # 179 # For explanations see 180 # http://dev.mysql.com/doc/mysql/en/server-system-variables.html 181 182 # This will be passed to all mysql clients 183 # It has been reported that passwords should be enclosed with ticks/quotes 184 # escpecially if they contain "#" chars... 185 # Remember to edit /etc/mysql/debian.cnf when changing the socket location. 186 [client] 187 port = 3306 188 socket = /var/run/mysqld/mysqld.sock 189 190 # Here is entries for some specific programs 191 # The following values assume you have at least 32M ram 192 193 # This was formally known as [safe_mysqld]. Both versions are currently parsed. 194 [mysqld_safe] 195 socket = /var/run/mysqld/mysqld.sock 196 nice = 0 197 198 [mysqld] 199 # 200 # * Basic Settings 201 # 202 203 # 204 # * IMPORTANT 205 # If you make changes to these settings and your system uses apparmor, you may 206 # also need to also adjust /etc/apparmor.d/usr.sbin.mysqld. 207 # 208 209 user = mysql 210 socket = /var/run/mysqld/mysqld.sock 211 port = 3306 212 basedir = /usr 213 datadir = /var/lib/mysql 214 tmpdir = /tmp 215 skip-external-locking 216 # 217 # Instead of skip-networking the default is now to listen only on 218 # localhost which is more compatible and is not less secure. 219 bind-address = 0.0.0.0 220 # 221 # * Fine Tuning 222 # 223 key_buffer = %(key-buffer)s 224 max_allowed_packet = 16M 225 # This replaces the startup script and checks MyISAM tables if needed 226 # the first time they are touched 227 myisam-recover = BACKUP 228 %(max-connections)s 229 #table_cache = 64 230 #thread_concurrency = 10 231 # 232 # * Query Cache Configuration 233 # 234 query_cache_limit = 1M 235 query_cache_size = %(query-cache-size)s 236 query_cache_type = %(query-cache-type)s 237 # 238 # * Logging and Replication 239 # 240 # Both location gets rotated by the cronjob. 241 # Be aware that this log type is a performance killer. 242 # As of 5.1 you can enable the log at runtime! 243 #general_log_file = /var/log/mysql/mysql.log 244 #general_log = 1 245 246 log_error = /var/log/mysql/error.log 247 248 # Here you can see queries with especially long duration 249 #log_slow_queries = /var/log/mysql/mysql-slow.log 250 #long_query_time = 2 251 #log-queries-not-using-indexes 252 # 253 # The following can be used as easy to replay backup logs or for replication. 254 # note: if you are setting up a replication slave, see README.Debian about 255 # other settings you may need to change. 256 #server-id = 1 257 #log_bin = /var/log/mysql/mysql-bin.log 258 expire_logs_days = 10 259 max_binlog_size = 100M 260 #binlog_do_db = include_database_name 261 #binlog_ignore_db = include_database_name 262 # 263 # * InnoDB 264 # 265 # InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/. 266 # Read the manual for more InnoDB related options. There are many! 267 # 268 innodb_buffer_pool_size = %(innodb-buffer-pool-size)s 269 innodb_flush_log_at_trx_commit = %(innodb-flush-log-at-trx-commit)s 270 sync_binlog = %(sync-binlog)s 271 default_storage_engine = %(default-storage-engine)s 272 skip-name-resolve 273 # * Security Features 274 # 275 # Read the manual, too, if you want chroot! 276 # chroot = /var/lib/mysql/ 277 # 278 # For generating SSL certificates I recommend the OpenSSL GUI "tinyca". 279 # 280 # ssl-ca=/etc/mysql/cacert.pem 281 # ssl-cert=/etc/mysql/server-cert.pem 282 # ssl-key=/etc/mysql/server-key.pem 283 284 285 286 [mysqldump] 287 quick 288 quote-names 289 max_allowed_packet = 16M 290 291 [mysql] 292 #no-auto-rehash # faster start of mysql but no tab completition 293 294 [isamchk] 295 key_buffer = 16M 296 297 # 298 # * IMPORTANT: Additional settings that can override those from this file! 299 # The files must end with '.cnf', otherwise they'll be ignored. 300 # 301 !includedir /etc/mysql/conf.d/ 302 """ 303 304 i_am_a_slave = os.path.isfile('/var/lib/juju/i.am.a.slave') 305 unit_id = os.environ['JUJU_UNIT_NAME'].split('/')[1] 306 307 if not i_am_a_slave and configs['tuning-level'] == 'fast': 308 binlog_cnf = '' 309 else: 310 # On slaves, this gets overwritten 311 binlog_template = """ 312 [mysqld] 313 server_id = %s 314 log_bin = /var/log/mysql/mysql-bin.log 315 binlog_format = %s 316 """ 317 318 binlog_cnf = binlog_template % (unit_id, 319 configs.get('binlog-format','MIXED')) 320 321 mycnf=template % configs 322 323 targets = {'/etc/mysql/conf.d/binlog.cnf': binlog_cnf, 324 '/etc/mysql/my.cnf': mycnf, 325 } 326 327 need_restart = False 328 for target,content in targets.iteritems(): 329 tdir = os.path.dirname(target) 330 if len(content) == 0 and os.path.exists(target): 331 os.unlink(target) 332 need_restart = True 333 continue 334 with tempfile.NamedTemporaryFile(mode='w',dir=tdir,delete=False) as t: 335 t.write(content) 336 t.flush() 337 tmd5 = hashlib.md5() 338 tmd5.update(content) 339 if os.path.exists(target): 340 with open(target,'r') as old: 341 md5=hashlib.md5() 342 md5.update(old.read()) 343 oldhash = md5.digest() 344 if oldhash != tmd5.digest(): 345 os.rename(target,'%s.%s' % (target, md5.hexdigest())) 346 need_restart = True 347 else: 348 need_restart = True 349 os.rename(t.name, target) 350 351 if need_restart: 352 try: 353 check_call(['service','mysql','stop']) 354 except CalledProcessError: 355 pass 356 check_call(['service','mysql','start'])