github.com/miketheprogrammer/deis@v1.12.2/database/tests/recovery_test.go (about)

     1  package tests
     2  
     3  import (
     4  	"database/sql"
     5  	"fmt"
     6  	"github.com/deis/deis/tests/dockercli"
     7  	"github.com/deis/deis/tests/mock"
     8  	"github.com/deis/deis/tests/utils"
     9  	"github.com/lib/pq"
    10  	"testing"
    11  	"time"
    12  )
    13  
    14  func OpenDeisDatabase(t *testing.T, host string, port string) *sql.DB {
    15  	db, err := sql.Open("postgres", "postgres://deis:changeme123@"+host+":"+port+"/deis?sslmode=disable&connect_timeout=4")
    16  	if err != nil {
    17  		t.Fatal(err)
    18  	}
    19  	WaitForDatabase(t, db)
    20  	return db
    21  }
    22  
    23  func WaitForDatabase(t *testing.T, db *sql.DB) {
    24  	fmt.Println("--- Waiting for pg to be ready")
    25  	for {
    26  		err := db.Ping()
    27  		if err, ok := err.(*pq.Error); ok {
    28  			if err.Code.Name() == "cannot_connect_now" {
    29  				fmt.Println(err.Message)
    30  				time.Sleep(1000 * time.Millisecond)
    31  				continue
    32  			}
    33  			t.Fatal(err)
    34  		}
    35  		fmt.Println("Ready")
    36  		break
    37  	}
    38  }
    39  
    40  func TryTableSelect(t *testing.T, db *sql.DB, tableName string, expectFailure bool) {
    41  	_, err := db.Query("select * from " + tableName)
    42  
    43  	if expectFailure {
    44  		if err == nil {
    45  			t.Fatal("The table should not exist")
    46  		}
    47  	} else {
    48  		if err != nil {
    49  			t.Fatal(err)
    50  		}
    51  	}
    52  }
    53  
    54  func execSql(t *testing.T, db *sql.DB, q string) {
    55  	_, err := db.Query(q)
    56  	if err != nil {
    57  		t.Fatal(err)
    58  	}
    59  }
    60  
    61  func TestDatabaseRecovery(t *testing.T) {
    62  	var err error
    63  	tag, etcdPort := utils.BuildTag(), utils.RandomPort()
    64  	cli, stdout, _ := dockercli.NewClient()
    65  	imageName := utils.ImagePrefix() + "database" + ":" + tag
    66  
    67  	// start etcd container
    68  	etcdName := "deis-etcd-" + tag
    69  	dockercli.RunTestEtcd(t, etcdName, etcdPort)
    70  	defer cli.CmdRm("-f", etcdName)
    71  
    72  	// run mock ceph containers
    73  	cephName := "deis-ceph-" + tag
    74  	mock.RunMockCeph(t, cephName, cli, etcdPort)
    75  	defer cli.CmdRm("-f", cephName)
    76  
    77  	// create volumes
    78  	databaseVolumeA := "deis-database-data-a-" + tag
    79  	databaseVolumeB := "deis-database-data-b-" + tag
    80  	defer cli.CmdRm("-f", databaseVolumeA)
    81  	defer cli.CmdRm("-f", databaseVolumeB)
    82  	go func() {
    83  		fmt.Printf("--- Creating Volume A\n")
    84  		_ = cli.CmdRm("-f", "-v", databaseVolumeA)
    85  		dockercli.CreateVolume(t, cli, databaseVolumeA, "/var/lib/postgresql")
    86  
    87  		fmt.Printf("--- Creating Volume B\n")
    88  
    89  		_ = cli.CmdRm("-f", databaseVolumeB)
    90  		dockercli.CreateVolume(t, cli, databaseVolumeB, "/var/lib/postgresql")
    91  	}()
    92  	dockercli.WaitForLine(t, stdout, databaseVolumeB, true)
    93  
    94  	// setup database container start/stop routines
    95  	host, port := utils.HostAddress(), utils.RandomPort()
    96  	fmt.Printf("--- Run deis/database:%s at %s:%s\n", tag, host, port)
    97  	name := "deis-database-" + tag
    98  	defer cli.CmdRm("-f", name)
    99  	startDatabase := func(volumeName string) {
   100  		_ = cli.CmdRm("-f", name)
   101  		err = dockercli.RunContainer(cli,
   102  			"--name", name,
   103  			"--volumes-from", volumeName,
   104  			"--rm",
   105  			"-p", port+":5432",
   106  			"-e", "EXTERNAL_PORT="+port,
   107  			"-e", "HOST="+host,
   108  			"-e", "ETCD_PORT="+etcdPort,
   109  			"-e", "ETCD_TTL=2",
   110  			"-e", "BACKUP_FREQUENCY=1s",
   111  			"-e", "BACKUPS_TO_RETAIN=100",
   112  			imageName)
   113  	}
   114  
   115  	stopDatabase := func() {
   116  		fmt.Println("--- Stopping data-database... ")
   117  		if err = stdout.Close(); err != nil {
   118  			t.Fatal("Failed to closeStdout")
   119  		}
   120  		_ = cli.CmdStop(name)
   121  		fmt.Println("Done")
   122  	}
   123  
   124  	//ACTION
   125  
   126  	//STEP 1: start db with volume A and wait for init to complete
   127  	fmt.Println("--- Starting database with Volume A... ")
   128  	go startDatabase(databaseVolumeA)
   129  	dockercli.WaitForLine(t, stdout, "database: postgres is running...", true)
   130  	fmt.Println("Done")
   131  
   132  	db := OpenDeisDatabase(t, host, port)
   133  	TryTableSelect(t, db, "api_foo", true)
   134  
   135  	stopDatabase()
   136  
   137  	//STEP 2a: start db with volume B, wait for init and create the table
   138  	cli, stdout, _ = dockercli.NewClient()
   139  	fmt.Printf("--- Starting database with Volume B... ")
   140  	go startDatabase(databaseVolumeB)
   141  	dockercli.WaitForLine(t, stdout, "database: postgres is running...", true)
   142  	fmt.Println("Done")
   143  
   144  	db = OpenDeisDatabase(t, host, port)
   145  	TryTableSelect(t, db, "api_foo", true)
   146  
   147  	fmt.Println("--- Creating the table")
   148  	execSql(t, db, "create table api_foo(t text)")
   149  
   150  	//STEP 2b: make sure we observed full backup cycle after forced checkpoint
   151  	fmt.Println("--- Waiting for the change to be backed up... ")
   152  	dockercli.WaitForLine(t, stdout, "database: performing a backup...", true)
   153  	dockercli.WaitForLine(t, stdout, "database: backup has been completed.", true)
   154  	fmt.Println("Done")
   155  
   156  	stopDatabase()
   157  
   158  	//STEP 3: start db with volume A again and assert table existence
   159  	cli, stdout, _ = dockercli.NewClient()
   160  	fmt.Printf("--- Starting database with Volume A again... ")
   161  	go startDatabase(databaseVolumeA)
   162  	dockercli.WaitForLine(t, stdout, "database: postgres is running...", true)
   163  	fmt.Println("Done")
   164  
   165  	db = OpenDeisDatabase(t, host, port)
   166  	TryTableSelect(t, db, "api_foo", false)
   167  
   168  }