github.com/tuhaihe/gpbackup@v1.0.3/integration/incremental_queries_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/tuhaihe/gp-common-go-libs/testhelper"
     7  	"github.com/tuhaihe/gpbackup/backup"
     8  	"github.com/tuhaihe/gpbackup/options"
     9  	"github.com/tuhaihe/gpbackup/toc"
    10  
    11  	. "github.com/onsi/ginkgo/v2"
    12  	. "github.com/onsi/gomega"
    13  )
    14  
    15  var _ = Describe("backup integration tests", func() {
    16  	const (
    17  		dropTableSQL  = "DROP TABLE %s"
    18  		addColumnSQL  = "ALTER TABLE %s ADD COLUMN k int DEFAULT 0"
    19  		dropColumnSQL = "ALTER TABLE %s DROP COLUMN k"
    20  		insertSQL     = "INSERT INTO %s values(10)"
    21  		deleteSQL     = "DELETE FROM %s"
    22  	)
    23  
    24  	var aoTableFQN = "public.ao_foo"
    25  	var aoCOTableFQN = "public.ao_co_foo"
    26  	var aoPartParentTableFQN = "public.ao_part"
    27  	var aoPartChildTableFQN = "public.ao_part_1_prt_child"
    28  	BeforeEach(func() {
    29  		testhelper.AssertQueryRuns(connectionPool,
    30  			fmt.Sprintf("CREATE TABLE %s (i int) WITH (appendonly=true)", aoTableFQN))
    31  		testhelper.AssertQueryRuns(connectionPool,
    32  			fmt.Sprintf("CREATE TABLE %s (i int) WITH (appendonly=true,orientation='column')", aoCOTableFQN))
    33  		testhelper.AssertQueryRuns(connectionPool,
    34  			fmt.Sprintf(`CREATE TABLE %s (i int) WITH (appendonly=true)
    35  	DISTRIBUTED BY (i)
    36  	PARTITION BY LIST (i)
    37  	(
    38  		PARTITION child VALUES (10)
    39  	);`, aoPartParentTableFQN))
    40  	})
    41  	AfterEach(func() {
    42  		testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoTableFQN))
    43  		testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoCOTableFQN))
    44  		testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoPartParentTableFQN))
    45  	})
    46  	Describe("GetAOIncrementalMetadata", func() {
    47  		Context("AO, AO_CO and AO partition tables are only just created", func() {
    48  			var aoIncrementalMetadata map[string]toc.AOEntry
    49  			BeforeEach(func() {
    50  				aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
    51  			})
    52  			It("should have a modcount of 0", func() {
    53  				Expect(aoIncrementalMetadata[aoTableFQN].Modcount).To(Equal(int64(0)))
    54  				Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount).To(Equal(int64(0)))
    55  				Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount).To(Equal(int64(0)))
    56  				Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount).To(Equal(int64(0)))
    57  			})
    58  			It("should have a last DDL timestamp", func() {
    59  				Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
    60  				Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
    61  				Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
    62  
    63  				// For GPDB 7+, the root partition is not included.
    64  				// TODO: Should the root be included?
    65  				if false {
    66  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
    67  				}
    68  
    69  			})
    70  		})
    71  		Context("AO, AO_CO and AO partition tables have DML changes", func() {
    72  			Context("After an insert(s)", func() {
    73  				var initialAOIncrementalMetadata map[string]toc.AOEntry
    74  				var aoIncrementalMetadata map[string]toc.AOEntry
    75  				BeforeEach(func() {
    76  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
    77  
    78  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN))
    79  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN))
    80  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN))
    81  
    82  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
    83  				})
    84  				AfterEach(func() {
    85  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoTableFQN))
    86  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoCOTableFQN))
    87  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoPartParentTableFQN))
    88  				})
    89  				It("should increase modcount for non partition tables", func() {
    90  					Expect(aoIncrementalMetadata[aoTableFQN].Modcount).
    91  						To(BeNumerically(">", initialAOIncrementalMetadata[aoTableFQN].Modcount))
    92  					Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount).
    93  						To(BeNumerically(">", initialAOIncrementalMetadata[aoCOTableFQN].Modcount))
    94  				})
    95  				It("should NOT increase modcount for parent partition tables", func() {
    96  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount).
    97  						To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].Modcount))
    98  				})
    99  				It("should increase modcount for modified child partition tables", func() {
   100  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount).
   101  						To(BeNumerically(">", initialAOIncrementalMetadata[aoPartChildTableFQN].Modcount))
   102  				})
   103  				It("should have a last DDL timestamp", func() {
   104  					Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   105  					Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   106  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   107  
   108  					// For GPDB 7+, the root partition is not included.
   109  					// TODO: Should the root be included?
   110  					if false {
   111  						Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   112  					}
   113  				})
   114  			})
   115  			Context("After a delete operation", func() {
   116  				var initialAOIncrementalMetadata map[string]toc.AOEntry
   117  				var aoIncrementalMetadata map[string]toc.AOEntry
   118  				BeforeEach(func() {
   119  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN))
   120  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN))
   121  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN))
   122  
   123  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   124  
   125  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoTableFQN))
   126  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoCOTableFQN))
   127  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoPartParentTableFQN))
   128  
   129  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   130  				})
   131  				It("should increase modcount for non partition tables", func() {
   132  					Expect(aoIncrementalMetadata[aoTableFQN].Modcount).
   133  						To(BeNumerically(">", initialAOIncrementalMetadata[aoTableFQN].Modcount))
   134  					Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount).
   135  						To(BeNumerically(">", initialAOIncrementalMetadata[aoCOTableFQN].Modcount))
   136  				})
   137  				It("should NOT increase modcount for parent partition tables", func() {
   138  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount).
   139  						To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].Modcount))
   140  				})
   141  				It("should increase modcount for modified child partition tables", func() {
   142  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount).
   143  						To(BeNumerically(">", initialAOIncrementalMetadata[aoPartChildTableFQN].Modcount))
   144  				})
   145  				It("should have a last DDL timestamp", func() {
   146  					Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   147  					Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   148  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   149  
   150  					// For GPDB 7+, the root partition is not included.
   151  					// TODO: Should the root be included?
   152  					if false {
   153  						Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty()))
   154  					}
   155  				})
   156  			})
   157  		})
   158  		Context("AO, AO_CO and AO partition tables have DDL changes", func() {
   159  			var initialAOIncrementalMetadata map[string]toc.AOEntry
   160  			var aoIncrementalMetadata map[string]toc.AOEntry
   161  			Context("After a column add", func() {
   162  				BeforeEach(func() {
   163  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   164  
   165  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN))
   166  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN))
   167  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN))
   168  
   169  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   170  				})
   171  				It("should NOT care about modcount", func() {})
   172  				It("should have a changed last DDL timestamp", func() {
   173  					Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).
   174  						To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp)))
   175  					Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).
   176  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   177  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).
   178  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   179  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).
   180  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   181  				})
   182  			})
   183  			Context("After a column drop", func() {
   184  				BeforeEach(func() {
   185  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN))
   186  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN))
   187  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN))
   188  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   189  
   190  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoTableFQN))
   191  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoCOTableFQN))
   192  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoPartParentTableFQN))
   193  
   194  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   195  				})
   196  				It("should NOT care about modcount", func() {})
   197  				It("should have a changed last DDL timestamp", func() {
   198  					Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).
   199  						To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp)))
   200  					Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).
   201  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   202  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).
   203  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   204  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).
   205  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   206  				})
   207  			})
   208  			Context("After truncating a child partition", func() {
   209  				BeforeEach(func() {
   210  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   211  
   212  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf("TRUNCATE TABLE %s", aoPartChildTableFQN))
   213  
   214  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   215  				})
   216  				It("should NOT care about modcount", func() {})
   217  				It("should have a changed last DDL timestamp for the child", func() {
   218  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).
   219  						To(Not(Equal(initialAOIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp)))
   220  				})
   221  				It("should NOT have a changed last DDL timestamp for the parent", func() {
   222  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).
   223  						To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp))
   224  				})
   225  			})
   226  		})
   227  		Context("AO, AO_CO and AO partition tables have DML and DDL changes", func() {
   228  			var initialAOIncrementalMetadata map[string]toc.AOEntry
   229  			var aoIncrementalMetadata map[string]toc.AOEntry
   230  			Context("After an insert followed by an ALTER table", func() {
   231  				BeforeEach(func() {
   232  					initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   233  
   234  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN))
   235  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN))
   236  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN))
   237  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN))
   238  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN))
   239  					testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN))
   240  
   241  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   242  				})
   243  				It("should NOT care about modcount", func() {
   244  					//We don't care about modcount because DDL operations can reset the modcount value
   245  				})
   246  				It("should have a changed last DDL timestamp", func() {
   247  					Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).
   248  						To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp)))
   249  					Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).
   250  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   251  					Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).
   252  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   253  					Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).
   254  						To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp)))
   255  				})
   256  			})
   257  		})
   258  		Context("Filtered backup", func() {
   259  			var aoIncrementalMetadata map[string]toc.AOEntry
   260  			Context("During a table-filtered backup", func() {
   261  				It("only retrieves ao metadata for specific tables", func() {
   262  					_ = backupCmdFlags.Set(options.INCLUDE_RELATION, aoTableFQN)
   263  
   264  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   265  					Expect(aoIncrementalMetadata).To(HaveLen(1))
   266  				})
   267  			})
   268  			Context("During a schema-filtered backup", func() {
   269  				It("only retrieves ao metadata for tables in a specific schema", func() {
   270  					testhelper.AssertQueryRuns(connectionPool, "CREATE SCHEMA testschema")
   271  					defer testhelper.AssertQueryRuns(connectionPool, "DROP SCHEMA testschema CASCADE")
   272  					testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE testschema.ao_foo (i int) WITH (appendonly=true)")
   273  
   274  					_ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema")
   275  
   276  					aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool)
   277  					Expect(aoIncrementalMetadata).To(HaveLen(1))
   278  				})
   279  			})
   280  		})
   281  	})
   282  })