github.com/tuhaihe/gpbackup@v1.0.3/integration/incremental_queries_test.go (about) 1 package integration 2 3 import ( 4 "fmt" 5 6 "github.com/tuhaihe/gp-common-go-libs/testhelper" 7 "github.com/tuhaihe/gpbackup/backup" 8 "github.com/tuhaihe/gpbackup/options" 9 "github.com/tuhaihe/gpbackup/toc" 10 11 . "github.com/onsi/ginkgo/v2" 12 . "github.com/onsi/gomega" 13 ) 14 15 var _ = Describe("backup integration tests", func() { 16 const ( 17 dropTableSQL = "DROP TABLE %s" 18 addColumnSQL = "ALTER TABLE %s ADD COLUMN k int DEFAULT 0" 19 dropColumnSQL = "ALTER TABLE %s DROP COLUMN k" 20 insertSQL = "INSERT INTO %s values(10)" 21 deleteSQL = "DELETE FROM %s" 22 ) 23 24 var aoTableFQN = "public.ao_foo" 25 var aoCOTableFQN = "public.ao_co_foo" 26 var aoPartParentTableFQN = "public.ao_part" 27 var aoPartChildTableFQN = "public.ao_part_1_prt_child" 28 BeforeEach(func() { 29 testhelper.AssertQueryRuns(connectionPool, 30 fmt.Sprintf("CREATE TABLE %s (i int) WITH (appendonly=true)", aoTableFQN)) 31 testhelper.AssertQueryRuns(connectionPool, 32 fmt.Sprintf("CREATE TABLE %s (i int) WITH (appendonly=true,orientation='column')", aoCOTableFQN)) 33 testhelper.AssertQueryRuns(connectionPool, 34 fmt.Sprintf(`CREATE TABLE %s (i int) WITH (appendonly=true) 35 DISTRIBUTED BY (i) 36 PARTITION BY LIST (i) 37 ( 38 PARTITION child VALUES (10) 39 );`, aoPartParentTableFQN)) 40 }) 41 AfterEach(func() { 42 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoTableFQN)) 43 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoCOTableFQN)) 44 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropTableSQL, aoPartParentTableFQN)) 45 }) 46 Describe("GetAOIncrementalMetadata", func() { 47 Context("AO, AO_CO and AO partition tables are only just created", func() { 48 var aoIncrementalMetadata map[string]toc.AOEntry 49 BeforeEach(func() { 50 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 51 }) 52 It("should have a modcount of 0", func() { 53 Expect(aoIncrementalMetadata[aoTableFQN].Modcount).To(Equal(int64(0))) 54 Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount).To(Equal(int64(0))) 55 Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount).To(Equal(int64(0))) 56 Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount).To(Equal(int64(0))) 57 }) 58 It("should have a last DDL timestamp", func() { 59 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 60 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 61 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 62 63 // For GPDB 7+, the root partition is not included. 64 // TODO: Should the root be included? 65 if false { 66 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 67 } 68 69 }) 70 }) 71 Context("AO, AO_CO and AO partition tables have DML changes", func() { 72 Context("After an insert(s)", func() { 73 var initialAOIncrementalMetadata map[string]toc.AOEntry 74 var aoIncrementalMetadata map[string]toc.AOEntry 75 BeforeEach(func() { 76 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 77 78 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN)) 79 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN)) 80 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN)) 81 82 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 83 }) 84 AfterEach(func() { 85 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoTableFQN)) 86 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoCOTableFQN)) 87 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoPartParentTableFQN)) 88 }) 89 It("should increase modcount for non partition tables", func() { 90 Expect(aoIncrementalMetadata[aoTableFQN].Modcount). 91 To(BeNumerically(">", initialAOIncrementalMetadata[aoTableFQN].Modcount)) 92 Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount). 93 To(BeNumerically(">", initialAOIncrementalMetadata[aoCOTableFQN].Modcount)) 94 }) 95 It("should NOT increase modcount for parent partition tables", func() { 96 Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount). 97 To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].Modcount)) 98 }) 99 It("should increase modcount for modified child partition tables", func() { 100 Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount). 101 To(BeNumerically(">", initialAOIncrementalMetadata[aoPartChildTableFQN].Modcount)) 102 }) 103 It("should have a last DDL timestamp", func() { 104 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 105 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 106 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 107 108 // For GPDB 7+, the root partition is not included. 109 // TODO: Should the root be included? 110 if false { 111 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 112 } 113 }) 114 }) 115 Context("After a delete operation", func() { 116 var initialAOIncrementalMetadata map[string]toc.AOEntry 117 var aoIncrementalMetadata map[string]toc.AOEntry 118 BeforeEach(func() { 119 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN)) 120 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN)) 121 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN)) 122 123 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 124 125 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoTableFQN)) 126 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoCOTableFQN)) 127 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(deleteSQL, aoPartParentTableFQN)) 128 129 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 130 }) 131 It("should increase modcount for non partition tables", func() { 132 Expect(aoIncrementalMetadata[aoTableFQN].Modcount). 133 To(BeNumerically(">", initialAOIncrementalMetadata[aoTableFQN].Modcount)) 134 Expect(aoIncrementalMetadata[aoCOTableFQN].Modcount). 135 To(BeNumerically(">", initialAOIncrementalMetadata[aoCOTableFQN].Modcount)) 136 }) 137 It("should NOT increase modcount for parent partition tables", func() { 138 Expect(aoIncrementalMetadata[aoPartParentTableFQN].Modcount). 139 To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].Modcount)) 140 }) 141 It("should increase modcount for modified child partition tables", func() { 142 Expect(aoIncrementalMetadata[aoPartChildTableFQN].Modcount). 143 To(BeNumerically(">", initialAOIncrementalMetadata[aoPartChildTableFQN].Modcount)) 144 }) 145 It("should have a last DDL timestamp", func() { 146 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 147 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 148 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 149 150 // For GPDB 7+, the root partition is not included. 151 // TODO: Should the root be included? 152 if false { 153 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp).To(Not(BeEmpty())) 154 } 155 }) 156 }) 157 }) 158 Context("AO, AO_CO and AO partition tables have DDL changes", func() { 159 var initialAOIncrementalMetadata map[string]toc.AOEntry 160 var aoIncrementalMetadata map[string]toc.AOEntry 161 Context("After a column add", func() { 162 BeforeEach(func() { 163 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 164 165 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN)) 166 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN)) 167 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN)) 168 169 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 170 }) 171 It("should NOT care about modcount", func() {}) 172 It("should have a changed last DDL timestamp", func() { 173 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp). 174 To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp))) 175 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp). 176 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 177 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp). 178 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 179 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp). 180 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 181 }) 182 }) 183 Context("After a column drop", func() { 184 BeforeEach(func() { 185 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN)) 186 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN)) 187 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN)) 188 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 189 190 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoTableFQN)) 191 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoCOTableFQN)) 192 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(dropColumnSQL, aoPartParentTableFQN)) 193 194 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 195 }) 196 It("should NOT care about modcount", func() {}) 197 It("should have a changed last DDL timestamp", func() { 198 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp). 199 To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp))) 200 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp). 201 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 202 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp). 203 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 204 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp). 205 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 206 }) 207 }) 208 Context("After truncating a child partition", func() { 209 BeforeEach(func() { 210 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 211 212 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf("TRUNCATE TABLE %s", aoPartChildTableFQN)) 213 214 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 215 }) 216 It("should NOT care about modcount", func() {}) 217 It("should have a changed last DDL timestamp for the child", func() { 218 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp). 219 To(Not(Equal(initialAOIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp))) 220 }) 221 It("should NOT have a changed last DDL timestamp for the parent", func() { 222 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp). 223 To(Equal(initialAOIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp)) 224 }) 225 }) 226 }) 227 Context("AO, AO_CO and AO partition tables have DML and DDL changes", func() { 228 var initialAOIncrementalMetadata map[string]toc.AOEntry 229 var aoIncrementalMetadata map[string]toc.AOEntry 230 Context("After an insert followed by an ALTER table", func() { 231 BeforeEach(func() { 232 initialAOIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 233 234 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoTableFQN)) 235 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoTableFQN)) 236 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoCOTableFQN)) 237 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoCOTableFQN)) 238 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(insertSQL, aoPartParentTableFQN)) 239 testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf(addColumnSQL, aoPartParentTableFQN)) 240 241 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 242 }) 243 It("should NOT care about modcount", func() { 244 //We don't care about modcount because DDL operations can reset the modcount value 245 }) 246 It("should have a changed last DDL timestamp", func() { 247 Expect(aoIncrementalMetadata[aoTableFQN].LastDDLTimestamp). 248 To(Not(Equal(initialAOIncrementalMetadata[aoTableFQN].LastDDLTimestamp))) 249 Expect(aoIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp). 250 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 251 Expect(aoIncrementalMetadata[aoPartParentTableFQN].LastDDLTimestamp). 252 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 253 Expect(aoIncrementalMetadata[aoPartChildTableFQN].LastDDLTimestamp). 254 To(Not(Equal(initialAOIncrementalMetadata[aoCOTableFQN].LastDDLTimestamp))) 255 }) 256 }) 257 }) 258 Context("Filtered backup", func() { 259 var aoIncrementalMetadata map[string]toc.AOEntry 260 Context("During a table-filtered backup", func() { 261 It("only retrieves ao metadata for specific tables", func() { 262 _ = backupCmdFlags.Set(options.INCLUDE_RELATION, aoTableFQN) 263 264 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 265 Expect(aoIncrementalMetadata).To(HaveLen(1)) 266 }) 267 }) 268 Context("During a schema-filtered backup", func() { 269 It("only retrieves ao metadata for tables in a specific schema", func() { 270 testhelper.AssertQueryRuns(connectionPool, "CREATE SCHEMA testschema") 271 defer testhelper.AssertQueryRuns(connectionPool, "DROP SCHEMA testschema CASCADE") 272 testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE testschema.ao_foo (i int) WITH (appendonly=true)") 273 274 _ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema") 275 276 aoIncrementalMetadata = backup.GetAOIncrementalMetadata(connectionPool) 277 Expect(aoIncrementalMetadata).To(HaveLen(1)) 278 }) 279 }) 280 }) 281 }) 282 })