github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/integration/options_integration_test.go (about) 1 package integration 2 3 import ( 4 "sort" 5 6 "github.com/cloudberrydb/gp-common-go-libs/testhelper" 7 "github.com/cloudberrydb/gpbackup/backup" 8 "github.com/cloudberrydb/gpbackup/options" 9 10 . "github.com/onsi/ginkgo/v2" 11 . "github.com/onsi/gomega" 12 ) 13 14 var _ = Describe("Options Integration", func() { 15 Describe("QuoteTablesNames", func() { 16 It("quotes identifiers as expected", func() { 17 tableList := []string{ 18 `foo.bar`, // no special characters 19 `foo.BAR`, // capital characters 20 `foo.'bar`, // make sure that single quotes are escaped before string is fed to quote_ident 21 `foo.2`, // numbers 22 `foo._bar`, // underscore 23 `foo ~#$%^&*()_-+[]{}><\|;:/?!,.bar`, 24 "foo.\tbar", // important to use double quotes to allow \t to become tab 25 "foo.\nbar", // important to use double quotes to allow \n to become a new line 26 `foo.\n`, 27 `foo."bar`, // quote ident should escape double-quote with another double-quote 28 } 29 expected := []string{ 30 `foo.bar`, 31 `foo."BAR"`, 32 `foo."'bar"`, 33 `foo."2"`, 34 `foo._bar`, // underscore is not a special character 35 `"foo ~#$%^&*()_-+[]{}><\|;:/?!,".bar`, 36 "foo.\"\tbar\"", 37 "foo.\"\nbar\"", 38 `foo."\n"`, 39 `foo."""bar"`, 40 } 41 42 resultFQNs, err := options.QuoteTableNames(connectionPool, tableList) 43 Expect(err).ToNot(HaveOccurred()) 44 Expect(expected).To(Equal(resultFQNs)) 45 }) 46 }) 47 Describe("ValidateFilterTables", func() { 48 It("validates special chars", func() { 49 createSpecialCharacterTables := ` 50 -- special chars 51 CREATE TABLE public."FOObar" (i int); 52 CREATE TABLE public."BAR" (i int); 53 CREATE SCHEMA "CAPschema"; 54 CREATE TABLE "CAPschema"."BAR" (i int); 55 CREATE TABLE "CAPschema".baz (i int); 56 CREATE TABLE public.foo_bar (i int); 57 CREATE TABLE public."foo ~#$%^&*()_-+[]{}><\|;:/?!bar" (i int); 58 -- special chars: embedded tab char 59 CREATE TABLE public."tab bar" (i int); 60 -- special chars: embedded newline char 61 CREATE TABLE public."newline 62 bar" (i int); 63 ` 64 dropSpecialCharacterTables := ` 65 -- special chars 66 DROP TABLE public."FOObar"; 67 DROP TABLE public."BAR"; 68 DROP SCHEMA "CAPschema" cascade; 69 DROP TABLE public.foo_bar; 70 DROP TABLE public."foo ~#$%^&*()_-+[]{}><\|;:/?!bar"; 71 -- special chars: embedded tab char 72 DROP TABLE public."tab bar"; 73 -- special chars: embedded newline char 74 DROP TABLE public."newline 75 bar"; 76 ` 77 testhelper.AssertQueryRuns(connectionPool, createSpecialCharacterTables) 78 defer testhelper.AssertQueryRuns(connectionPool, dropSpecialCharacterTables) 79 80 tableList := []string{ 81 `public.BAR`, 82 `CAPschema.BAR`, 83 `CAPschema.baz`, 84 `public.foo_bar`, 85 `public.foo ~#$%^&*()_-+[]{}><\|;:/?!bar`, 86 "public.tab\tbar", // important to use double quotes to allow \t to become tab 87 "public.newline\nbar", // important to use double quotes to allow \n to become newline 88 } 89 90 backup.ValidateTablesExist(connectionPool, tableList, false) 91 }) 92 }) 93 Describe("ExpandIncludesForPartitions", func() { 94 BeforeEach(func() { 95 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public."CAPpart" 96 (id int, rank int, year int, gender char(1), count int ) 97 DISTRIBUTED BY (id) 98 PARTITION BY LIST (gender) 99 ( PARTITION girls VALUES ('F'), 100 PARTITION boys VALUES ('M'), 101 DEFAULT PARTITION other ) 102 `) 103 }) 104 105 AfterEach(func() { 106 testhelper.AssertQueryRuns(connectionPool, `DROP TABLE public."CAPpart"`) 107 }) 108 109 It("adds parent table when child partition with special chars is included", func() { 110 err := backupCmdFlags.Set(options.INCLUDE_RELATION, `public.CAPpart_1_prt_girls`) 111 Expect(err).ToNot(HaveOccurred()) 112 subject, err := options.NewOptions(backupCmdFlags) 113 Expect(err).To(Not(HaveOccurred())) 114 Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart_1_prt_girls")) 115 Expect(subject.GetIncludedTables()).To(HaveLen(1)) 116 117 err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags) 118 Expect(err).To(Not(HaveOccurred())) 119 Expect(subject.GetIncludedTables()).To(HaveLen(2)) 120 Expect(backupCmdFlags.GetStringArray(options.INCLUDE_RELATION)).To(HaveLen(2)) 121 Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart_1_prt_girls")) 122 Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart")) 123 124 // ensure that ExpandIncludesForPartitions does not disturb the original value 125 // that the user typed in, which is used by initializeBackupReport() and 126 // is important for incremental backups which must exactly match all flag input 127 Expect(subject.GetOriginalIncludedTables()).To(Equal([]string{`public.CAPpart_1_prt_girls`})) 128 }) 129 It("adds parent table when child partition with embedded quote character is included", func() { 130 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public."""hasquote""" 131 (id int, rank int, year int, gender char(1), count int ) 132 DISTRIBUTED BY (id) 133 PARTITION BY LIST (gender) 134 ( PARTITION girls VALUES ('F'), 135 PARTITION boys VALUES ('M'), 136 DEFAULT PARTITION other ) 137 `) 138 defer testhelper.AssertQueryRuns(connectionPool, `DROP TABLE public."""hasquote"""`) 139 140 err := backupCmdFlags.Set(options.INCLUDE_RELATION, `public."hasquote"_1_prt_girls`) 141 Expect(err).ToNot(HaveOccurred()) 142 subject, err := options.NewOptions(backupCmdFlags) 143 Expect(err).To(Not(HaveOccurred())) 144 Expect(subject.GetIncludedTables()).To(ContainElement(`public."hasquote"_1_prt_girls`)) 145 Expect(subject.GetIncludedTables()).To(HaveLen(1)) 146 147 err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags) 148 Expect(err).To(Not(HaveOccurred())) 149 Expect(subject.GetIncludedTables()).To(HaveLen(2)) 150 Expect(backupCmdFlags.GetStringArray(options.INCLUDE_RELATION)).To(HaveLen(2)) 151 Expect(subject.GetIncludedTables()[0]).To(Equal(`public."hasquote"_1_prt_girls`)) 152 Expect(subject.GetIncludedTables()[1]).To(Equal(`public."hasquote"`)) 153 }) 154 It("returns child partition tables for an included parent table if the leaf-partition-data flag is set and the filter includes a parent partition table", func() { 155 _ = backupCmdFlags.Set(options.LEAF_PARTITION_DATA, "true") 156 _ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.rank") 157 158 createStmt := `CREATE TABLE public.rank (id int, rank int, year int, gender 159 char(1), count int ) 160 DISTRIBUTED BY (id) 161 PARTITION BY LIST (gender) 162 ( PARTITION girls VALUES ('F'), 163 PARTITION boys VALUES ('M'), 164 DEFAULT PARTITION other );` 165 testhelper.AssertQueryRuns(connectionPool, createStmt) 166 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.rank") 167 testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.test_table(i int)") 168 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.test_table") 169 170 subject, err := options.NewOptions(backupCmdFlags) 171 Expect(err).To(Not(HaveOccurred())) 172 173 err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags) 174 Expect(err).To(Not(HaveOccurred())) 175 176 expectedTableNames := []string{ 177 "public.rank", 178 "public.rank_1_prt_boys", 179 "public.rank_1_prt_girls", 180 "public.rank_1_prt_other", 181 } 182 183 tables := subject.GetIncludedTables() 184 sort.Strings(tables) 185 Expect(tables).To(HaveLen(4)) 186 Expect(tables).To(Equal(expectedTableNames)) 187 }) 188 It("returns parent and external leaf partition table if the filter includes a leaf table and leaf-partition-data is set", func() { 189 _ = backupCmdFlags.Set(options.LEAF_PARTITION_DATA, "true") 190 _ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table_1_prt_boys") 191 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table (id int, gender char(1)) 192 DISTRIBUTED BY (id) 193 PARTITION BY LIST (gender) 194 ( PARTITION girls VALUES ('F'), 195 PARTITION boys VALUES ('M'), 196 DEFAULT PARTITION other );`) 197 testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table_ext_part_ (like public.partition_table_1_prt_girls) 198 EXECUTE 'echo -e "2\n1"' on host 199 FORMAT 'csv';`) 200 testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table EXCHANGE PARTITION girls WITH TABLE public.partition_table_ext_part_ WITHOUT VALIDATION;`) 201 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table") 202 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table_ext_part_") 203 204 subject, err := options.NewOptions(backupCmdFlags) 205 Expect(err).To(Not(HaveOccurred())) 206 207 err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags) 208 Expect(err).To(Not(HaveOccurred())) 209 210 expectedTableNames := []string{ 211 "public.partition_table", 212 "public.partition_table_1_prt_boys", 213 } 214 if false { 215 // For GPDB 4, 5, and 6, the leaf partition metadata is a part of the large root partition DDL. 216 // For GPDB 7+, the leaf partition has its own separate DDL and attaches onto the root partition 217 // with ALTER TABLE ATTACH PARTITION. Therefore, only the included leaf partition and its root 218 // partition will have their metadata dumped. 219 expectedTableNames = append(expectedTableNames, "public.partition_table_1_prt_girls") 220 } 221 222 tables := subject.GetIncludedTables() 223 sort.Strings(tables) 224 Expect(tables).To(HaveLen(len(expectedTableNames))) 225 Expect(tables).To(Equal(expectedTableNames)) 226 }) 227 It("returns external partition tables for an included parent table if the filter includes a parent partition table", func() { 228 _ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table1") 229 _ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table2_1_prt_other") 230 231 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table1 (id int, gender char(1)) 232 DISTRIBUTED BY (id) 233 PARTITION BY LIST (gender) 234 ( PARTITION girls VALUES ('F'), 235 PARTITION boys VALUES ('M'), 236 DEFAULT PARTITION other );`) 237 testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table1_ext_part_ (like public.partition_table1_1_prt_boys) 238 EXECUTE 'echo -e "2\n1"' on host 239 FORMAT 'csv';`) 240 testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table1 EXCHANGE PARTITION boys WITH TABLE public.partition_table1_ext_part_ WITHOUT VALIDATION;`) 241 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table1") 242 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table1_ext_part_") 243 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table2 (id int, gender char(1)) 244 DISTRIBUTED BY (id) 245 PARTITION BY LIST (gender) 246 ( PARTITION girls VALUES ('F'), 247 PARTITION boys VALUES ('M'), 248 DEFAULT PARTITION other );`) 249 testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table2_ext_part_ (like public.partition_table2_1_prt_girls) 250 EXECUTE 'echo -e "2\n1"' on host 251 FORMAT 'csv';`) 252 testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table2 EXCHANGE PARTITION girls WITH TABLE public.partition_table2_ext_part_ WITHOUT VALIDATION;`) 253 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table2") 254 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table2_ext_part_") 255 testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table3 (id int, gender char(1)) 256 DISTRIBUTED BY (id) 257 PARTITION BY LIST (gender) 258 ( PARTITION girls VALUES ('F'), 259 PARTITION boys VALUES ('M'), 260 DEFAULT PARTITION other );`) 261 testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table3_ext_part_ (like public.partition_table3_1_prt_girls) 262 EXECUTE 'echo -e "2\n1"' on host 263 FORMAT 'csv';`) 264 testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table3 EXCHANGE PARTITION girls WITH TABLE public.partition_table3_ext_part_ WITHOUT VALIDATION;`) 265 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table3") 266 defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table3_ext_part_") 267 268 subject, err := options.NewOptions(backupCmdFlags) 269 Expect(err).To(Not(HaveOccurred())) 270 271 err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags) 272 Expect(err).To(Not(HaveOccurred())) 273 274 var expectedTableNames []string 275 if false { 276 expectedTableNames = []string{ 277 "public.partition_table1", 278 "public.partition_table1_1_prt_boys", 279 "public.partition_table2", 280 "public.partition_table2_1_prt_girls", 281 "public.partition_table2_1_prt_other", 282 } 283 } else { 284 // For GPDB 7+, each leaf partition of the included partition_table1 will have their own 285 // separate DDL along with an ATTACH PARTITION. Since only partition_table2_1_prt_other 286 // is included, only it and its root partition will be a part of the list. 287 expectedTableNames = []string{ 288 "public.partition_table1", 289 "public.partition_table1_1_prt_boys", 290 "public.partition_table1_1_prt_girls", 291 "public.partition_table1_1_prt_other", 292 "public.partition_table2", 293 "public.partition_table2_1_prt_other", 294 } 295 } 296 297 tables := subject.GetIncludedTables() 298 sort.Strings(tables) 299 Expect(tables).To(HaveLen(len(expectedTableNames))) 300 Expect(tables).To(Equal(expectedTableNames)) 301 }) 302 }) 303 })