github.com/greenplum-db/gpbackup@v0.0.0-20240517212602-89daab1885b3/end_to_end/filtered_test.go (about) 1 package end_to_end_test 2 3 import ( 4 "os" 5 6 "github.com/greenplum-db/gp-common-go-libs/iohelper" 7 "github.com/greenplum-db/gp-common-go-libs/testhelper" 8 "github.com/greenplum-db/gpbackup/utils" 9 . "github.com/onsi/ginkgo/v2" 10 . "github.com/onsi/gomega" 11 ) 12 13 var _ = Describe("End to End Filtered tests", func() { 14 BeforeEach(func() { 15 end_to_end_setup() 16 }) 17 AfterEach(func() { 18 end_to_end_teardown() 19 }) 20 21 Describe("Backup include filtering", func() { 22 It("runs gpbackup and gprestore with include-schema backup flag and compression level", func() { 23 output := gpbackup(gpbackupPath, backupHelperPath, 24 "--include-schema", "public", 25 "--compression-level", "2") 26 timestamp := getBackupTimestamp(string(output)) 27 28 gprestore(gprestorePath, restoreHelperPath, timestamp, 29 "--redirect-db", "restoredb") 30 31 assertRelationsCreated(restoreConn, 20) 32 assertDataRestored(restoreConn, publicSchemaTupleCounts) 33 assertArtifactsCleaned(timestamp) 34 }) 35 It("runs gpbackup and gprestore with include-table backup flag", func() { 36 skipIfOldBackupVersionBefore("1.4.0") 37 output := gpbackup(gpbackupPath, backupHelperPath, 38 "--include-table", "public.foo", 39 "--include-table", "public.sales", 40 "--include-table", "public.myseq1", 41 "--include-table", "public.myview1") 42 43 timestamp := getBackupTimestamp(string(output)) 44 45 gprestore(gprestorePath, restoreHelperPath, timestamp, 46 "--redirect-db", "restoredb") 47 48 assertRelationsCreated(restoreConn, 16) 49 assertDataRestored(restoreConn, map[string]int{"public.foo": 40000}) 50 }) 51 It("runs gpbackup and gprestore with include-table-file backup flag", func() { 52 skipIfOldBackupVersionBefore("1.4.0") 53 includeFile := iohelper.MustOpenFileForWriting("/tmp/include-tables.txt") 54 utils.MustPrintln(includeFile, "public.sales\npublic.foo\npublic.myseq1\npublic.myview1") 55 output := gpbackup(gpbackupPath, backupHelperPath, 56 "--include-table-file", "/tmp/include-tables.txt") 57 timestamp := getBackupTimestamp(string(output)) 58 59 gprestore(gprestorePath, restoreHelperPath, timestamp, 60 "--redirect-db", "restoredb") 61 62 assertRelationsCreated(restoreConn, 16) 63 assertDataRestored(restoreConn, map[string]int{"public.sales": 13, "public.foo": 40000}) 64 65 _ = os.Remove("/tmp/include-tables.txt") 66 }) 67 It("runs gpbackup and gprestore with include-schema-file backup flag", func() { 68 skipIfOldBackupVersionBefore("1.17.0") 69 includeFile := iohelper.MustOpenFileForWriting("/tmp/include-schema.txt") 70 utils.MustPrintln(includeFile, "public") 71 output := gpbackup(gpbackupPath, backupHelperPath, 72 "--include-schema-file", "/tmp/include-schema.txt") 73 timestamp := getBackupTimestamp(string(output)) 74 75 gprestore(gprestorePath, restoreHelperPath, timestamp, 76 "--redirect-db", "restoredb") 77 78 assertRelationsCreated(restoreConn, 20) 79 80 _ = os.Remove("/tmp/include-schema.txt") 81 }) 82 It("runs gpbackup with --include-table flag with partitions (non-special chars)", func() { 83 testhelper.AssertQueryRuns(backupConn, 84 `CREATE TABLE public.testparent (id int, rank int, year int, gender 85 char(1), count int ) 86 DISTRIBUTED BY (id) 87 PARTITION BY LIST (gender) 88 ( PARTITION girls VALUES ('F'), 89 PARTITION boys VALUES ('M'), 90 DEFAULT PARTITION other ); 91 `) 92 defer testhelper.AssertQueryRuns(backupConn, 93 `DROP TABLE public.testparent`) 94 95 testhelper.AssertQueryRuns(backupConn, 96 `insert into public.testparent values (1,1,1,'M',1)`) 97 testhelper.AssertQueryRuns(backupConn, 98 `insert into public.testparent values (0,0,0,'F',1)`) 99 100 output := gpbackup(gpbackupPath, backupHelperPath, 101 "--backup-dir", backupDir, 102 "--include-table", `public.testparent_1_prt_girls`, 103 "--leaf-partition-data") 104 timestamp := getBackupTimestamp(string(output)) 105 106 gprestore(gprestorePath, restoreHelperPath, timestamp, 107 "--redirect-db", "restoredb", 108 "--backup-dir", backupDir) 109 110 // When running against GPDB 7+, only the root partition and the included leaf partition 111 // will be created due to the new flexible GPDB 7+ partitioning logic. For versions 112 // before GPDB 7, there is only one big DDL for the entire partition table. 113 if backupConn.Version.AtLeast("7") { 114 assertRelationsCreated(restoreConn, 2) 115 } else { 116 assertRelationsCreated(restoreConn, 4) 117 } 118 119 localSchemaTupleCounts := map[string]int{ 120 `public.testparent_1_prt_girls`: 1, 121 `public.testparent`: 1, 122 } 123 assertDataRestored(restoreConn, localSchemaTupleCounts) 124 assertArtifactsCleaned(timestamp) 125 }) 126 It("backs up a child table inheriting from a parent when only the parent is included", func() { 127 skipIfOldBackupVersionBefore("1.29.0") // Inheritance behavior changed in this version 128 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.parent(a int);`) 129 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.parent`) 130 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.child() INHERITS (public.parent);`) 131 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.child`) 132 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.unrelated(c int);`) 133 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.unrelated`) 134 135 output := gpbackup(gpbackupPath, backupHelperPath, 136 "--backup-dir", backupDir, 137 "--include-table", `public.parent`, 138 "--metadata-only") 139 timestamp := getBackupTimestamp(string(output)) 140 141 gprestore(gprestorePath, restoreHelperPath, timestamp, 142 "--redirect-db", "restoredb", 143 "--backup-dir", backupDir) 144 145 assertTablesRestored(restoreConn, []string{"public.parent", "public.child"}) 146 assertTablesNotRestored(restoreConn, []string{"public.unrelated"}) 147 assertArtifactsCleaned(timestamp) 148 }) 149 It("backs up a table inheriting from multiple parents when only one parent is included", func() { 150 skipIfOldBackupVersionBefore("1.29.0") // Inheritance behavior changed in this version 151 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.parent_a(a int);`) 152 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.parent_a`) 153 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.parent_b(b int);`) 154 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.parent_b`) 155 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.child() INHERITS (public.parent_a, public.parent_b);`) 156 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.child`) 157 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public.unrelated(c int);`) 158 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public.unrelated`) 159 160 output := gpbackup(gpbackupPath, backupHelperPath, 161 "--backup-dir", backupDir, 162 "--include-table", `public.parent_a`, 163 "--metadata-only") 164 timestamp := getBackupTimestamp(string(output)) 165 166 gprestore(gprestorePath, restoreHelperPath, timestamp, 167 "--redirect-db", "restoredb", 168 "--backup-dir", backupDir) 169 170 assertTablesRestored(restoreConn, []string{"public.parent_a", "public.parent_b", "public.child"}) 171 assertTablesNotRestored(restoreConn, []string{"public.unrelated"}) 172 assertArtifactsCleaned(timestamp) 173 }) 174 It("gpbackup with --include-table does not backup protocols and functions", func() { 175 testhelper.AssertQueryRuns(backupConn, 176 `CREATE TABLE t1(i int)`) 177 defer testhelper.AssertQueryRuns(backupConn, 178 `DROP TABLE public.t1`) 179 testhelper.AssertQueryRuns(backupConn, 180 `CREATE FUNCTION f1() RETURNS int AS 'SELECT 1' LANGUAGE SQL;`) 181 defer testhelper.AssertQueryRuns(backupConn, 182 `DROP FUNCTION public.f1()`) 183 testhelper.AssertQueryRuns(backupConn, 184 `CREATE PROTOCOL p1 (readfunc = f1);`) 185 defer testhelper.AssertQueryRuns(backupConn, 186 `DROP PROTOCOL p1`) 187 188 output := gpbackup(gpbackupPath, backupHelperPath, 189 "--backup-dir", backupDir, "--include-table", "public.t1") 190 timestamp := getBackupTimestamp(string(output)) 191 192 metadataFileContents := getMetdataFileContents(backupDir, timestamp, "metadata.sql") 193 Expect(string(metadataFileContents)).To(ContainSubstring("t1")) 194 Expect(string(metadataFileContents)).ToNot(ContainSubstring("public.f1()")) 195 Expect(string(metadataFileContents)).ToNot(ContainSubstring("read_from_s3")) 196 Expect(string(metadataFileContents)).ToNot(ContainSubstring("s3_protocol")) 197 }) 198 }) 199 Describe("Restore include filtering", func() { 200 It("runs gpbackup and gprestore with include-schema restore flag", func() { 201 output := gpbackup(gpbackupPath, backupHelperPath, 202 "--backup-dir", backupDir) 203 timestamp := getBackupTimestamp(string(output)) 204 205 gprestore(gprestorePath, restoreHelperPath, timestamp, 206 "--redirect-db", "restoredb", 207 "--backup-dir", backupDir, 208 "--include-schema", "schema2") 209 210 assertRelationsCreated(restoreConn, 17) 211 assertDataRestored(restoreConn, schema2TupleCounts) 212 }) 213 It("runs gpbackup and gprestore with include-schema-file restore flag", func() { 214 includeFile := iohelper.MustOpenFileForWriting("/tmp/include-schema.txt") 215 utils.MustPrintln(includeFile, "schema2") 216 utils.MustPrintln(includeFile, "public") 217 output := gpbackup(gpbackupPath, backupHelperPath, 218 "--backup-dir", backupDir) 219 timestamp := getBackupTimestamp(string(output)) 220 221 gprestore(gprestorePath, restoreHelperPath, timestamp, 222 "--redirect-db", "restoredb", 223 "--backup-dir", backupDir, 224 "--include-schema-file", "/tmp/include-schema.txt") 225 226 assertRelationsCreated(restoreConn, 37) 227 assertDataRestored(restoreConn, schema2TupleCounts) 228 }) 229 It("runs gpbackup and gprestore with include-table restore flag", func() { 230 output := gpbackup(gpbackupPath, backupHelperPath) 231 timestamp := getBackupTimestamp(string(output)) 232 233 gprestore(gprestorePath, restoreHelperPath, timestamp, 234 "--redirect-db", "restoredb", 235 "--include-table", "public.foo", 236 "--include-table", "public.sales", 237 "--include-table", "public.myseq1", 238 "--include-table", "public.myview1") 239 240 assertRelationsCreated(restoreConn, 16) 241 assertDataRestored(restoreConn, map[string]int{ 242 "public.sales": 13, "public.foo": 40000}) 243 }) 244 It("runs gpbackup and gprestore with include-table-file restore flag", func() { 245 includeFile := iohelper.MustOpenFileForWriting("/tmp/include-tables.txt") 246 utils.MustPrintln(includeFile, 247 "public.sales\npublic.foo\npublic.myseq1\npublic.myview1") 248 output := gpbackup(gpbackupPath, backupHelperPath, 249 "--backup-dir", backupDir) 250 timestamp := getBackupTimestamp(string(output)) 251 252 gprestore(gprestorePath, restoreHelperPath, timestamp, 253 "--redirect-db", "restoredb", 254 "--backup-dir", backupDir, 255 "--include-table-file", "/tmp/include-tables.txt") 256 257 assertRelationsCreated(restoreConn, 16) 258 assertDataRestored(restoreConn, map[string]int{ 259 "public.sales": 13, "public.foo": 40000}) 260 261 _ = os.Remove("/tmp/include-tables.txt") 262 }) 263 It("runs gpbackup and gprestore with include-table restore flag against a leaf partition", func() { 264 skipIfOldBackupVersionBefore("1.7.2") 265 output := gpbackup(gpbackupPath, backupHelperPath, 266 "--leaf-partition-data") 267 timestamp := getBackupTimestamp(string(output)) 268 269 gprestore(gprestorePath, restoreHelperPath, timestamp, 270 "--redirect-db", "restoredb", 271 "--include-table", "public.sales_1_prt_jan17") 272 273 assertRelationsCreated(restoreConn, 13) 274 assertDataRestored(restoreConn, map[string]int{ 275 "public.sales": 1, "public.sales_1_prt_jan17": 1}) 276 }) 277 It("runs gpbackup and gprestore with include-table restore flag which implicitly filters schema restore list", func() { 278 output := gpbackup(gpbackupPath, backupHelperPath, 279 "--backup-dir", backupDir) 280 timestamp := getBackupTimestamp(string(output)) 281 282 gprestore(gprestorePath, restoreHelperPath, timestamp, 283 "--redirect-db", "restoredb", 284 "--backup-dir", backupDir, 285 "--include-table", "schema2.foo3") 286 assertRelationsCreated(restoreConn, 1) 287 assertDataRestored(restoreConn, map[string]int{"schema2.foo3": 100}) 288 }) 289 It("runs gprestore with --include-table flag to only restore tables specified ", func() { 290 testhelper.AssertQueryRuns(backupConn, 291 "CREATE TABLE public.table_to_include_with_stats(i int)") 292 defer testhelper.AssertQueryRuns(backupConn, 293 "DROP TABLE public.table_to_include_with_stats") 294 testhelper.AssertQueryRuns(backupConn, 295 "INSERT INTO public.table_to_include_with_stats VALUES (1)") 296 output := gpbackup(gpbackupPath, backupHelperPath, 297 "--backup-dir", backupDir) 298 timestamp := getBackupTimestamp(string(output)) 299 300 gprestore(gprestorePath, restoreHelperPath, timestamp, 301 "--redirect-db", "restoredb", 302 "--backup-dir", backupDir, 303 "--include-table", "public.table_to_include_with_stats") 304 305 assertRelationsCreated(restoreConn, 1) 306 307 localSchemaTupleCounts := map[string]int{ 308 `public."table_to_include_with_stats"`: 1, 309 } 310 assertDataRestored(restoreConn, localSchemaTupleCounts) 311 assertArtifactsCleaned(timestamp) 312 }) 313 314 }) 315 Describe("Backup exclude filtering", func() { 316 It("runs gpbackup and gprestore with exclude-schema backup flag", func() { 317 output := gpbackup(gpbackupPath, backupHelperPath, 318 "--exclude-schema", "public") 319 timestamp := getBackupTimestamp(string(output)) 320 321 gprestore(gprestorePath, restoreHelperPath, timestamp, 322 "--redirect-db", "restoredb") 323 324 assertRelationsCreated(restoreConn, 17) 325 assertDataRestored(restoreConn, schema2TupleCounts) 326 }) 327 It("runs gpbackup and gprestore with exclude-schema-file backup flag", func() { 328 skipIfOldBackupVersionBefore("1.17.0") 329 excludeFile := iohelper.MustOpenFileForWriting("/tmp/exclude-schema.txt") 330 utils.MustPrintln(excludeFile, "public") 331 output := gpbackup(gpbackupPath, backupHelperPath, 332 "--exclude-schema-file", "/tmp/exclude-schema.txt") 333 timestamp := getBackupTimestamp(string(output)) 334 335 gprestore(gprestorePath, restoreHelperPath, timestamp, 336 "--redirect-db", "restoredb") 337 338 assertRelationsCreated(restoreConn, 17) 339 assertDataRestored(restoreConn, schema2TupleCounts) 340 341 _ = os.Remove("/tmp/exclude-schema.txt") 342 }) 343 It("runs gpbackup and gprestore with exclude-table backup flag", func() { 344 skipIfOldBackupVersionBefore("1.4.0") 345 output := gpbackup(gpbackupPath, backupHelperPath, 346 "--exclude-table", "schema2.foo2", 347 "--exclude-table", "schema2.returns", 348 "--exclude-table", "public.myseq2", 349 "--exclude-table", "public.myview2") 350 timestamp := getBackupTimestamp(string(output)) 351 352 gprestore(gprestorePath, restoreHelperPath, timestamp, 353 "--redirect-db", "restoredb") 354 355 assertRelationsCreated(restoreConn, TOTAL_RELATIONS_AFTER_EXCLUDE) 356 assertDataRestored(restoreConn, map[string]int{ 357 "schema2.foo3": 100, 358 "public.foo": 40000, 359 "public.holds": 50000, 360 "public.sales": 13}) 361 }) 362 It("runs gpbackup and gprestore with exclude-table-file backup flag", func() { 363 skipIfOldBackupVersionBefore("1.4.0") 364 excludeFile := iohelper.MustOpenFileForWriting("/tmp/exclude-tables.txt") 365 utils.MustPrintln(excludeFile, 366 "schema2.foo2\nschema2.returns\npublic.sales\npublic.myseq2\npublic.myview2") 367 output := gpbackup(gpbackupPath, backupHelperPath, 368 "--exclude-table-file", "/tmp/exclude-tables.txt") 369 timestamp := getBackupTimestamp(string(output)) 370 371 gprestore(gprestorePath, restoreHelperPath, timestamp, 372 "--redirect-db", "restoredb") 373 374 assertRelationsCreated(restoreConn, 8) 375 assertDataRestored(restoreConn, map[string]int{ 376 "schema2.foo3": 100, 377 "public.foo": 40000, 378 "public.holds": 50000}) 379 380 _ = os.Remove("/tmp/exclude-tables.txt") 381 }) 382 It("gpbackup with --exclude-table and then runs gprestore when functions and tables depending on each other", func() { 383 skipIfOldBackupVersionBefore("1.19.0") 384 385 testhelper.AssertQueryRuns(backupConn, "CREATE TABLE to_use_for_function (n int);") 386 defer testhelper.AssertQueryRuns(backupConn, "DROP TABLE to_use_for_function;") 387 388 testhelper.AssertQueryRuns(backupConn, "INSERT INTO to_use_for_function values (1);") 389 testhelper.AssertQueryRuns(backupConn, "CREATE FUNCTION func1(val integer) RETURNS integer AS $$ BEGIN RETURN val + (SELECT n FROM to_use_for_function); END; $$ LANGUAGE PLPGSQL;;") 390 defer testhelper.AssertQueryRuns(backupConn, "DROP FUNCTION func1(val integer);") 391 392 testhelper.AssertQueryRuns(backupConn, "CREATE TABLE test_depends_on_function (id integer, claim_id character varying(20) DEFAULT ('WC-'::text || func1(10)::text)) DISTRIBUTED BY (id);") 393 defer testhelper.AssertQueryRuns(backupConn, "DROP TABLE test_depends_on_function;") 394 testhelper.AssertQueryRuns(backupConn, "INSERT INTO test_depends_on_function values (1);") 395 396 output := gpbackup(gpbackupPath, backupHelperPath, 397 "--exclude-table", "public.holds") 398 timestamp := getBackupTimestamp(string(output)) 399 400 gprestore(gprestorePath, restoreHelperPath, timestamp, 401 "--redirect-db", "restoredb") 402 403 assertRelationsCreated(restoreConn, TOTAL_RELATIONS-1+2) // -1 for exclude +2 for 2 new tables 404 assertDataRestored(restoreConn, map[string]int{ 405 "public.foo": 40000, 406 "public.sales": 13, 407 "public.to_use_for_function": 1, 408 "public.test_depends_on_function": 1}) 409 assertArtifactsCleaned(timestamp) 410 }) 411 }) 412 Describe("Restore exclude filtering", func() { 413 It("runs gpbackup and gprestore with exclude-schema restore flag", func() { 414 output := gpbackup(gpbackupPath, backupHelperPath) 415 timestamp := getBackupTimestamp(string(output)) 416 417 gprestore(gprestorePath, restoreHelperPath, timestamp, 418 "--redirect-db", "restoredb", 419 "--exclude-schema", "public") 420 421 assertRelationsCreated(restoreConn, 17) 422 assertDataRestored(restoreConn, schema2TupleCounts) 423 }) 424 It("runs gpbackup and gprestore with exclude-schema-file restore flag", func() { 425 includeFile := iohelper.MustOpenFileForWriting("/tmp/exclude-schema.txt") 426 utils.MustPrintln(includeFile, "public") 427 output := gpbackup(gpbackupPath, backupHelperPath, 428 "--backup-dir", backupDir) 429 timestamp := getBackupTimestamp(string(output)) 430 431 gprestore(gprestorePath, restoreHelperPath, timestamp, 432 "--backup-dir", backupDir, 433 "--redirect-db", "restoredb", 434 "--exclude-schema-file", "/tmp/exclude-schema.txt") 435 436 assertRelationsCreated(restoreConn, 17) 437 assertDataRestored(restoreConn, schema2TupleCounts) 438 439 _ = os.Remove("/tmp/exclude-schema.txt") 440 }) 441 It("runs gpbackup and gprestore with exclude-table restore flag", func() { 442 // Create keyword table to make sure we properly escape it during the exclusion check. 443 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public."user"(i int)`) 444 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public."user"`) 445 446 output := gpbackup(gpbackupPath, backupHelperPath) 447 timestamp := getBackupTimestamp(string(output)) 448 449 gprestore(gprestorePath, restoreHelperPath, timestamp, 450 "--redirect-db", "restoredb", 451 "--exclude-table", "schema2.foo2", 452 "--exclude-table", "schema2.returns", 453 "--exclude-table", "public.myseq2", 454 "--exclude-table", "public.myview2", 455 "--exclude-table", "public.user") 456 457 assertRelationsCreated(restoreConn, TOTAL_RELATIONS_AFTER_EXCLUDE) 458 assertDataRestored(restoreConn, map[string]int{ 459 "schema2.foo3": 100, "public.foo": 40000, "public.holds": 50000, "public.sales": 13}) 460 }) 461 It("runs gpbackup and gprestore with exclude-table-file restore flag", func() { 462 // Create keyword table to make sure we properly escape it during the exclusion check. 463 testhelper.AssertQueryRuns(backupConn, `CREATE TABLE public."user"(i int)`) 464 defer testhelper.AssertQueryRuns(backupConn, `DROP TABLE public."user"`) 465 466 includeFile := iohelper.MustOpenFileForWriting("/tmp/exclude-tables.txt") 467 utils.MustPrintln(includeFile, 468 "schema2.foo2\nschema2.returns\npublic.myseq2\npublic.myview2\npublic.user") 469 output := gpbackup(gpbackupPath, backupHelperPath, 470 "--backup-dir", backupDir) 471 timestamp := getBackupTimestamp(string(output)) 472 473 gprestore(gprestorePath, restoreHelperPath, timestamp, 474 "--redirect-db", "restoredb", 475 "--backup-dir", backupDir, 476 "--exclude-table-file", "/tmp/exclude-tables.txt") 477 478 assertRelationsCreated(restoreConn, TOTAL_RELATIONS_AFTER_EXCLUDE) 479 assertDataRestored(restoreConn, map[string]int{ 480 "public.sales": 13, 481 "public.foo": 40000}) 482 483 _ = os.Remove("/tmp/exclude-tables.txt") 484 }) 485 }) 486 })