github.com/cloudberrydb/gpbackup@v1.0.3-0.20240118031043-5410fd45eed6/integration/options_integration_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"sort"
     5  
     6  	"github.com/cloudberrydb/gp-common-go-libs/testhelper"
     7  	"github.com/cloudberrydb/gpbackup/backup"
     8  	"github.com/cloudberrydb/gpbackup/options"
     9  
    10  	. "github.com/onsi/ginkgo/v2"
    11  	. "github.com/onsi/gomega"
    12  )
    13  
    14  var _ = Describe("Options Integration", func() {
    15  	Describe("QuoteTablesNames", func() {
    16  		It("quotes identifiers as expected", func() {
    17  			tableList := []string{
    18  				`foo.bar`,  // no special characters
    19  				`foo.BAR`,  // capital characters
    20  				`foo.'bar`, // make sure that single quotes are escaped before string is fed to quote_ident
    21  				`foo.2`,    // numbers
    22  				`foo._bar`, // underscore
    23  				`foo ~#$%^&*()_-+[]{}><\|;:/?!,.bar`,
    24  				"foo.\tbar", // important to use double quotes to allow \t to become tab
    25  				"foo.\nbar", // important to use double quotes to allow \n to become a new line
    26  				`foo.\n`,
    27  				`foo."bar`, // quote ident should escape double-quote with another double-quote
    28  			}
    29  			expected := []string{
    30  				`foo.bar`,
    31  				`foo."BAR"`,
    32  				`foo."'bar"`,
    33  				`foo."2"`,
    34  				`foo._bar`, // underscore is not a special character
    35  				`"foo ~#$%^&*()_-+[]{}><\|;:/?!,".bar`,
    36  				"foo.\"\tbar\"",
    37  				"foo.\"\nbar\"",
    38  				`foo."\n"`,
    39  				`foo."""bar"`,
    40  			}
    41  
    42  			resultFQNs, err := options.QuoteTableNames(connectionPool, tableList)
    43  			Expect(err).ToNot(HaveOccurred())
    44  			Expect(expected).To(Equal(resultFQNs))
    45  		})
    46  	})
    47  	Describe("ValidateFilterTables", func() {
    48  		It("validates special chars", func() {
    49  			createSpecialCharacterTables := `
    50  -- special chars
    51  CREATE TABLE public."FOObar" (i int);
    52  CREATE TABLE public."BAR" (i int);
    53  CREATE SCHEMA "CAPschema";
    54  CREATE TABLE "CAPschema"."BAR" (i int);
    55  CREATE TABLE "CAPschema".baz (i int);
    56  CREATE TABLE public.foo_bar (i int);
    57  CREATE TABLE public."foo ~#$%^&*()_-+[]{}><\|;:/?!bar" (i int);
    58  -- special chars: embedded tab char
    59  CREATE TABLE public."tab	bar" (i int);
    60  -- special chars: embedded newline char
    61  CREATE TABLE public."newline
    62  bar" (i int);
    63  `
    64  			dropSpecialCharacterTables := `
    65  -- special chars
    66  DROP TABLE public."FOObar";
    67  DROP TABLE public."BAR";
    68  DROP SCHEMA "CAPschema" cascade;
    69  DROP TABLE public.foo_bar;
    70  DROP TABLE public."foo ~#$%^&*()_-+[]{}><\|;:/?!bar";
    71  -- special chars: embedded tab char
    72  DROP TABLE public."tab	bar";
    73  -- special chars: embedded newline char
    74  DROP TABLE public."newline
    75  bar";
    76  `
    77  			testhelper.AssertQueryRuns(connectionPool, createSpecialCharacterTables)
    78  			defer testhelper.AssertQueryRuns(connectionPool, dropSpecialCharacterTables)
    79  
    80  			tableList := []string{
    81  				`public.BAR`,
    82  				`CAPschema.BAR`,
    83  				`CAPschema.baz`,
    84  				`public.foo_bar`,
    85  				`public.foo ~#$%^&*()_-+[]{}><\|;:/?!bar`,
    86  				"public.tab\tbar",     // important to use double quotes to allow \t to become tab
    87  				"public.newline\nbar", // important to use double quotes to allow \n to become newline
    88  			}
    89  
    90  			backup.ValidateTablesExist(connectionPool, tableList, false)
    91  		})
    92  	})
    93  	Describe("ExpandIncludesForPartitions", func() {
    94  		BeforeEach(func() {
    95  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public."CAPpart"
    96  				(id int, rank int, year int, gender char(1), count int )
    97  					DISTRIBUTED BY (id)
    98  					PARTITION BY LIST (gender)
    99  					( PARTITION girls VALUES ('F'),
   100  					  PARTITION boys VALUES ('M'),
   101  					  DEFAULT PARTITION other )
   102  			`)
   103  		})
   104  
   105  		AfterEach(func() {
   106  			testhelper.AssertQueryRuns(connectionPool, `DROP TABLE public."CAPpart"`)
   107  		})
   108  
   109  		It("adds parent table when child partition with special chars is included", func() {
   110  			err := backupCmdFlags.Set(options.INCLUDE_RELATION, `public.CAPpart_1_prt_girls`)
   111  			Expect(err).ToNot(HaveOccurred())
   112  			subject, err := options.NewOptions(backupCmdFlags)
   113  			Expect(err).To(Not(HaveOccurred()))
   114  			Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart_1_prt_girls"))
   115  			Expect(subject.GetIncludedTables()).To(HaveLen(1))
   116  
   117  			err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags)
   118  			Expect(err).To(Not(HaveOccurred()))
   119  			Expect(subject.GetIncludedTables()).To(HaveLen(2))
   120  			Expect(backupCmdFlags.GetStringArray(options.INCLUDE_RELATION)).To(HaveLen(2))
   121  			Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart_1_prt_girls"))
   122  			Expect(subject.GetIncludedTables()).To(ContainElement("public.CAPpart"))
   123  
   124  			// ensure that ExpandIncludesForPartitions does not disturb the original value
   125  			// that the user typed in, which is used by initializeBackupReport() and
   126  			// is important for incremental backups which must exactly match all flag input
   127  			Expect(subject.GetOriginalIncludedTables()).To(Equal([]string{`public.CAPpart_1_prt_girls`}))
   128  		})
   129  		It("adds parent table when child partition with embedded quote character is included", func() {
   130  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public."""hasquote"""
   131  				(id int, rank int, year int, gender char(1), count int )
   132  					DISTRIBUTED BY (id)
   133  					PARTITION BY LIST (gender)
   134  					( PARTITION girls VALUES ('F'),
   135  					  PARTITION boys VALUES ('M'),
   136  					  DEFAULT PARTITION other )
   137  			`)
   138  			defer testhelper.AssertQueryRuns(connectionPool, `DROP TABLE public."""hasquote"""`)
   139  
   140  			err := backupCmdFlags.Set(options.INCLUDE_RELATION, `public."hasquote"_1_prt_girls`)
   141  			Expect(err).ToNot(HaveOccurred())
   142  			subject, err := options.NewOptions(backupCmdFlags)
   143  			Expect(err).To(Not(HaveOccurred()))
   144  			Expect(subject.GetIncludedTables()).To(ContainElement(`public."hasquote"_1_prt_girls`))
   145  			Expect(subject.GetIncludedTables()).To(HaveLen(1))
   146  
   147  			err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags)
   148  			Expect(err).To(Not(HaveOccurred()))
   149  			Expect(subject.GetIncludedTables()).To(HaveLen(2))
   150  			Expect(backupCmdFlags.GetStringArray(options.INCLUDE_RELATION)).To(HaveLen(2))
   151  			Expect(subject.GetIncludedTables()[0]).To(Equal(`public."hasquote"_1_prt_girls`))
   152  			Expect(subject.GetIncludedTables()[1]).To(Equal(`public."hasquote"`))
   153  		})
   154  		It("returns child partition tables for an included parent table if the leaf-partition-data flag is set and the filter includes a parent partition table", func() {
   155  			_ = backupCmdFlags.Set(options.LEAF_PARTITION_DATA, "true")
   156  			_ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.rank")
   157  
   158  			createStmt := `CREATE TABLE public.rank (id int, rank int, year int, gender
   159  char(1), count int )
   160  DISTRIBUTED BY (id)
   161  PARTITION BY LIST (gender)
   162  ( PARTITION girls VALUES ('F'),
   163    PARTITION boys VALUES ('M'),
   164    DEFAULT PARTITION other );`
   165  			testhelper.AssertQueryRuns(connectionPool, createStmt)
   166  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.rank")
   167  			testhelper.AssertQueryRuns(connectionPool, "CREATE TABLE public.test_table(i int)")
   168  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.test_table")
   169  
   170  			subject, err := options.NewOptions(backupCmdFlags)
   171  			Expect(err).To(Not(HaveOccurred()))
   172  
   173  			err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags)
   174  			Expect(err).To(Not(HaveOccurred()))
   175  
   176  			expectedTableNames := []string{
   177  				"public.rank",
   178  				"public.rank_1_prt_boys",
   179  				"public.rank_1_prt_girls",
   180  				"public.rank_1_prt_other",
   181  			}
   182  
   183  			tables := subject.GetIncludedTables()
   184  			sort.Strings(tables)
   185  			Expect(tables).To(HaveLen(4))
   186  			Expect(tables).To(Equal(expectedTableNames))
   187  		})
   188  		It("returns parent and external leaf partition table if the filter includes a leaf table and leaf-partition-data is set", func() {
   189  			_ = backupCmdFlags.Set(options.LEAF_PARTITION_DATA, "true")
   190  			_ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table_1_prt_boys")
   191  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table (id int, gender char(1))
   192  DISTRIBUTED BY (id)
   193  PARTITION BY LIST (gender)
   194  ( PARTITION girls VALUES ('F'),
   195    PARTITION boys VALUES ('M'),
   196    DEFAULT PARTITION other );`)
   197  			testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table_ext_part_ (like public.partition_table_1_prt_girls)
   198  EXECUTE 'echo -e "2\n1"' on host
   199  FORMAT 'csv';`)
   200  			testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table EXCHANGE PARTITION girls WITH TABLE public.partition_table_ext_part_ WITHOUT VALIDATION;`)
   201  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table")
   202  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table_ext_part_")
   203  
   204  			subject, err := options.NewOptions(backupCmdFlags)
   205  			Expect(err).To(Not(HaveOccurred()))
   206  
   207  			err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags)
   208  			Expect(err).To(Not(HaveOccurred()))
   209  
   210  			expectedTableNames := []string{
   211  				"public.partition_table",
   212  				"public.partition_table_1_prt_boys",
   213  			}
   214  			if false {
   215  				// For GPDB 4, 5, and 6, the leaf partition metadata is a part of the large root partition DDL.
   216  				// For GPDB 7+, the leaf partition has its own separate DDL and attaches onto the root partition
   217  				// with ALTER TABLE ATTACH PARTITION. Therefore, only the included leaf partition and its root
   218  				// partition will have their metadata dumped.
   219  				expectedTableNames = append(expectedTableNames, "public.partition_table_1_prt_girls")
   220  			}
   221  
   222  			tables := subject.GetIncludedTables()
   223  			sort.Strings(tables)
   224  			Expect(tables).To(HaveLen(len(expectedTableNames)))
   225  			Expect(tables).To(Equal(expectedTableNames))
   226  		})
   227  		It("returns external partition tables for an included parent table if the filter includes a parent partition table", func() {
   228  			_ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table1")
   229  			_ = backupCmdFlags.Set(options.INCLUDE_RELATION, "public.partition_table2_1_prt_other")
   230  
   231  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table1 (id int, gender char(1))
   232  DISTRIBUTED BY (id)
   233  PARTITION BY LIST (gender)
   234  ( PARTITION girls VALUES ('F'),
   235    PARTITION boys VALUES ('M'),
   236    DEFAULT PARTITION other );`)
   237  			testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table1_ext_part_ (like public.partition_table1_1_prt_boys)
   238  EXECUTE 'echo -e "2\n1"' on host
   239  FORMAT 'csv';`)
   240  			testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table1 EXCHANGE PARTITION boys WITH TABLE public.partition_table1_ext_part_ WITHOUT VALIDATION;`)
   241  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table1")
   242  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table1_ext_part_")
   243  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table2 (id int, gender char(1))
   244  DISTRIBUTED BY (id)
   245  PARTITION BY LIST (gender)
   246  ( PARTITION girls VALUES ('F'),
   247    PARTITION boys VALUES ('M'),
   248    DEFAULT PARTITION other );`)
   249  			testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table2_ext_part_ (like public.partition_table2_1_prt_girls)
   250  EXECUTE 'echo -e "2\n1"' on host
   251  FORMAT 'csv';`)
   252  			testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table2 EXCHANGE PARTITION girls WITH TABLE public.partition_table2_ext_part_ WITHOUT VALIDATION;`)
   253  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table2")
   254  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table2_ext_part_")
   255  			testhelper.AssertQueryRuns(connectionPool, `CREATE TABLE public.partition_table3 (id int, gender char(1))
   256  DISTRIBUTED BY (id)
   257  PARTITION BY LIST (gender)
   258  ( PARTITION girls VALUES ('F'),
   259    PARTITION boys VALUES ('M'),
   260    DEFAULT PARTITION other );`)
   261  			testhelper.AssertQueryRuns(connectionPool, `CREATE EXTERNAL WEB TABLE public.partition_table3_ext_part_ (like public.partition_table3_1_prt_girls)
   262  EXECUTE 'echo -e "2\n1"' on host
   263  FORMAT 'csv';`)
   264  			testhelper.AssertQueryRuns(connectionPool, `ALTER TABLE public.partition_table3 EXCHANGE PARTITION girls WITH TABLE public.partition_table3_ext_part_ WITHOUT VALIDATION;`)
   265  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table3")
   266  			defer testhelper.AssertQueryRuns(connectionPool, "DROP TABLE public.partition_table3_ext_part_")
   267  
   268  			subject, err := options.NewOptions(backupCmdFlags)
   269  			Expect(err).To(Not(HaveOccurred()))
   270  
   271  			err = subject.ExpandIncludesForPartitions(connectionPool, backupCmdFlags)
   272  			Expect(err).To(Not(HaveOccurred()))
   273  
   274  			var expectedTableNames []string
   275  			if false {
   276  				expectedTableNames = []string{
   277  					"public.partition_table1",
   278  					"public.partition_table1_1_prt_boys",
   279  					"public.partition_table2",
   280  					"public.partition_table2_1_prt_girls",
   281  					"public.partition_table2_1_prt_other",
   282  				}
   283  			} else {
   284  				// For GPDB 7+, each leaf partition of the included partition_table1 will have their own
   285  				// separate DDL along with an ATTACH PARTITION. Since only partition_table2_1_prt_other
   286  				// is included, only it and its root partition will be a part of the list.
   287  				expectedTableNames = []string{
   288  					"public.partition_table1",
   289  					"public.partition_table1_1_prt_boys",
   290  					"public.partition_table1_1_prt_girls",
   291  					"public.partition_table1_1_prt_other",
   292  					"public.partition_table2",
   293  					"public.partition_table2_1_prt_other",
   294  				}
   295  			}
   296  
   297  			tables := subject.GetIncludedTables()
   298  			sort.Strings(tables)
   299  			Expect(tables).To(HaveLen(len(expectedTableNames)))
   300  			Expect(tables).To(Equal(expectedTableNames))
   301  		})
   302  	})
   303  })