github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/libraries/doltcore/sqle/tables.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package sqle
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"errors"
    21  	"fmt"
    22  	"io"
    23  	"os"
    24  	"runtime"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  
    29  	"github.com/dolthub/go-mysql-server/sql"
    30  	"github.com/dolthub/vitess/go/sqltypes"
    31  
    32  	"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
    33  	"github.com/dolthub/dolt/go/libraries/doltcore/schema"
    34  	"github.com/dolthub/dolt/go/libraries/doltcore/schema/alterschema"
    35  	"github.com/dolthub/dolt/go/libraries/doltcore/schema/encoding"
    36  	"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
    37  	"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
    38  	"github.com/dolthub/dolt/go/libraries/doltcore/table/editor"
    39  	"github.com/dolthub/dolt/go/libraries/utils/set"
    40  	"github.com/dolthub/dolt/go/store/hash"
    41  	"github.com/dolthub/dolt/go/store/types"
    42  )
    43  
    44  const (
    45  	partitionMultiplier = 2.0
    46  )
    47  
    48  var MinRowsPerPartition uint64 = 1024
    49  
    50  func init() {
    51  	isTest := false
    52  	for _, arg := range os.Args {
    53  		lwr := strings.ToLower(arg)
    54  		if lwr == "-test.v" ||
    55  			lwr == "-test.run" ||
    56  			strings.HasPrefix(lwr, "-test.testlogfile") ||
    57  			strings.HasPrefix(lwr, "-test.timeout") ||
    58  			strings.HasPrefix(lwr, "-test.count") {
    59  			isTest = true
    60  			break
    61  		}
    62  	}
    63  
    64  	if isTest {
    65  		MinRowsPerPartition = 2
    66  	}
    67  }
    68  
    69  type projected interface {
    70  	Project() []string
    71  }
    72  
    73  // DoltTable implements the sql.Table interface and gives access to dolt table rows and schema.
    74  type DoltTable struct {
    75  	tableName    string
    76  	sqlSch       sql.Schema
    77  	db           SqlDatabase
    78  	lockedToRoot *doltdb.RootValue
    79  	nbf          *types.NomsBinFormat
    80  	sch          schema.Schema
    81  	autoIncCol   schema.Column
    82  
    83  	projectedCols []string
    84  	temporary     bool
    85  }
    86  
    87  func NewDoltTable(name string, sch schema.Schema, tbl *doltdb.Table, db SqlDatabase, isTemporary bool) *DoltTable {
    88  	var autoCol schema.Column
    89  	_ = sch.GetAllCols().Iter(func(tag uint64, col schema.Column) (stop bool, err error) {
    90  		if col.AutoIncrement {
    91  			autoCol = col
    92  			stop = true
    93  		}
    94  		return
    95  	})
    96  
    97  	return &DoltTable{
    98  		tableName:     name,
    99  		db:            db,
   100  		nbf:           tbl.Format(),
   101  		sch:           sch,
   102  		autoIncCol:    autoCol,
   103  		projectedCols: nil,
   104  		temporary:     isTemporary,
   105  	}
   106  }
   107  
   108  // LockedToRoot returns a version of this table with its root value locked to the given value. The table's values will
   109  // not change as the session's root value changes. Appropriate for AS OF queries, or other use cases where the table's
   110  // values should not change throughout execution of a session.
   111  func (t DoltTable) LockedToRoot(rootValue *doltdb.RootValue) *DoltTable {
   112  	t.lockedToRoot = rootValue
   113  	return &t
   114  }
   115  
   116  var _ sql.Table = (*DoltTable)(nil)
   117  var _ sql.TemporaryTable = (*DoltTable)(nil)
   118  var _ sql.IndexedTable = (*DoltTable)(nil)
   119  var _ sql.ForeignKeyTable = (*DoltTable)(nil)
   120  var _ sql.StatisticsTable = (*DoltTable)(nil)
   121  
   122  // projected tables disabled for now.  Looks like some work needs to be done in the analyzer as there are cases
   123  // where the projected columns do not contain every column needed.  Seed this with natural and other joins.  There
   124  // may be other cases.
   125  //var _ sql.ProjectedTable = (*DoltTable)(nil)
   126  
   127  // WithIndexLookup implements sql.IndexedTable
   128  func (t *DoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table {
   129  	dil, ok := lookup.(*doltIndexLookup)
   130  	if !ok {
   131  		return sqlutil.NewStaticErrorTable(t, fmt.Errorf("Unrecognized indexLookup %T", lookup))
   132  	}
   133  
   134  	return &IndexedDoltTable{
   135  		table:       t,
   136  		indexLookup: dil,
   137  	}
   138  }
   139  
   140  // doltTable returns the underlying doltTable from the current session
   141  func (t *DoltTable) doltTable(ctx *sql.Context) (*doltdb.Table, error) {
   142  	root := t.lockedToRoot
   143  	var err error
   144  	if root == nil {
   145  		root, err = t.getRoot(ctx)
   146  		if err != nil {
   147  			return nil, err
   148  		}
   149  	}
   150  
   151  	table, ok, err := root.GetTable(ctx, t.tableName)
   152  	if err != nil {
   153  		return nil, err
   154  	}
   155  	if !ok {
   156  		return nil, fmt.Errorf("table not found: %s", t.tableName)
   157  	}
   158  
   159  	return table, nil
   160  }
   161  
   162  // getRoot returns the appropriate root value for this session. The only controlling factor
   163  // is whether this is a temporary table or not.
   164  func (t *DoltTable) getRoot(ctx *sql.Context) (*doltdb.RootValue, error) {
   165  	if t.temporary {
   166  		root, ok := t.db.GetTemporaryTablesRoot(ctx)
   167  		if !ok {
   168  			return nil, fmt.Errorf("error: manipulating temporary table root when it does not exist")
   169  		}
   170  
   171  		return root, nil
   172  	}
   173  
   174  	return t.db.GetRoot(ctx)
   175  }
   176  
   177  // GetIndexes implements sql.IndexedTable
   178  func (t *DoltTable) GetIndexes(ctx *sql.Context) ([]sql.Index, error) {
   179  	tbl, err := t.doltTable(ctx)
   180  	if err != nil {
   181  		return nil, err
   182  	}
   183  
   184  	sch, err := tbl.GetSchema(ctx)
   185  	if err != nil {
   186  		return nil, err
   187  	}
   188  
   189  	rowData, err := tbl.GetRowData(ctx)
   190  	if err != nil {
   191  		return nil, err
   192  	}
   193  
   194  	var sqlIndexes []sql.Index
   195  	cols := sch.GetPKCols().GetColumns()
   196  
   197  	if len(cols) > 0 {
   198  		sqlIndexes = append(sqlIndexes, &doltIndex{
   199  			cols:         cols,
   200  			db:           t.db,
   201  			id:           "PRIMARY",
   202  			indexRowData: rowData,
   203  			indexSch:     sch,
   204  			table:        tbl,
   205  			tableData:    rowData,
   206  			tableName:    t.Name(),
   207  			tableSch:     sch,
   208  			unique:       true,
   209  			generated:    false,
   210  		})
   211  		for i := 1; i < len(cols); i++ {
   212  			sqlIndexes = append(sqlIndexes, &doltIndex{
   213  				cols:         cols[:i],
   214  				db:           t.db,
   215  				id:           fmt.Sprintf("PRIMARY_PARTIAL_%d", i),
   216  				indexRowData: rowData,
   217  				indexSch:     sch,
   218  				table:        tbl,
   219  				tableData:    rowData,
   220  				tableName:    t.Name(),
   221  				tableSch:     sch,
   222  				unique:       false,
   223  				comment:      fmt.Sprintf("partial of PRIMARY multi-column index on %d column(s)", i),
   224  				generated:    true,
   225  			})
   226  		}
   227  	}
   228  
   229  	for _, index := range sch.Indexes().AllIndexes() {
   230  		indexRowData, err := tbl.GetIndexRowData(ctx, index.Name())
   231  		if err != nil {
   232  			return nil, err
   233  		}
   234  		cols := make([]schema.Column, index.Count())
   235  		for i, tag := range index.IndexedColumnTags() {
   236  			cols[i], _ = index.GetColumn(tag)
   237  		}
   238  		sqlIndexes = append(sqlIndexes, &doltIndex{
   239  			cols:         cols,
   240  			db:           t.db,
   241  			id:           index.Name(),
   242  			indexRowData: indexRowData,
   243  			indexSch:     index.Schema(),
   244  			table:        tbl,
   245  			tableData:    rowData,
   246  			tableName:    t.Name(),
   247  			tableSch:     sch,
   248  			unique:       index.IsUnique(),
   249  			comment:      index.Comment(),
   250  			generated:    false,
   251  		})
   252  		for i := 1; i < len(cols); i++ {
   253  			sqlIndexes = append(sqlIndexes, &doltIndex{
   254  				cols:         cols[:i],
   255  				db:           t.db,
   256  				id:           fmt.Sprintf("%s_PARTIAL_%d", index.Name(), i),
   257  				indexRowData: indexRowData,
   258  				indexSch:     index.Schema(),
   259  				table:        tbl,
   260  				tableData:    rowData,
   261  				tableName:    t.Name(),
   262  				tableSch:     sch,
   263  				unique:       false,
   264  				comment:      fmt.Sprintf("prefix of %s multi-column index on %d column(s)", index.Name(), i),
   265  				generated:    true,
   266  			})
   267  		}
   268  	}
   269  
   270  	return sqlIndexes, nil
   271  }
   272  
   273  // GetAutoIncrementValue gets the last AUTO_INCREMENT value
   274  func (t *DoltTable) GetAutoIncrementValue(ctx *sql.Context) (interface{}, error) {
   275  	table, err := t.doltTable(ctx)
   276  	if err != nil {
   277  		return nil, err
   278  	}
   279  
   280  	val, err := table.GetAutoIncrementValue(ctx)
   281  	if err != nil {
   282  		return nil, err
   283  	}
   284  	return t.autoIncCol.TypeInfo.ConvertNomsValueToValue(val)
   285  }
   286  
   287  // Name returns the name of the table.
   288  func (t *DoltTable) Name() string {
   289  	return t.tableName
   290  }
   291  
   292  // String returns a human-readable string to display the name of this SQL node.
   293  func (t *DoltTable) String() string {
   294  	return t.tableName
   295  }
   296  
   297  // NumRows returns the unfiltered count of rows contained in the table
   298  func (t *DoltTable) NumRows(ctx *sql.Context) (uint64, error) {
   299  	table, err := t.doltTable(ctx)
   300  	if err != nil {
   301  		return 0, err
   302  	}
   303  
   304  	m, err := table.GetRowData(ctx)
   305  	if err != nil {
   306  		return 0, err
   307  	}
   308  
   309  	return m.Len(), nil
   310  }
   311  
   312  // Format returns the NomsBinFormat for the underlying table
   313  func (t *DoltTable) Format() *types.NomsBinFormat {
   314  	return t.nbf
   315  }
   316  
   317  // Schema returns the schema for this table.
   318  func (t *DoltTable) Schema() sql.Schema {
   319  	return t.sqlSchema()
   320  }
   321  
   322  func (t *DoltTable) sqlSchema() sql.Schema {
   323  	if t.sqlSch != nil {
   324  		return t.sqlSch
   325  	}
   326  
   327  	// TODO: fix panics
   328  	sqlSch, err := sqlutil.FromDoltSchema(t.tableName, t.sch)
   329  	if err != nil {
   330  		panic(err)
   331  	}
   332  
   333  	t.sqlSch = sqlSch
   334  	return sqlSch
   335  }
   336  
   337  // Returns the partitions for this table.
   338  func (t *DoltTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
   339  	table, err := t.doltTable(ctx)
   340  	if err != nil {
   341  		return nil, err
   342  	}
   343  
   344  	rowData, err := table.GetRowData(ctx)
   345  
   346  	if err != nil {
   347  		return nil, err
   348  	}
   349  
   350  	numElements := rowData.Len()
   351  
   352  	if numElements == 0 {
   353  		return newDoltTablePartitionIter(rowData, doltTablePartition{0, 0, rowData}), nil
   354  	}
   355  
   356  	maxPartitions := uint64(partitionMultiplier * runtime.NumCPU())
   357  	numPartitions := (numElements / MinRowsPerPartition) + 1
   358  
   359  	if numPartitions > maxPartitions {
   360  		numPartitions = maxPartitions
   361  	}
   362  
   363  	if schema.IsKeyless(t.sch) {
   364  		numPartitions = 1
   365  	}
   366  
   367  	partitions := make([]doltTablePartition, numPartitions)
   368  	itemsPerPartition := numElements / numPartitions
   369  	for i := uint64(0); i < numPartitions-1; i++ {
   370  		partitions[i] = doltTablePartition{i * itemsPerPartition, (i + 1) * itemsPerPartition, rowData}
   371  	}
   372  	partitions[numPartitions-1] = doltTablePartition{(numPartitions - 1) * itemsPerPartition, numElements, rowData}
   373  
   374  	return newDoltTablePartitionIter(rowData, partitions...), nil
   375  }
   376  
   377  func (t *DoltTable) IsTemporary() bool {
   378  	return t.temporary
   379  }
   380  
   381  func (t *DoltTable) DataLength(ctx *sql.Context) (uint64, error) {
   382  	schema := t.Schema()
   383  	var numBytesPerRow uint64 = 0
   384  	for _, col := range schema {
   385  		switch n := col.Type.(type) {
   386  		case sql.NumberType:
   387  			numBytesPerRow += 8
   388  		case sql.StringType:
   389  			numBytesPerRow += uint64(n.MaxByteLength())
   390  		case sql.BitType:
   391  			numBytesPerRow += 1
   392  		case sql.DatetimeType:
   393  			numBytesPerRow += 8
   394  		case sql.DecimalType:
   395  			numBytesPerRow += uint64(n.MaximumScale())
   396  		case sql.EnumType:
   397  			numBytesPerRow += 2
   398  		case sql.JsonType:
   399  			numBytesPerRow += 20
   400  		case sql.NullType:
   401  			numBytesPerRow += 1
   402  		case sql.TimeType:
   403  			numBytesPerRow += 16
   404  		case sql.YearType:
   405  			numBytesPerRow += 8
   406  		}
   407  	}
   408  
   409  	numRows, err := t.NumRows(ctx)
   410  	if err != nil {
   411  		return 0, err
   412  	}
   413  
   414  	return numBytesPerRow * numRows, nil
   415  }
   416  
   417  type emptyRowIterator struct{}
   418  
   419  func (itr emptyRowIterator) Next() (sql.Row, error) {
   420  	return nil, io.EOF
   421  }
   422  
   423  func (itr emptyRowIterator) Close(*sql.Context) error {
   424  	return nil
   425  }
   426  
   427  // PartitionRows returns the table rows for the partition given
   428  func (t *DoltTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) {
   429  	table, err := t.doltTable(ctx)
   430  	if err != nil {
   431  		return nil, err
   432  	}
   433  
   434  	return partitionRows(ctx, table, t.projectedCols, partition)
   435  }
   436  
   437  func partitionRows(ctx *sql.Context, t *doltdb.Table, projCols []string, partition sql.Partition) (sql.RowIter, error) {
   438  	switch typedPartition := partition.(type) {
   439  	case doltTablePartition:
   440  		if typedPartition.end == 0 {
   441  			return emptyRowIterator{}, nil
   442  		}
   443  
   444  		return newRowIterator(ctx, t, projCols, &typedPartition)
   445  	case sqlutil.SinglePartition:
   446  		return newRowIterator(ctx, t, projCols, &doltTablePartition{rowData: typedPartition.RowData, end: NoUpperBound})
   447  	}
   448  
   449  	return nil, errors.New("unsupported partition type")
   450  }
   451  
   452  // WritableDoltTable allows updating, deleting, and inserting new rows. It implements sql.UpdatableTable and friends.
   453  type WritableDoltTable struct {
   454  	*DoltTable
   455  	db Database
   456  	ed *sqlTableEditor
   457  }
   458  
   459  var _ sql.UpdatableTable = (*WritableDoltTable)(nil)
   460  var _ sql.DeletableTable = (*WritableDoltTable)(nil)
   461  var _ sql.InsertableTable = (*WritableDoltTable)(nil)
   462  var _ sql.ReplaceableTable = (*WritableDoltTable)(nil)
   463  var _ sql.AutoIncrementTable = (*WritableDoltTable)(nil)
   464  var _ sql.TruncateableTable = (*WritableDoltTable)(nil)
   465  var _ sql.CheckTable = (*WritableDoltTable)(nil)
   466  
   467  func (t *WritableDoltTable) setRoot(ctx *sql.Context, newRoot *doltdb.RootValue) error {
   468  	if t.temporary {
   469  		return t.db.SetTemporaryRoot(ctx, newRoot)
   470  	}
   471  
   472  	return t.db.SetRoot(ctx, newRoot)
   473  }
   474  
   475  func (t *WritableDoltTable) WithIndexLookup(lookup sql.IndexLookup) sql.Table {
   476  	dil, ok := lookup.(*doltIndexLookup)
   477  	if !ok {
   478  		return sqlutil.NewStaticErrorTable(t, fmt.Errorf("Unrecognized indexLookup %T", lookup))
   479  	}
   480  	return &WritableIndexedDoltTable{
   481  		WritableDoltTable: t,
   482  		indexLookup:       dil,
   483  	}
   484  }
   485  
   486  func (t *WritableDoltTable) WithProjection(colNames []string) sql.Table {
   487  	return &WritableDoltTable{
   488  		DoltTable: t.DoltTable.WithProjection(colNames).(*DoltTable),
   489  		db:        t.db,
   490  		ed:        t.ed,
   491  	}
   492  }
   493  
   494  // Inserter implements sql.InsertableTable
   495  func (t *WritableDoltTable) Inserter(ctx *sql.Context) sql.RowInserter {
   496  	te, err := t.getTableEditor(ctx)
   497  	if err != nil {
   498  		return sqlutil.NewStaticErrorEditor(err)
   499  	}
   500  	return te
   501  }
   502  
   503  func (t *WritableDoltTable) getTableEditor(ctx *sql.Context) (*sqlTableEditor, error) {
   504  	sess := DSessFromSess(ctx.Session)
   505  
   506  	// In batched mode, reuse the same table editor. Otherwise, hand out a new one
   507  	if sess.batchMode == batched {
   508  		if t.ed != nil {
   509  			return t.ed, nil
   510  		}
   511  		var err error
   512  		t.ed, err = newSqlTableEditor(ctx, t)
   513  		return t.ed, err
   514  	}
   515  	return newSqlTableEditor(ctx, t)
   516  }
   517  
   518  func (t *WritableDoltTable) flushBatchedEdits(ctx *sql.Context) error {
   519  	if t.ed != nil {
   520  		err := t.ed.flush(ctx)
   521  		t.ed = nil
   522  		return err
   523  	}
   524  	return nil
   525  }
   526  
   527  // Deleter implements sql.DeletableTable
   528  func (t *WritableDoltTable) Deleter(ctx *sql.Context) sql.RowDeleter {
   529  	te, err := t.getTableEditor(ctx)
   530  	if err != nil {
   531  		return sqlutil.NewStaticErrorEditor(err)
   532  	}
   533  	return te
   534  }
   535  
   536  // Replacer implements sql.ReplaceableTable
   537  func (t *WritableDoltTable) Replacer(ctx *sql.Context) sql.RowReplacer {
   538  	te, err := t.getTableEditor(ctx)
   539  	if err != nil {
   540  		return sqlutil.NewStaticErrorEditor(err)
   541  	}
   542  	return te
   543  }
   544  
   545  // Truncate implements sql.TruncateableTable
   546  func (t *WritableDoltTable) Truncate(ctx *sql.Context) (int, error) {
   547  	table, err := t.doltTable(ctx)
   548  	if err != nil {
   549  		return 0, err
   550  	}
   551  
   552  	rowData, err := table.GetRowData(ctx)
   553  	if err != nil {
   554  		return 0, err
   555  	}
   556  	numOfRows := int(rowData.Len())
   557  	schVal, err := encoding.MarshalSchemaAsNomsValue(ctx, table.ValueReadWriter(), t.sch)
   558  	if err != nil {
   559  		return 0, err
   560  	}
   561  	empty, err := types.NewMap(ctx, table.ValueReadWriter())
   562  	if err != nil {
   563  		return 0, err
   564  	}
   565  	// truncate table resets auto-increment value
   566  	newTable, err := doltdb.NewTable(ctx, table.ValueReadWriter(), schVal, empty, empty, nil)
   567  	if err != nil {
   568  		return 0, err
   569  	}
   570  	newTable, err = editor.RebuildAllIndexes(ctx, newTable)
   571  	if err != nil {
   572  		return 0, err
   573  	}
   574  
   575  	root, err := t.getRoot(ctx)
   576  	if err != nil {
   577  		return 0, err
   578  	}
   579  	newRoot, err := root.PutTable(ctx, t.tableName, newTable)
   580  	if err != nil {
   581  		return 0, err
   582  	}
   583  	err = t.setRoot(ctx, newRoot)
   584  	if err != nil {
   585  		return 0, err
   586  	}
   587  
   588  	return numOfRows, nil
   589  }
   590  
   591  // Updater implements sql.UpdatableTable
   592  func (t *WritableDoltTable) Updater(ctx *sql.Context) sql.RowUpdater {
   593  	te, err := t.getTableEditor(ctx)
   594  	if err != nil {
   595  		return sqlutil.NewStaticErrorEditor(err)
   596  	}
   597  	return te
   598  }
   599  
   600  // AutoIncrementSetter implements sql.AutoIncrementTable
   601  func (t *WritableDoltTable) AutoIncrementSetter(ctx *sql.Context) sql.AutoIncrementSetter {
   602  	te, err := t.getTableEditor(ctx)
   603  	if err != nil {
   604  		return sqlutil.NewStaticErrorEditor(err)
   605  	}
   606  	return te
   607  }
   608  
   609  // GetAutoIncrementValue gets the last AUTO_INCREMENT value
   610  func (t *WritableDoltTable) GetAutoIncrementValue(ctx *sql.Context) (interface{}, error) {
   611  	if !t.autoIncCol.AutoIncrement {
   612  		return nil, sql.ErrNoAutoIncrementCol
   613  	}
   614  	if t.ed != nil {
   615  		return t.ed.GetAutoIncrementValue()
   616  	}
   617  	return t.DoltTable.GetAutoIncrementValue(ctx)
   618  }
   619  
   620  func (t *WritableDoltTable) GetChecks(ctx *sql.Context) ([]sql.CheckDefinition, error) {
   621  	table, err := t.doltTable(ctx)
   622  	if err != nil {
   623  		return nil, err
   624  	}
   625  
   626  	sch, err := table.GetSchema(ctx)
   627  	if err != nil {
   628  		return nil, err
   629  	}
   630  
   631  	checks := make([]sql.CheckDefinition, sch.Checks().Count())
   632  	for i, check := range sch.Checks().AllChecks() {
   633  		checks[i] = sql.CheckDefinition{
   634  			Name:            check.Name(),
   635  			CheckExpression: check.Expression(),
   636  			Enforced:        check.Enforced(),
   637  		}
   638  	}
   639  
   640  	return checks, nil
   641  }
   642  
   643  // GetForeignKeys implements sql.ForeignKeyTable
   644  func (t *DoltTable) GetForeignKeys(ctx *sql.Context) ([]sql.ForeignKeyConstraint, error) {
   645  	root, err := t.getRoot(ctx)
   646  	if err != nil {
   647  		return nil, err
   648  	}
   649  
   650  	fkc, err := root.GetForeignKeyCollection(ctx)
   651  	if err != nil {
   652  		return nil, err
   653  	}
   654  
   655  	declaredFk, _ := fkc.KeysForTable(t.tableName)
   656  	toReturn := make([]sql.ForeignKeyConstraint, len(declaredFk))
   657  
   658  	for i, fk := range declaredFk {
   659  		parent, ok, err := root.GetTable(ctx, fk.ReferencedTableName)
   660  		if err != nil {
   661  			return nil, err
   662  		}
   663  		if !ok {
   664  			return nil, fmt.Errorf("cannot find table %s "+
   665  				"referenced in foreign key %s", fk.ReferencedTableName, fk.Name)
   666  		}
   667  
   668  		parentSch, err := parent.GetSchema(ctx)
   669  		if err != nil {
   670  			return nil, err
   671  		}
   672  
   673  		toReturn[i], err = toForeignKeyConstraint(fk, t.sch, parentSch)
   674  		if err != nil {
   675  			return nil, err
   676  		}
   677  	}
   678  
   679  	return toReturn, nil
   680  }
   681  
   682  func (t *DoltTable) Projection() []string {
   683  	return t.projectedCols
   684  }
   685  
   686  func (t DoltTable) WithProjection(colNames []string) sql.Table {
   687  	t.projectedCols = colNames
   688  	return &t
   689  }
   690  
   691  var _ sql.PartitionIter = (*doltTablePartitionIter)(nil)
   692  
   693  // doltTablePartitionIter, an object that knows how to return the single partition exactly once.
   694  type doltTablePartitionIter struct {
   695  	i          int
   696  	mu         *sync.Mutex
   697  	rowData    types.Map
   698  	partitions []doltTablePartition
   699  }
   700  
   701  func newDoltTablePartitionIter(rowData types.Map, partitions ...doltTablePartition) *doltTablePartitionIter {
   702  	return &doltTablePartitionIter{0, &sync.Mutex{}, rowData, partitions}
   703  }
   704  
   705  // Close is required by the sql.PartitionIter interface. Does nothing.
   706  func (itr *doltTablePartitionIter) Close(*sql.Context) error {
   707  	return nil
   708  }
   709  
   710  // Next returns the next partition if there is one, or io.EOF if there isn't.
   711  func (itr *doltTablePartitionIter) Next() (sql.Partition, error) {
   712  	itr.mu.Lock()
   713  	defer itr.mu.Unlock()
   714  
   715  	if itr.i >= len(itr.partitions) {
   716  		return nil, io.EOF
   717  	}
   718  
   719  	partition := itr.partitions[itr.i]
   720  	itr.i++
   721  
   722  	return partition, nil
   723  }
   724  
   725  var _ sql.Partition = (*doltTablePartition)(nil)
   726  
   727  const NoUpperBound = 0xffffffffffffffff
   728  
   729  type doltTablePartition struct {
   730  	// start is the first index of this partition (inclusive)
   731  	start uint64
   732  	// all elements in the partition will be less than end (exclusive)
   733  	end     uint64
   734  	rowData types.Map
   735  }
   736  
   737  // Key returns the key for this partition, which must uniquely identity the partition.
   738  func (p doltTablePartition) Key() []byte {
   739  	return []byte(strconv.FormatUint(p.start, 10) + " >= i < " + strconv.FormatUint(p.end, 10))
   740  }
   741  
   742  // IteratorForPartition returns a types.MapIterator implementation which will iterate through the values
   743  // for index = start; index < end.  This iterator is not thread safe and should only be used from a single go routine
   744  // unless paired with a mutex
   745  func (p doltTablePartition) IteratorForPartition(ctx context.Context, m types.Map) (types.MapTupleIterator, error) {
   746  	return m.RangeIterator(ctx, p.start, p.end)
   747  }
   748  
   749  type partitionIter struct {
   750  	pos  uint64
   751  	end  uint64
   752  	iter types.MapIterator
   753  }
   754  
   755  func newPartitionIter(ctx context.Context, m types.Map, start, end uint64) (*partitionIter, error) {
   756  	iter, err := m.BufferedIteratorAt(ctx, start)
   757  
   758  	if err != nil {
   759  		return nil, err
   760  	}
   761  
   762  	return &partitionIter{
   763  		start,
   764  		end,
   765  		iter,
   766  	}, nil
   767  }
   768  
   769  func (p *partitionIter) Next(ctx context.Context) (k, v types.Value, err error) {
   770  	if p.pos >= p.end {
   771  		// types package does not use io.EOF
   772  		return nil, nil, nil
   773  	}
   774  
   775  	p.pos++
   776  	return p.iter.Next(ctx)
   777  }
   778  
   779  // AlterableDoltTable allows altering the schema of the table. It implements sql.AlterableTable.
   780  type AlterableDoltTable struct {
   781  	WritableDoltTable
   782  }
   783  
   784  var _ sql.AlterableTable = (*AlterableDoltTable)(nil)
   785  var _ sql.IndexAlterableTable = (*AlterableDoltTable)(nil)
   786  var _ sql.ForeignKeyAlterableTable = (*AlterableDoltTable)(nil)
   787  var _ sql.ForeignKeyTable = (*AlterableDoltTable)(nil)
   788  var _ sql.CheckAlterableTable = (*AlterableDoltTable)(nil)
   789  
   790  // AddColumn implements sql.AlterableTable
   791  func (t *AlterableDoltTable) AddColumn(ctx *sql.Context, column *sql.Column, order *sql.ColumnOrder) error {
   792  	root, err := t.getRoot(ctx)
   793  
   794  	if err != nil {
   795  		return err
   796  	}
   797  
   798  	table, _, err := root.GetTable(ctx, t.tableName)
   799  	if err != nil {
   800  		return err
   801  	}
   802  
   803  	ti, err := typeinfo.FromSqlType(column.Type)
   804  	if err != nil {
   805  		return err
   806  	}
   807  	tags, err := root.GenerateTagsForNewColumns(ctx, t.tableName, []string{column.Name}, []types.NomsKind{ti.NomsKind()}, nil)
   808  	if err != nil {
   809  		return err
   810  	}
   811  
   812  	col, err := sqlutil.ToDoltCol(tags[0], column)
   813  	if err != nil {
   814  		return err
   815  	}
   816  
   817  	if col.IsPartOfPK {
   818  		return errors.New("adding primary keys is not supported")
   819  	}
   820  
   821  	nullable := alterschema.NotNull
   822  	if col.IsNullable() {
   823  		nullable = alterschema.Null
   824  	}
   825  
   826  	updatedTable, err := alterschema.AddColumnToTable(ctx, root, table, t.tableName, col.Tag, col.Name, col.TypeInfo, nullable, col.Default, col.Comment, orderToOrder(order))
   827  	if err != nil {
   828  		return err
   829  	}
   830  
   831  	newRoot, err := root.PutTable(ctx, t.tableName, updatedTable)
   832  	if err != nil {
   833  		return err
   834  	}
   835  
   836  	return t.setRoot(ctx, newRoot)
   837  }
   838  
   839  func orderToOrder(order *sql.ColumnOrder) *alterschema.ColumnOrder {
   840  	if order == nil {
   841  		return nil
   842  	}
   843  	return &alterschema.ColumnOrder{
   844  		First: order.First,
   845  		After: order.AfterColumn,
   846  	}
   847  }
   848  
   849  // DropColumn implements sql.AlterableTable
   850  func (t *AlterableDoltTable) DropColumn(ctx *sql.Context, columnName string) error {
   851  	root, err := t.getRoot(ctx)
   852  	if err != nil {
   853  		return err
   854  	}
   855  
   856  	updatedTable, _, err := root.GetTable(ctx, t.tableName)
   857  	if err != nil {
   858  		return err
   859  	}
   860  
   861  	sch, err := updatedTable.GetSchema(ctx)
   862  	if err != nil {
   863  		return err
   864  	}
   865  
   866  	for _, index := range sch.Indexes().IndexesWithColumn(columnName) {
   867  		_, err = sch.Indexes().RemoveIndex(index.Name())
   868  		if err != nil {
   869  			return err
   870  		}
   871  		updatedTable, err = updatedTable.DeleteIndexRowData(ctx, index.Name())
   872  		if err != nil {
   873  			return err
   874  		}
   875  	}
   876  
   877  	updatedTable, err = updatedTable.UpdateSchema(ctx, sch)
   878  	if err != nil {
   879  		return err
   880  	}
   881  
   882  	fkCollection, err := root.GetForeignKeyCollection(ctx)
   883  	if err != nil {
   884  		return err
   885  	}
   886  	declaresFk, referencesFk := fkCollection.KeysForTable(t.tableName)
   887  
   888  	updatedTable, err = alterschema.DropColumn(ctx, updatedTable, columnName, append(declaresFk, referencesFk...))
   889  	if err != nil {
   890  		return err
   891  	}
   892  
   893  	newRoot, err := root.PutTable(ctx, t.tableName, updatedTable)
   894  	if err != nil {
   895  		return err
   896  	}
   897  
   898  	return t.setRoot(ctx, newRoot)
   899  }
   900  
   901  // ModifyColumn implements sql.AlterableTable
   902  func (t *AlterableDoltTable) ModifyColumn(ctx *sql.Context, columnName string, column *sql.Column, order *sql.ColumnOrder) error {
   903  	root, err := t.getRoot(ctx)
   904  
   905  	if err != nil {
   906  		return err
   907  	}
   908  
   909  	table, _, err := root.GetTable(ctx, t.tableName)
   910  	if err != nil {
   911  		return err
   912  	}
   913  
   914  	sch, err := table.GetSchema(ctx)
   915  	if err != nil {
   916  		return err
   917  	}
   918  
   919  	existingCol, ok := sch.GetAllCols().GetByNameCaseInsensitive(columnName)
   920  	if !ok {
   921  		panic(fmt.Sprintf("Column %s not found. This is a bug.", columnName))
   922  	}
   923  
   924  	col, err := sqlutil.ToDoltCol(existingCol.Tag, column)
   925  	if err != nil {
   926  		return err
   927  	}
   928  
   929  	fkCollection, err := root.GetForeignKeyCollection(ctx)
   930  	if err != nil {
   931  		return err
   932  	}
   933  	declaresFk, referencedByFk := fkCollection.KeysForTable(t.tableName)
   934  	for _, foreignKey := range declaresFk {
   935  		if (foreignKey.OnUpdate == doltdb.ForeignKeyReferenceOption_SetNull || foreignKey.OnDelete == doltdb.ForeignKeyReferenceOption_SetNull) &&
   936  			col.IsNullable() {
   937  			return fmt.Errorf("foreign key `%s` has SET NULL thus column `%s` cannot be altered to accept null values", foreignKey.Name, col.Name)
   938  		}
   939  	}
   940  
   941  	if !existingCol.TypeInfo.Equals(col.TypeInfo) {
   942  		for _, foreignKey := range declaresFk {
   943  			for _, tag := range foreignKey.TableColumns {
   944  				if tag == existingCol.Tag {
   945  					return fmt.Errorf("cannot alter a column's type when it is used in a foreign key")
   946  				}
   947  			}
   948  		}
   949  		for _, foreignKey := range referencedByFk {
   950  			for _, tag := range foreignKey.ReferencedTableColumns {
   951  				if tag == existingCol.Tag {
   952  					return fmt.Errorf("cannot alter a column's type when it is used in a foreign key")
   953  				}
   954  			}
   955  		}
   956  		if existingCol.Kind != col.Kind { // We only change the tag when the underlying Noms kind changes
   957  			tags, err := root.GenerateTagsForNewColumns(ctx, t.tableName, []string{col.Name}, []types.NomsKind{col.Kind}, nil)
   958  			if err != nil {
   959  				return err
   960  			}
   961  			if len(tags) != 1 {
   962  				return fmt.Errorf("expected a generated tag length of 1")
   963  			}
   964  			col.Tag = tags[0]
   965  		}
   966  	}
   967  
   968  	updatedTable, err := alterschema.ModifyColumn(ctx, table, existingCol, col, orderToOrder(order))
   969  	if err != nil {
   970  		return err
   971  	}
   972  
   973  	newRoot, err := root.PutTable(ctx, t.tableName, updatedTable)
   974  	if err != nil {
   975  		return err
   976  	}
   977  
   978  	return t.setRoot(ctx, newRoot)
   979  }
   980  
   981  // CreateIndex implements sql.IndexAlterableTable
   982  func (t *AlterableDoltTable) CreateIndex(
   983  	ctx *sql.Context,
   984  	indexName string,
   985  	using sql.IndexUsing,
   986  	constraint sql.IndexConstraint,
   987  	columns []sql.IndexColumn,
   988  	comment string,
   989  ) error {
   990  	if schema.IsKeyless(t.sch) {
   991  		return fmt.Errorf("indexes on keyless tables are not supported")
   992  	}
   993  
   994  	table, err := t.doltTable(ctx)
   995  	if err != nil {
   996  		return err
   997  	}
   998  
   999  	ret, err := createIndexForTable(ctx, table, indexName, using, constraint, columns, true, comment)
  1000  	if err != nil {
  1001  		return err
  1002  	}
  1003  	root, err := t.getRoot(ctx)
  1004  	if err != nil {
  1005  		return err
  1006  	}
  1007  	if ret.oldIndex != nil && ret.oldIndex != ret.newIndex { // old index was replaced, so we update foreign keys
  1008  		fkc, err := root.GetForeignKeyCollection(ctx)
  1009  		if err != nil {
  1010  			return err
  1011  		}
  1012  		for _, fk := range fkc.AllKeys() {
  1013  			newFk := fk
  1014  			if t.tableName == fk.TableName && fk.TableIndex == ret.oldIndex.Name() {
  1015  				newFk.TableIndex = ret.newIndex.Name()
  1016  			}
  1017  			if t.tableName == fk.ReferencedTableName && fk.ReferencedTableIndex == ret.oldIndex.Name() {
  1018  				newFk.ReferencedTableIndex = ret.newIndex.Name()
  1019  			}
  1020  			fkc.RemoveKeys(fk)
  1021  			err = fkc.AddKeys(newFk)
  1022  			if err != nil {
  1023  				return err
  1024  			}
  1025  		}
  1026  		root, err = root.PutForeignKeyCollection(ctx, fkc)
  1027  		if err != nil {
  1028  			return err
  1029  		}
  1030  	}
  1031  	newRoot, err := root.PutTable(ctx, t.tableName, ret.newTable)
  1032  	if err != nil {
  1033  		return err
  1034  	}
  1035  
  1036  	err = t.setRoot(ctx, newRoot)
  1037  
  1038  	if err != nil {
  1039  		return err
  1040  	}
  1041  	return t.updateFromRoot(ctx, newRoot)
  1042  }
  1043  
  1044  // DropIndex implements sql.IndexAlterableTable
  1045  func (t *AlterableDoltTable) DropIndex(ctx *sql.Context, indexName string) error {
  1046  	// We disallow removing internal dolt_ tables from SQL directly
  1047  	if strings.HasPrefix(indexName, "dolt_") {
  1048  		return fmt.Errorf("dolt internal indexes may not be dropped")
  1049  	}
  1050  	root, err := t.getRoot(ctx)
  1051  	if err != nil {
  1052  		return err
  1053  	}
  1054  	fkc, err := root.GetForeignKeyCollection(ctx)
  1055  	if err != nil {
  1056  		return err
  1057  	}
  1058  	ourKeys, referencingKeys := fkc.KeysForTable(t.tableName)
  1059  	for _, k := range ourKeys {
  1060  		if k.TableIndex == indexName {
  1061  			return fmt.Errorf("cannot drop index: %s is referenced by foreign key %s",
  1062  				k.TableIndex, k.Name)
  1063  		}
  1064  	}
  1065  	for _, k := range referencingKeys {
  1066  		if k.ReferencedTableIndex == indexName {
  1067  			return fmt.Errorf("cannot drop index: %s is referenced by foreign key %s",
  1068  				k.ReferencedTableIndex, k.Name)
  1069  		}
  1070  	}
  1071  
  1072  	newTable, _, err := t.dropIndex(ctx, indexName)
  1073  	if err != nil {
  1074  		return err
  1075  	}
  1076  	newRoot, err := root.PutTable(ctx, t.tableName, newTable)
  1077  	if err != nil {
  1078  		return err
  1079  	}
  1080  	err = t.setRoot(ctx, newRoot)
  1081  	if err != nil {
  1082  		return err
  1083  	}
  1084  	return t.updateFromRoot(ctx, newRoot)
  1085  }
  1086  
  1087  // RenameIndex implements sql.IndexAlterableTable
  1088  func (t *AlterableDoltTable) RenameIndex(ctx *sql.Context, fromIndexName string, toIndexName string) error {
  1089  	// RenameIndex will error if there is a name collision or an index does not exist
  1090  	_, err := t.sch.Indexes().RenameIndex(fromIndexName, toIndexName)
  1091  	if err != nil {
  1092  		return err
  1093  	}
  1094  
  1095  	table, err := t.doltTable(ctx)
  1096  	if err != nil {
  1097  		return err
  1098  	}
  1099  
  1100  	newTable, err := table.UpdateSchema(ctx, t.sch)
  1101  	if err != nil {
  1102  		return err
  1103  	}
  1104  	newTable, err = newTable.RenameIndexRowData(ctx, fromIndexName, toIndexName)
  1105  	if err != nil {
  1106  		return err
  1107  	}
  1108  
  1109  	root, err := t.getRoot(ctx)
  1110  	if err != nil {
  1111  		return err
  1112  	}
  1113  	newRoot, err := root.PutTable(ctx, t.tableName, newTable)
  1114  	if err != nil {
  1115  		return err
  1116  	}
  1117  
  1118  	err = t.setRoot(ctx, newRoot)
  1119  	if err != nil {
  1120  		return err
  1121  	}
  1122  	return t.updateFromRoot(ctx, newRoot)
  1123  }
  1124  
  1125  // CreateForeignKey implements sql.ForeignKeyAlterableTable
  1126  func (t *AlterableDoltTable) CreateForeignKey(
  1127  	ctx *sql.Context,
  1128  	fkName string,
  1129  	columns []string,
  1130  	refTblName string,
  1131  	refColumns []string,
  1132  	onUpdate, onDelete sql.ForeignKeyReferenceOption) error {
  1133  	if fkName != "" && !doltdb.IsValidForeignKeyName(fkName) {
  1134  		return fmt.Errorf("invalid foreign key name `%s` as it must match the regular expression %s", fkName, doltdb.ForeignKeyNameRegexStr)
  1135  	}
  1136  	isSelfFk := strings.ToLower(t.tableName) == strings.ToLower(refTblName)
  1137  	if isSelfFk {
  1138  		if len(columns) > 1 {
  1139  			return fmt.Errorf("support for self referential composite foreign keys is not yet implemented")
  1140  		}
  1141  	}
  1142  
  1143  	table, err := t.doltTable(ctx)
  1144  	if err != nil {
  1145  		return err
  1146  	}
  1147  
  1148  	tblCols := make([]schema.Column, len(columns))
  1149  	colTags := make([]uint64, len(columns))
  1150  	sqlColNames := make([]sql.IndexColumn, len(columns))
  1151  	for i, col := range columns {
  1152  		tableCol, ok := t.sch.GetAllCols().GetByNameCaseInsensitive(col)
  1153  		if !ok {
  1154  			//TODO: fix go-mysql-server equivalent check, needs two vals
  1155  			return fmt.Errorf("table `%s` does not have column `%s`", t.tableName, col)
  1156  		}
  1157  		if (onUpdate == sql.ForeignKeyReferenceOption_SetNull || onDelete == sql.ForeignKeyReferenceOption_SetNull) &&
  1158  			!tableCol.IsNullable() {
  1159  			return fmt.Errorf("cannot use SET NULL as column `%s` is non-nullable", tableCol.Name)
  1160  		}
  1161  		tblCols[i] = tableCol
  1162  		colTags[i] = tableCol.Tag
  1163  		sqlColNames[i] = sql.IndexColumn{
  1164  			Name:   tableCol.Name,
  1165  			Length: 0,
  1166  		}
  1167  	}
  1168  
  1169  	root, err := t.getRoot(ctx)
  1170  	if err != nil {
  1171  		return err
  1172  	}
  1173  	var refTbl *doltdb.Table
  1174  	var ok bool
  1175  	var refSch schema.Schema
  1176  	if isSelfFk {
  1177  		refTbl = table
  1178  		refSch = t.sch
  1179  	} else {
  1180  		refTbl, _, ok, err = root.GetTableInsensitive(ctx, refTblName)
  1181  		if err != nil {
  1182  			return err
  1183  		}
  1184  		if !ok {
  1185  			return fmt.Errorf("referenced table `%s` does not exist", refTblName)
  1186  		}
  1187  		refSch, err = refTbl.GetSchema(ctx)
  1188  		if err != nil {
  1189  			return err
  1190  		}
  1191  	}
  1192  
  1193  	refColTags := make([]uint64, len(refColumns))
  1194  	for i, name := range refColumns {
  1195  		refCol, ok := refSch.GetAllCols().GetByNameCaseInsensitive(name)
  1196  		if !ok {
  1197  			return fmt.Errorf("table `%s` does not have column `%s`", refTblName, name)
  1198  		}
  1199  		if !tblCols[i].TypeInfo.Equals(refCol.TypeInfo) {
  1200  			return fmt.Errorf("column type mismatch on `%s` and `%s`", columns[i], refCol.Name)
  1201  		}
  1202  		sqlParserType := refCol.TypeInfo.ToSqlType().Type()
  1203  		if sqlParserType == sqltypes.Blob || sqlParserType == sqltypes.Text {
  1204  			return fmt.Errorf("TEXT/BLOB are not valid types for foreign keys")
  1205  		}
  1206  		refColTags[i] = refCol.Tag
  1207  	}
  1208  
  1209  	if isSelfFk {
  1210  		for i := range colTags {
  1211  			if colTags[i] == refColTags[i] {
  1212  				return fmt.Errorf("the same column `%s` cannot be used in self referential foreign keys", tblCols[i].Name)
  1213  			}
  1214  		}
  1215  	}
  1216  
  1217  	onUpdateRefOp, err := parseFkReferenceOption(onUpdate)
  1218  	if err != nil {
  1219  		return err
  1220  	}
  1221  	onDeleteRefOp, err := parseFkReferenceOption(onDelete)
  1222  	if err != nil {
  1223  		return err
  1224  	}
  1225  
  1226  	tableIndex, ok := t.sch.Indexes().GetIndexByTags(colTags...)
  1227  	if !ok {
  1228  		// if child index doesn't exist, create it
  1229  		ret, err := createIndexForTable(ctx, table, "", sql.IndexUsing_Default, sql.IndexConstraint_None, sqlColNames, false, "")
  1230  		if err != nil {
  1231  			return err
  1232  		}
  1233  
  1234  		table = ret.newTable
  1235  		tableIndex = ret.newIndex
  1236  		root, err = root.PutTable(ctx, t.tableName, table)
  1237  		if err != nil {
  1238  			return err
  1239  		}
  1240  		if isSelfFk {
  1241  			refTbl = table
  1242  		}
  1243  	}
  1244  
  1245  	refTableIndex, ok := refSch.Indexes().GetIndexByTags(refColTags...)
  1246  	if !ok {
  1247  		parentPKs := set.NewUint64Set(refSch.GetPKCols().Tags)
  1248  		if parentPKs.ContainsAll(refColTags) {
  1249  			// special exception for parent table primary keys
  1250  			// todo: make clustered PK index usable as parent table FK index
  1251  			var colNames []sql.IndexColumn
  1252  			for _, t := range refColTags {
  1253  				c, _ := refSch.GetAllCols().GetByTag(t)
  1254  				colNames = append(colNames, sql.IndexColumn{Name: c.Name})
  1255  			}
  1256  			ret, err := createIndexForTable(ctx, refTbl, "", sql.IndexUsing_Default, sql.IndexConstraint_None, colNames, false, "")
  1257  			if err != nil {
  1258  				return err
  1259  			}
  1260  			refTbl = ret.newTable
  1261  			refTableIndex = ret.newIndex
  1262  			root, err = root.PutTable(ctx, refTblName, refTbl)
  1263  			if err != nil {
  1264  				return err
  1265  			}
  1266  		} else {
  1267  			// parent index must exist
  1268  			return fmt.Errorf("missing index for constraint '%s' in the referenced table '%s'", fkName, refTblName)
  1269  		}
  1270  	}
  1271  
  1272  	foreignKeyCollection, err := root.GetForeignKeyCollection(ctx)
  1273  	if err != nil {
  1274  		return err
  1275  	}
  1276  	foreignKey := doltdb.ForeignKey{
  1277  		Name:                   fkName,
  1278  		TableName:              t.tableName,
  1279  		TableIndex:             tableIndex.Name(),
  1280  		TableColumns:           colTags,
  1281  		ReferencedTableName:    refTblName,
  1282  		ReferencedTableIndex:   refTableIndex.Name(),
  1283  		ReferencedTableColumns: refColTags,
  1284  		OnUpdate:               onUpdateRefOp,
  1285  		OnDelete:               onDeleteRefOp,
  1286  	}
  1287  	err = foreignKeyCollection.AddKeys(foreignKey)
  1288  	if err != nil {
  1289  		return err
  1290  	}
  1291  	newRoot, err := root.PutForeignKeyCollection(ctx, foreignKeyCollection)
  1292  	if err != nil {
  1293  		return err
  1294  	}
  1295  
  1296  	tableIndexData, err := table.GetIndexRowData(ctx, tableIndex.Name())
  1297  	if err != nil {
  1298  		return err
  1299  	}
  1300  	refTableIndexData, err := refTbl.GetIndexRowData(ctx, refTableIndex.Name())
  1301  	if err != nil {
  1302  		return err
  1303  	}
  1304  	err = foreignKey.ValidateData(ctx, tableIndexData, refTableIndexData, tableIndex, refTableIndex)
  1305  	if err != nil {
  1306  		return err
  1307  	}
  1308  
  1309  	err = t.setRoot(ctx, newRoot)
  1310  	if err != nil {
  1311  		return err
  1312  	}
  1313  	return t.updateFromRoot(ctx, newRoot)
  1314  }
  1315  
  1316  // DropForeignKey implements sql.ForeignKeyAlterableTable
  1317  func (t *AlterableDoltTable) DropForeignKey(ctx *sql.Context, fkName string) error {
  1318  	root, err := t.getRoot(ctx)
  1319  	if err != nil {
  1320  		return err
  1321  	}
  1322  	fkc, err := root.GetForeignKeyCollection(ctx)
  1323  	if err != nil {
  1324  		return err
  1325  	}
  1326  	err = fkc.RemoveKeyByName(fkName)
  1327  	if err != nil {
  1328  		return err
  1329  	}
  1330  	newRoot, err := root.PutForeignKeyCollection(ctx, fkc)
  1331  	if err != nil {
  1332  		return err
  1333  	}
  1334  
  1335  	err = t.setRoot(ctx, newRoot)
  1336  	if err != nil {
  1337  		return err
  1338  	}
  1339  	return t.updateFromRoot(ctx, newRoot)
  1340  }
  1341  
  1342  func toForeignKeyConstraint(fk doltdb.ForeignKey, childSch, parentSch schema.Schema) (cst sql.ForeignKeyConstraint, err error) {
  1343  	cst = sql.ForeignKeyConstraint{
  1344  		Name:              fk.Name,
  1345  		Columns:           make([]string, len(fk.TableColumns)),
  1346  		ReferencedTable:   fk.ReferencedTableName,
  1347  		ReferencedColumns: make([]string, len(fk.ReferencedTableColumns)),
  1348  		OnUpdate:          toReferenceOption(fk.OnUpdate),
  1349  		OnDelete:          toReferenceOption(fk.OnDelete),
  1350  	}
  1351  
  1352  	for i, tag := range fk.TableColumns {
  1353  		c, ok := childSch.GetAllCols().GetByTag(tag)
  1354  		if !ok {
  1355  			return cst, fmt.Errorf("cannot find column for tag %d "+
  1356  				"in table %s used in foreign key %s", tag, fk.TableName, fk.Name)
  1357  		}
  1358  		cst.Columns[i] = c.Name
  1359  	}
  1360  
  1361  	for i, tag := range fk.ReferencedTableColumns {
  1362  		c, ok := parentSch.GetAllCols().GetByTag(tag)
  1363  		if !ok {
  1364  			return cst, fmt.Errorf("cannot find column for tag %d "+
  1365  				"in table %s used in foreign key %s", tag, fk.ReferencedTableName, fk.Name)
  1366  		}
  1367  		cst.ReferencedColumns[i] = c.Name
  1368  
  1369  	}
  1370  
  1371  	return cst, nil
  1372  }
  1373  
  1374  func toReferenceOption(opt doltdb.ForeignKeyReferenceOption) sql.ForeignKeyReferenceOption {
  1375  	switch opt {
  1376  	case doltdb.ForeignKeyReferenceOption_DefaultAction:
  1377  		return sql.ForeignKeyReferenceOption_DefaultAction
  1378  	case doltdb.ForeignKeyReferenceOption_Cascade:
  1379  		return sql.ForeignKeyReferenceOption_Cascade
  1380  	case doltdb.ForeignKeyReferenceOption_NoAction:
  1381  		return sql.ForeignKeyReferenceOption_NoAction
  1382  	case doltdb.ForeignKeyReferenceOption_Restrict:
  1383  		return sql.ForeignKeyReferenceOption_Restrict
  1384  	case doltdb.ForeignKeyReferenceOption_SetNull:
  1385  		return sql.ForeignKeyReferenceOption_SetNull
  1386  	default:
  1387  		panic(fmt.Sprintf("Unhandled foreign key reference option %v", opt))
  1388  	}
  1389  }
  1390  
  1391  func parseFkReferenceOption(refOp sql.ForeignKeyReferenceOption) (doltdb.ForeignKeyReferenceOption, error) {
  1392  	switch refOp {
  1393  	case sql.ForeignKeyReferenceOption_DefaultAction:
  1394  		return doltdb.ForeignKeyReferenceOption_DefaultAction, nil
  1395  	case sql.ForeignKeyReferenceOption_Restrict:
  1396  		return doltdb.ForeignKeyReferenceOption_Restrict, nil
  1397  	case sql.ForeignKeyReferenceOption_Cascade:
  1398  		return doltdb.ForeignKeyReferenceOption_Cascade, nil
  1399  	case sql.ForeignKeyReferenceOption_NoAction:
  1400  		return doltdb.ForeignKeyReferenceOption_NoAction, nil
  1401  	case sql.ForeignKeyReferenceOption_SetNull:
  1402  		return doltdb.ForeignKeyReferenceOption_SetNull, nil
  1403  	case sql.ForeignKeyReferenceOption_SetDefault:
  1404  		return doltdb.ForeignKeyReferenceOption_DefaultAction, fmt.Errorf(`"SET DEFAULT" is not supported`)
  1405  	default:
  1406  		return doltdb.ForeignKeyReferenceOption_DefaultAction, fmt.Errorf("unknown foreign key reference option: %v", refOp)
  1407  	}
  1408  }
  1409  
  1410  type createIndexReturn struct {
  1411  	newTable *doltdb.Table
  1412  	sch      schema.Schema
  1413  	oldIndex schema.Index
  1414  	newIndex schema.Index
  1415  }
  1416  
  1417  // createIndex creates the given index on the given table with the given schema. Returns the updated table, updated schema, and created index.
  1418  func createIndexForTable(
  1419  	ctx *sql.Context,
  1420  	table *doltdb.Table,
  1421  	indexName string,
  1422  	using sql.IndexUsing,
  1423  	constraint sql.IndexConstraint,
  1424  	columns []sql.IndexColumn,
  1425  	isUserDefined bool,
  1426  	comment string,
  1427  ) (*createIndexReturn, error) {
  1428  	if constraint != sql.IndexConstraint_None && constraint != sql.IndexConstraint_Unique {
  1429  		return nil, fmt.Errorf("not yet supported")
  1430  	}
  1431  
  1432  	sch, err := table.GetSchema(ctx)
  1433  	if err != nil {
  1434  		return nil, err
  1435  	}
  1436  
  1437  	// get the real column names as CREATE INDEX columns are case-insensitive
  1438  	var realColNames []string
  1439  	allTableCols := sch.GetAllCols()
  1440  	for _, indexCol := range columns {
  1441  		tableCol, ok := allTableCols.GetByNameCaseInsensitive(indexCol.Name)
  1442  		if !ok {
  1443  			return nil, fmt.Errorf("column `%s` does not exist for the table", indexCol.Name)
  1444  		}
  1445  		realColNames = append(realColNames, tableCol.Name)
  1446  	}
  1447  
  1448  	if indexName == "" {
  1449  		indexName = strings.Join(realColNames, "")
  1450  		_, ok := sch.Indexes().GetByNameCaseInsensitive(indexName)
  1451  		var i int
  1452  		for ok {
  1453  			i++
  1454  			indexName = fmt.Sprintf("%s_%d", strings.Join(realColNames, ""), i)
  1455  			_, ok = sch.Indexes().GetByNameCaseInsensitive(indexName)
  1456  		}
  1457  	}
  1458  	if !doltdb.IsValidIndexName(indexName) {
  1459  		return nil, fmt.Errorf("invalid index name `%s` as they must match the regular expression %s", indexName, doltdb.IndexNameRegexStr)
  1460  	}
  1461  
  1462  	// if an index was already created for the column set but was not generated by the user then we replace it
  1463  	replacingIndex := false
  1464  	existingIndex, ok := sch.Indexes().GetIndexByColumnNames(realColNames...)
  1465  	if ok && !existingIndex.IsUserDefined() {
  1466  		replacingIndex = true
  1467  		_, err = sch.Indexes().RemoveIndex(existingIndex.Name())
  1468  		if err != nil {
  1469  			return nil, err
  1470  		}
  1471  	}
  1472  
  1473  	// create the index metadata, will error if index names are taken or an index with the same columns in the same order exists
  1474  	index, err := sch.Indexes().AddIndexByColNames(
  1475  		indexName,
  1476  		realColNames,
  1477  		schema.IndexProperties{
  1478  			IsUnique:      constraint == sql.IndexConstraint_Unique,
  1479  			IsUserDefined: isUserDefined,
  1480  			Comment:       comment,
  1481  		},
  1482  	)
  1483  	if err != nil {
  1484  		return nil, err
  1485  	}
  1486  
  1487  	// update the table schema with the new index
  1488  	newTable, err := table.UpdateSchema(ctx, sch)
  1489  	if err != nil {
  1490  		return nil, err
  1491  	}
  1492  
  1493  	if replacingIndex { // verify that the pre-existing index data is valid
  1494  		newTable, err = newTable.RenameIndexRowData(ctx, existingIndex.Name(), index.Name())
  1495  		if err != nil {
  1496  			return nil, err
  1497  		}
  1498  		err = newTable.VerifyIndexRowData(ctx, index.Name())
  1499  		if err != nil {
  1500  			return nil, err
  1501  		}
  1502  	} else { // set the index row data and get a new root with the updated table
  1503  		indexRowData, err := editor.RebuildIndex(ctx, newTable, index.Name())
  1504  		if err != nil {
  1505  			return nil, err
  1506  		}
  1507  		newTable, err = newTable.SetIndexRowData(ctx, index.Name(), indexRowData)
  1508  		if err != nil {
  1509  			return nil, err
  1510  		}
  1511  	}
  1512  	return &createIndexReturn{
  1513  		newTable: newTable,
  1514  		sch:      sch,
  1515  		oldIndex: existingIndex,
  1516  		newIndex: index,
  1517  	}, nil
  1518  }
  1519  
  1520  // dropIndex drops the given index on the given table with the given schema. Returns the updated table and updated schema.
  1521  func (t *AlterableDoltTable) dropIndex(ctx *sql.Context, indexName string) (*doltdb.Table, schema.Schema, error) {
  1522  	// RemoveIndex returns an error if the index does not exist, no need to do twice
  1523  	_, err := t.sch.Indexes().RemoveIndex(indexName)
  1524  	if err != nil {
  1525  		return nil, nil, err
  1526  	}
  1527  
  1528  	table, err := t.doltTable(ctx)
  1529  	if err != nil {
  1530  		return nil, nil, err
  1531  	}
  1532  
  1533  	newTable, err := table.UpdateSchema(ctx, t.sch)
  1534  	if err != nil {
  1535  		return nil, nil, err
  1536  	}
  1537  	newTable, err = newTable.DeleteIndexRowData(ctx, indexName)
  1538  	if err != nil {
  1539  		return nil, nil, err
  1540  	}
  1541  	tblSch, err := newTable.GetSchema(ctx)
  1542  	if err != nil {
  1543  		return nil, nil, err
  1544  	}
  1545  	return newTable, tblSch, nil
  1546  }
  1547  
  1548  func (t *AlterableDoltTable) updateFromRoot(ctx *sql.Context, root *doltdb.RootValue) error {
  1549  	updatedTableSql, ok, err := t.db.getTable(ctx, root, t.tableName, t.temporary)
  1550  	if err != nil {
  1551  		return err
  1552  	}
  1553  	if !ok {
  1554  		return fmt.Errorf("table `%s` cannot find itself", t.tableName)
  1555  	}
  1556  	var updatedTable *AlterableDoltTable
  1557  	if doltdb.HasDoltPrefix(t.tableName) && !doltdb.IsReadOnlySystemTable(t.tableName) {
  1558  		updatedTable = &AlterableDoltTable{*updatedTableSql.(*WritableDoltTable)}
  1559  	} else {
  1560  		updatedTable = updatedTableSql.(*AlterableDoltTable)
  1561  	}
  1562  	t.WritableDoltTable.DoltTable = updatedTable.WritableDoltTable.DoltTable
  1563  	return nil
  1564  }
  1565  
  1566  func (t *AlterableDoltTable) CreateCheck(ctx *sql.Context, check *sql.CheckDefinition) error {
  1567  	sch := t.sch
  1568  
  1569  	check = &(*check)
  1570  	if check.Name == "" {
  1571  		var err error
  1572  		check.Name, err = t.generateCheckName(ctx, check)
  1573  		if err != nil {
  1574  			return err
  1575  		}
  1576  	}
  1577  
  1578  	_, err := sch.Checks().AddCheck(check.Name, check.CheckExpression, check.Enforced)
  1579  	if err != nil {
  1580  		return err
  1581  	}
  1582  
  1583  	table, err := t.doltTable(ctx)
  1584  	if err != nil {
  1585  		return err
  1586  	}
  1587  
  1588  	newTable, err := table.UpdateSchema(ctx, sch)
  1589  	if err != nil {
  1590  		return err
  1591  	}
  1592  
  1593  	root, err := t.getRoot(ctx)
  1594  	if err != nil {
  1595  		return err
  1596  	}
  1597  
  1598  	newRoot, err := root.PutTable(ctx, t.tableName, newTable)
  1599  	if err != nil {
  1600  		return err
  1601  	}
  1602  
  1603  	err = t.setRoot(ctx, newRoot)
  1604  	if err != nil {
  1605  		return err
  1606  	}
  1607  
  1608  	return t.updateFromRoot(ctx, newRoot)
  1609  }
  1610  
  1611  func (t *AlterableDoltTable) DropCheck(ctx *sql.Context, chName string) error {
  1612  	sch := t.sch
  1613  
  1614  	err := sch.Checks().DropCheck(chName)
  1615  	if err != nil {
  1616  		return err
  1617  	}
  1618  
  1619  	table, err := t.doltTable(ctx)
  1620  	if err != nil {
  1621  		return err
  1622  	}
  1623  
  1624  	newTable, err := table.UpdateSchema(ctx, sch)
  1625  	if err != nil {
  1626  		return err
  1627  	}
  1628  
  1629  	root, err := t.getRoot(ctx)
  1630  	if err != nil {
  1631  		return err
  1632  	}
  1633  
  1634  	newRoot, err := root.PutTable(ctx, t.tableName, newTable)
  1635  	if err != nil {
  1636  		return err
  1637  	}
  1638  
  1639  	err = t.setRoot(ctx, newRoot)
  1640  	if err != nil {
  1641  		return err
  1642  	}
  1643  
  1644  	return t.updateFromRoot(ctx, newRoot)
  1645  }
  1646  
  1647  func (t *AlterableDoltTable) generateCheckName(ctx *sql.Context, check *sql.CheckDefinition) (string, error) {
  1648  	var bb bytes.Buffer
  1649  	bb.Write([]byte(check.CheckExpression))
  1650  	hash := hash.Of(bb.Bytes())
  1651  
  1652  	hashedName := fmt.Sprintf("chk_%s", hash.String()[:8])
  1653  	name := hashedName
  1654  
  1655  	var i int
  1656  	for {
  1657  		exists, err := t.constraintNameExists(ctx, name)
  1658  		if err != nil {
  1659  			return "", err
  1660  		}
  1661  		if !exists {
  1662  			break
  1663  		}
  1664  
  1665  		name = fmt.Sprintf("%s_%d", hashedName, i)
  1666  		i++
  1667  	}
  1668  
  1669  	return name, nil
  1670  }
  1671  
  1672  func (t *AlterableDoltTable) constraintNameExists(ctx *sql.Context, name string) (bool, error) {
  1673  	keys, err := t.GetForeignKeys(ctx)
  1674  	if err != nil {
  1675  		return false, err
  1676  	}
  1677  
  1678  	for _, key := range keys {
  1679  		if strings.ToLower(key.Name) == strings.ToLower(name) {
  1680  			return true, nil
  1681  		}
  1682  	}
  1683  
  1684  	checks, err := t.GetChecks(ctx)
  1685  	if err != nil {
  1686  		return false, err
  1687  	}
  1688  
  1689  	for _, check := range checks {
  1690  		if strings.ToLower(check.Name) == strings.ToLower(name) {
  1691  			return true, nil
  1692  		}
  1693  	}
  1694  
  1695  	return false, nil
  1696  }