github.com/hyperledger/burrow@v0.34.5-0.20220512172541-77f09336001d/vent/sqlsol/projection.go

package sqlsol

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/hyperledger/burrow/vent/types"
	"github.com/pkg/errors"
	"github.com/xeipuuv/gojsonschema"
)

// Projection contains EventTable, Event & Abi specifications
type Projection struct {
	Tables types.EventTables
	Spec   types.ProjectionSpec
}

// NewProjectionFromBytes creates a Projection from a stream of bytes
func NewProjectionFromBytes(bs []byte) (*Projection, error) {
	spec := types.ProjectionSpec{}

	err := ValidateJSONSpec(bs)
	if err != nil {
		return nil, err
	}

	err = json.Unmarshal(bs, &spec)
	if err != nil {
		return nil, errors.Wrap(err, "Error unmarshalling spec")
	}

	return NewProjection(spec)
}

// NewProjectionFromFolder creates a Projection from a folder containing spec files
func NewProjectionFromFolder(specFileOrDirs ...string) (*Projection, error) {
	spec := types.ProjectionSpec{}

	const errHeader = "NewProjectionFromFolder():"

	for _, dir := range specFileOrDirs {
		err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error {
			if err != nil {
				return fmt.Errorf("error walking event spec files location '%s': %v", dir, err)
			}
			if filepath.Ext(path) == ".json" {
				bs, err := readFile(path)
				if err != nil {
					return fmt.Errorf("error reading spec file '%s': %v", path, err)
				}

				err = ValidateJSONSpec(bs)
				if err != nil {
					return fmt.Errorf("could not validate spec file '%s': %v", path, err)
				}

				fileEventSpec := types.ProjectionSpec{}
				err = json.Unmarshal(bs, &fileEventSpec)
				if err != nil {
					return fmt.Errorf("error reading spec file '%s': %v", path, err)
				}

				spec = append(spec, fileEventSpec...)
			}

			return nil
		})
		if err != nil {
			return nil, fmt.Errorf("%s %v", errHeader, err)
		}
	}

	return NewProjection(spec)
}

// Takes a sqlsol event specification
// and returns a pointer to a filled projection structure
// that contains event types mapped to SQL column types
// and Event tables structures with table and columns info
func NewProjection(spec types.ProjectionSpec) (*Projection, error) {
	// builds abi information from specification
	tables := make(types.EventTables)

	for _, eventClass := range spec {
		// validate json structure
		if err := eventClass.Validate(); err != nil {
			return nil, fmt.Errorf("validation error on %v: %v", eventClass, err)
		}

		// build columns mapping
		var columns []*types.SQLTableColumn
		channels := make(map[string][]string)

		// do we have a primary key
		primary := false
		for _, mapping := range eventClass.FieldMappings {
			if mapping.Primary {
				primary = true
				break
			}
		}

		if !primary && eventClass.DeleteMarkerField != "" {
			return nil, fmt.Errorf("no DeleteMarkerField allowed if no primary key on %v", eventClass)
		}

		// Add the global mappings
		if primary {
			eventClass.FieldMappings = append(getGlobalFieldMappings(), eventClass.FieldMappings...)
		} else {
			eventClass.FieldMappings = append(getGlobalFieldMappingsLogMode(), eventClass.FieldMappings...)
		}

		i := 0
		for _, mapping := range eventClass.FieldMappings {
			var bytesMapping BytesMapping
			if mapping.BytesToHex {
				bytesMapping = BytesToHex
			} else if mapping.BytesToString {
				bytesMapping = BytesToString
			}
			sqlType, sqlTypeLength, err := getSQLType(mapping.Type, bytesMapping)
			if err != nil {
				return nil, err
			}

			i++

			// Update channels broadcast payload subsets with this column
			for _, channel := range mapping.Notify {
				channels[channel] = append(channels[channel], mapping.ColumnName)
			}

			columns = append(columns, &types.SQLTableColumn{
				Name:    mapping.ColumnName,
				Type:    sqlType,
				Primary: mapping.Primary,
				Length:  sqlTypeLength,
			})
		}

		// Allow for compatible composition of tables
		var err error
		tables[eventClass.TableName], err = mergeTables(tables[eventClass.TableName],
			&types.SQLTable{
				Name:           eventClass.TableName,
				NotifyChannels: channels,
				Columns:        columns,
			})
		if err != nil {
			return nil, err
		}

	}

	// check if there are duplicated duplicated column names (for a given table)
	colName := make(map[string]int)

	for _, table := range tables {
		for _, column := range table.Columns {
			colName[table.Name+column.Name]++
			if colName[table.Name+column.Name] > 1 {
				return nil, fmt.Errorf("duplicated column name: '%s' in table '%s'", column.Name, table.Name)
			}
		}
	}

	return &Projection{
		Tables: tables,
		Spec:   spec,
	}, nil
}

// Get the column for a particular table and column name
func (p *Projection) GetColumn(tableName, columnName string) (*types.SQLTableColumn, error) {
	if table, ok := p.Tables[tableName]; ok {
		column := table.GetColumn(columnName)
		if column == nil {
			return nil, fmt.Errorf("GetColumn: table '%s' has no column '%s'",
				tableName, columnName)
		}
		return column, nil
	}

	return nil, fmt.Errorf("GetColumn: table does not exist projection: %s ", tableName)
}

func ValidateJSONSpec(bs []byte) error {
	schemaLoader := gojsonschema.NewGoLoader(types.ProjectionSpecSchema())
	specLoader := gojsonschema.NewBytesLoader(bs)
	result, err := gojsonschema.Validate(schemaLoader, specLoader)
	if err != nil {
		return fmt.Errorf("could not validate using JSONSchema: %v", err)
	}

	if !result.Valid() {
		errs := make([]string, len(result.Errors()))
		for i, err := range result.Errors() {
			errs[i] = err.String()
		}
		return fmt.Errorf("ProjectionSpec failed JSONSchema validation:\n%s", strings.Join(errs, "\n"))
	}
	return nil
}

// readFile opens a given file and reads it contents into a stream of bytes
func readFile(file string) ([]byte, error) {
	theFile, err := os.Open(file)
	if err != nil {
		return nil, err
	}
	defer theFile.Close()

	byteValue, err := ioutil.ReadAll(theFile)
	if err != nil {
		return nil, err
	}

	return byteValue, nil
}

type BytesMapping int

const (
	BytesToBytes = iota
	BytesToString
	BytesToHex
)

// getSQLType maps event input types with corresponding SQL column types
// takes into account related solidity types info and element indexed or hashed
func getSQLType(evmSignature string, bytesMapping BytesMapping) (types.SQLColumnType, int, error) {
	evmSignature = strings.ToLower(evmSignature)
	re := regexp.MustCompile("[0-9]+")
	typeSize, _ := strconv.Atoi(re.FindString(evmSignature))

	switch {
	// solidity address => sql varchar
	case evmSignature == types.EventFieldTypeAddress:
		return types.SQLColumnTypeVarchar, 40, nil
		// solidity bool => sql bool
	case evmSignature == types.EventFieldTypeBool:
		return types.SQLColumnTypeBool, 0, nil
		// solidity bytes => sql bytes
		// bytesToString == true means there is a string in there so => sql varchar
	case strings.HasPrefix(evmSignature, types.EventFieldTypeBytes):
		switch bytesMapping {
		case BytesToString:
			return types.SQLColumnTypeVarchar, 32, nil
		case BytesToHex:
			return types.SQLColumnTypeVarchar, 64, nil
		default:
			return types.SQLColumnTypeByteA, 0, nil
		}
		// solidity string => sql text
	case evmSignature == types.EventFieldTypeString:
		return types.SQLColumnTypeText, 0, nil

	case strings.HasPrefix(evmSignature, types.EventFieldTypeInt):
		return evmIntegerSizeToSqlType(typeSize, true), 0, nil

	case strings.HasPrefix(evmSignature, types.EventFieldTypeUInt):
		return evmIntegerSizeToSqlType(typeSize, false), 0, nil
	default:
		return -1, 0, fmt.Errorf("do not know how to map evmSignature: %s ", evmSignature)
	}
}

func evmIntegerSizeToSqlType(size int, signed bool) types.SQLColumnType {
	// Unsized ints default to 256-bit
	if size == 0 {
		return types.SQLColumnTypeNumeric
	}
	// since SQL integers are signed we need an extra bit headroom so high-order bit of a uint isn't interpreted as
	// the sign bit
	if !signed {
		size++
	}
	switch {
	case size <= 32:
		return types.SQLColumnTypeInt
	case size <= 64:
		return types.SQLColumnTypeBigInt
	}
	return types.SQLColumnTypeNumeric
}

// getGlobalColumns returns global columns for event table structures,
// these columns will be part of every SQL event table to relate data with source events
func getGlobalFieldMappings() []*types.EventFieldMapping {
	return []*types.EventFieldMapping{
		{
			ColumnName: columns.ChainID,
			Field:      types.ChainIDLabel,
			Type:       types.EventFieldTypeString,
		},
		{
			ColumnName: columns.Height,
			Field:      types.BlockHeightLabel,
			Type:       types.EventFieldTypeUInt,
		},
		{
			ColumnName: columns.TxIndex,
			Field:      types.TxIndexLabel,
			Type:       types.EventFieldTypeUInt,
		},
		{
			ColumnName: columns.EventIndex,
			Field:      types.EventIndexLabel,
			Type:       types.EventFieldTypeUInt,
		},
		{
			ColumnName: columns.TxHash,
			Field:      types.TxTxHashLabel,
			Type:       types.EventFieldTypeString,
		},
		{
			ColumnName: columns.EventType,
			Field:      types.EventTypeLabel,
			Type:       types.EventFieldTypeString,
		},
		{
			ColumnName: columns.EventName,
			Field:      types.EventNameLabel,
			Type:       types.EventFieldTypeString,
		},
	}
}

// getGlobalColumns returns global columns for event table structures,
// these columns will be part of every SQL event table to relate data with source events
func getGlobalFieldMappingsLogMode() []*types.EventFieldMapping {
	return []*types.EventFieldMapping{
		{
			ColumnName: columns.ChainID,
			Field:      types.ChainIDLabel,
			Type:       types.EventFieldTypeString,
			Primary:    true,
		},
		{
			ColumnName: columns.Height,
			Field:      types.BlockHeightLabel,
			Type:       types.EventFieldTypeUInt,
			Primary:    true,
		},
		{
			ColumnName: columns.TxIndex,
			Field:      types.TxIndexLabel,
			Type:       types.EventFieldTypeUInt,
			Primary:    true,
		},
		{
			ColumnName: columns.EventIndex,
			Field:      types.EventIndexLabel,
			Type:       types.EventFieldTypeUInt,
			Primary:    true,
		},
		{
			ColumnName: columns.TxHash,
			Field:      types.TxTxHashLabel,
			Type:       types.EventFieldTypeString,
		},
		{
			ColumnName: columns.EventType,
			Field:      types.EventTypeLabel,
			Type:       types.EventFieldTypeString,
		},
		{
			ColumnName: columns.EventName,
			Field:      types.EventNameLabel,
			Type:       types.EventFieldTypeString,
		},
	}
}

// Merges tables a and b provided the intersection of their columns (by name) are identical
func mergeTables(tables ...*types.SQLTable) (*types.SQLTable, error) {
	table := &types.SQLTable{
		NotifyChannels: make(map[string][]string),
	}

	columns := make(map[string]*types.SQLTableColumn)
	notifications := make(map[string]map[string]struct{})

	for _, t := range tables {
		if t != nil {
			table.Name = t.Name
			for _, columnB := range t.Columns {
				if columnA, ok := columns[columnB.Name]; ok {
					if !columnA.Equals(columnB) {
						return nil, fmt.Errorf("cannot merge event class tables for %s because of "+
							"conflicting columns: %v and %v", t.Name, columnA, columnB)
					}
					// Just keep existing column from A - they match
				} else {
					// Add as new column
					table.Columns = append(table.Columns, columnB)
					columns[columnB.Name] = columnB
				}
			}
			for channel, columnNames := range t.NotifyChannels {
				for _, columnName := range columnNames {
					if notifications[channel] == nil {
						notifications[channel] = make(map[string]struct{})
					}
					notifications[channel][columnName] = struct{}{}
				}
			}
		}
	}

	// Merge notification channels requested by specs
	for channel, colMap := range notifications {
		for columnName := range colMap {
			table.NotifyChannels[channel] = append(table.NotifyChannels[channel], columnName)
		}
		sort.Strings(table.NotifyChannels[channel])
	}

	return table, nil
}