github.com/kyma-incubator/compass/components/director@v0.0.0-20230623144113-d764f56ff805/internal/tenantfetchersvc/resync/tenant_manager.go

package resync

import (
	"context"
	"fmt"
	"math"
	"strconv"
	"sync"
	"time"

	"github.com/avast/retry-go/v4"
	"github.com/kyma-incubator/compass/components/director/internal/model"
	"github.com/kyma-incubator/compass/components/director/pkg/apperrors"
	"github.com/kyma-incubator/compass/components/director/pkg/graphql"
	"github.com/kyma-incubator/compass/components/director/pkg/log"
	tenantpkg "github.com/kyma-incubator/compass/components/director/pkg/tenant"
	"github.com/pkg/errors"
	"github.com/tidwall/gjson"
)

// EventAPIClient missing godoc
//
//go:generate mockery --name=EventAPIClient --output=automock --outpkg=automock --case=underscore --disable-version-string
type EventAPIClient interface {
	FetchTenantEventsPage(ctx context.Context, eventsType EventsType, additionalQueryParams QueryParams) (*EventsPage, error)
}

// TenantConverter expects tenant converter implementation
//
//go:generate mockery --name=TenantConverter --output=automock --outpkg=automock --case=underscore --disable-version-string
type TenantConverter interface {
	MultipleInputToGraphQLInput([]model.BusinessTenantMappingInput) []graphql.BusinessTenantMappingInput
	ToGraphQLInput(model.BusinessTenantMappingInput) graphql.BusinessTenantMappingInput
}

// DirectorGraphQLClient expects graphql implementation
//
//go:generate mockery --name=DirectorGraphQLClient --output=automock --outpkg=automock --case=underscore --disable-version-string
type DirectorGraphQLClient interface {
	WriteTenants(context.Context, []graphql.BusinessTenantMappingInput) error
	DeleteTenants(ctx context.Context, tenants []graphql.BusinessTenantMappingInput) error
	UpdateTenant(ctx context.Context, id string, tenant graphql.BusinessTenantMappingInput) error
}

type supportedEvents struct {
	createdTenantEvent EventsType
	updatedTenantEvent EventsType
	deletedTenantEvent EventsType
}

type externalTenantsManager struct {
	config         EventsConfig
	tenantProvider string

	gqlClient      DirectorGraphQLClient
	eventAPIClient EventAPIClient

	tenantConverter TenantConverter
}

// TenantsManager is responsible for creating, updating and deleting tenants associated with an external tenants registry
type TenantsManager struct {
	externalTenantsManager

	regionalClients     map[string]EventAPIClient
	supportedEventTypes supportedEvents
}

// NewTenantsManager returns a new tenants manager built with the provided configurations and API clients
func NewTenantsManager(jobConfig JobConfig, directorClient DirectorGraphQLClient, universalClient EventAPIClient, regionalDetails map[string]EventAPIClient, tenantConverter TenantConverter) (*TenantsManager, error) {
	supportedEvents, err := supportedEventTypes(jobConfig.TenantType)
	if err != nil {
		return nil, err
	}

	tenantsManager := &TenantsManager{
		regionalClients:     regionalDetails,
		supportedEventTypes: supportedEvents,
		externalTenantsManager: externalTenantsManager{
			gqlClient:       directorClient,
			eventAPIClient:  universalClient,
			config:          jobConfig.EventsConfig,
			tenantConverter: tenantConverter,
			tenantProvider:  jobConfig.TenantProvider,
		},
	}
	return tenantsManager, nil
}

// TenantsToCreate returns all tenants that are missing in Compass but are present in the external tenants registry
func (tm *TenantsManager) TenantsToCreate(ctx context.Context, region, fromTimestamp string) ([]model.BusinessTenantMappingInput, error) {
	fetchedTenants, err := tm.fetchTenantsForEventType(ctx, region, fromTimestamp, tm.supportedEventTypes.createdTenantEvent)
	if err != nil {
		return nil, errors.Wrapf(err, "while fetching created tenants for region %s ", region)
	}
	updatedTenants, err := tm.fetchTenantsForEventType(ctx, region, fromTimestamp, tm.supportedEventTypes.updatedTenantEvent)
	if err != nil {
		return nil, errors.Wrapf(err, "while fetching updated tenants for region %s ", region)
	}
	updatedTenants = excludeTenants(updatedTenants, fetchedTenants)
	fetchedTenants = append(fetchedTenants, updatedTenants...)
	return fetchedTenants, nil
}

// TenantsToDelete returns all tenants that are no longer associated with their parent tenant and should be moved from one parent tenant to another
func (tm *TenantsManager) TenantsToDelete(ctx context.Context, region, fromTimestamp string) ([]model.BusinessTenantMappingInput, error) {
	res, err := tm.fetchTenantsForEventType(ctx, region, fromTimestamp, tm.supportedEventTypes.deletedTenantEvent)
	if err != nil {
		return nil, errors.Wrapf(err, "while fetching deleted tenants for region %s ", region)
	}
	return res, nil
}

// FetchTenant retrieves a given tenant from all available regions and updates or creates it in Compass
func (tm *TenantsManager) FetchTenant(ctx context.Context, externalTenantID string) (*model.BusinessTenantMappingInput, error) {
	additionalFields := map[string]string{
		tm.config.QueryConfig.EntityField: externalTenantID,
	}
	configProvider := eventsQueryConfigProviderWithAdditionalFields(tm.config, additionalFields)

	fetchedTenants, err := fetchCreatedTenantsWithRetries(ctx, tm.eventAPIClient, tm.config.RetryAttempts, tm.supportedEventTypes, configProvider)
	if err != nil {
		return nil, err
	}

	if len(fetchedTenants) >= 1 {
		log.C(ctx).Infof("Tenant found from central region with universal client")
		return &fetchedTenants[0], err
	}

	log.C(ctx).Infof("Tenant not found from central region, checking regional APIs")

	tenantChan := make(chan *model.BusinessTenantMappingInput, len(tm.regionalClients))
	for region, regionalClient := range tm.regionalClients {
		go func(ctx context.Context, region string, regionalClient EventAPIClient, ch chan *model.BusinessTenantMappingInput) {
			ctx = context.WithValue(ctx, TenantRegionCtxKey, region)
			createdRegionalTenants, err := fetchCreatedTenantsWithRetries(ctx, regionalClient, tm.config.RetryAttempts, tm.supportedEventTypes, configProvider)
			if err != nil {
				log.C(ctx).WithError(err).Errorf("Failed to fetch created tenants from region %s: %v", region, err)
			}

			if len(createdRegionalTenants) == 1 {
				log.C(ctx).Infof("Tenant found in region %s", region)
				ch <- &createdRegionalTenants[0]
			} else {
				log.C(ctx).Warnf("Tenant not found in region %s", region)
				ch <- nil
			}
		}(ctx, region, regionalClient, tenantChan)
	}

	pendingRegionalInfo := len(tm.regionalClients)
	if pendingRegionalInfo == 0 {
		// TODO return error when lazy store is reverted
		log.C(ctx).Error("no regions are configured")
		return nil, nil
	}

	var tenant *model.BusinessTenantMappingInput
	for result := range tenantChan {
		if result != nil {
			tenant = result
			break
		}
		pendingRegionalInfo--
		if pendingRegionalInfo == 0 {
			// TODO return error when lazy store is reverted
			log.C(ctx).Error("tenant not found in all configured regions")
			return nil, nil
		}
	}

	return tenant, nil
}

// CreateTenants takes care of creating all missing tenants in Compass by calling Director in chunks
func (tm *TenantsManager) CreateTenants(ctx context.Context, tenants []model.BusinessTenantMappingInput) error {
	tenantsToCreateGQL := tm.tenantConverter.MultipleInputToGraphQLInput(tenants)
	return runInChunks(ctx, tm.config.TenantOperationChunkSize, tenantsToCreateGQL, func(ctx context.Context, chunk []graphql.BusinessTenantMappingInput) error {
		return tm.gqlClient.WriteTenants(ctx, chunk)
	})
}

// DeleteTenants takes care of deleting all  tenants marked as deleted in the external tenants registry
func (tm *TenantsManager) DeleteTenants(ctx context.Context, tenantsToDelete []model.BusinessTenantMappingInput) error {
	tenantsToDeleteGQL := tm.tenantConverter.MultipleInputToGraphQLInput(tenantsToDelete)
	return runInChunks(ctx, tm.config.TenantOperationChunkSize, tenantsToDeleteGQL, func(ctx context.Context, chunk []graphql.BusinessTenantMappingInput) error {
		return tm.gqlClient.DeleteTenants(ctx, chunk)
	})
}

func (tm *TenantsManager) fetchTenantsForEventType(ctx context.Context, region, fromTimestamp string, eventsType EventsType) ([]model.BusinessTenantMappingInput, error) {
	configProvider := eventsQueryConfigProviderWithRegion(tm.config, fromTimestamp, region)
	fetchedTenants, err := fetchTenantsWithRetries(ctx, tm.eventAPIClient, tm.config.RetryAttempts, eventsType, configProvider)
	if err != nil {
		return nil, errors.Wrap(err, "while fetching tenants with universal client")
	}

	regionClient, ok := tm.regionalClients[region]
	if !ok {
		log.C(ctx).Infof("Region %s does not have local events client enabled", region)
		return fetchedTenants, nil
	}
	configProvider = eventsQueryConfigProvider(tm.config, fromTimestamp)
	createdRegionalTenants, err := fetchTenantsWithRetries(ctx, regionClient, tm.config.RetryAttempts, eventsType, configProvider)
	if err != nil {
		return nil, err
	}

	createdRegionalTenants = excludeTenants(createdRegionalTenants, fetchedTenants)
	fetchedTenants = append(fetchedTenants, createdRegionalTenants...)

	return fetchedTenants, nil
}

func supportedEventTypes(tenantType tenantpkg.Type) (supportedEvents, error) {
	switch tenantType {
	case tenantpkg.Account:
		return supportedEvents{
			createdTenantEvent: CreatedAccountType,
			updatedTenantEvent: UpdatedAccountType,
			deletedTenantEvent: DeletedAccountType,
		}, nil
	case tenantpkg.Subaccount:
		return supportedEvents{
			createdTenantEvent: CreatedSubaccountType,
			updatedTenantEvent: UpdatedSubaccountType,
			deletedTenantEvent: DeletedSubaccountType,
		}, nil
	}
	return supportedEvents{}, errors.Errorf("Tenant events for type %s are not supported", tenantType)
}

func eventsQueryConfigProvider(config EventsConfig, fromTimestamp string) func() (QueryParams, PageConfig) {
	return eventsQueryConfigProviderWithRegion(config, fromTimestamp, "")
}

func eventsQueryConfigProviderWithRegion(config EventsConfig, fromTimestamp, region string) func() (QueryParams, PageConfig) {
	additionalFields := map[string]string{
		config.QueryConfig.TimestampField: fromTimestamp,
	}
	if len(region) > 0 && len(config.QueryConfig.RegionField) > 0 {
		additionalFields[config.QueryConfig.RegionField] = region
	}
	return eventsQueryConfigProviderWithAdditionalFields(config, additionalFields)
}

func eventsQueryConfigProviderWithAdditionalFields(config EventsConfig, additionalQueryParams map[string]string) func() (QueryParams, PageConfig) {
	return func() (QueryParams, PageConfig) {
		qp := QueryParams{
			config.QueryConfig.PageNumField:  config.QueryConfig.PageStartValue,
			config.QueryConfig.PageSizeField: config.QueryConfig.PageSizeValue,
		}
		for field, value := range additionalQueryParams {
			qp[field] = value
		}

		pc := PageConfig{
			TotalPagesField:   config.PagingConfig.TotalPagesField,
			TotalResultsField: config.PagingConfig.TotalResultsField,
			PageNumField:      config.QueryConfig.PageNumField,
			PageWorkers:       config.PageWorkers,
		}
		return qp, pc
	}
}

func fetchCreatedTenantsWithRetries(ctx context.Context, eventAPIClient EventAPIClient, retryNumber uint, supportedEvents supportedEvents, configProvider func() (QueryParams, PageConfig)) ([]model.BusinessTenantMappingInput, error) {
	var fetchedTenants []model.BusinessTenantMappingInput

	createdTenants, err := fetchTenantsWithRetries(ctx, eventAPIClient, retryNumber, supportedEvents.createdTenantEvent, configProvider)
	if err != nil {
		return nil, fmt.Errorf("while fetching created tenants: %v", err)
	}
	fetchedTenants = append(fetchedTenants, createdTenants...)

	updatedTenants, err := fetchTenantsWithRetries(ctx, eventAPIClient, retryNumber, supportedEvents.updatedTenantEvent, configProvider)
	if err != nil {
		return nil, fmt.Errorf("while fetching updated tenants: %v", err)
	}

	updatedTenants = excludeTenants(updatedTenants, createdTenants)
	fetchedTenants = append(fetchedTenants, updatedTenants...)
	return fetchedTenants, nil
}

func fetchTenantsWithRetries(ctx context.Context, eventAPIClient EventAPIClient, retryNumber uint, eventsType EventsType, configProvider func() (QueryParams, PageConfig)) ([]model.BusinessTenantMappingInput, error) {
	var tenants []model.BusinessTenantMappingInput
	if err := fetchWithRetries(retryNumber, func() error {
		fetchedTenants, err := fetchTenants(ctx, eventAPIClient, eventsType, configProvider)
		if err != nil {
			return fmt.Errorf("while fetching tenants: %v", err)
		}
		tenants = fetchedTenants
		return nil
	}); err != nil {
		return nil, err
	}

	return tenants, nil
}

func fetchTenants(ctx context.Context, eventAPIClient EventAPIClient, eventsType EventsType, configProvider func() (QueryParams, PageConfig)) ([]model.BusinessTenantMappingInput, error) {
	tenants := make([]model.BusinessTenantMappingInput, 0)
	if err := walkThroughPages(ctx, eventAPIClient, eventsType, configProvider, func(page *EventsPage) error {
		mappings := page.GetTenantMappings(ctx, eventsType)
		tenants = append(tenants, mappings...)
		return nil
	}); err != nil {
		return nil, fmt.Errorf("while walking through pages: %v", err)
	}

	return tenants, nil
}

func fetchWithRetries(retryAttempts uint, applyFunc func() error) error {
	return retry.Do(applyFunc, retry.Attempts(retryAttempts), retry.Delay(retryDelaySeconds*time.Second))
}

func walkThroughPages(ctx context.Context, eventAPIClient EventAPIClient, eventsType EventsType, configProvider func() (QueryParams, PageConfig), applyFunc func(*EventsPage) error) error {
	params, pageConfig := configProvider()
	firstPage, err := eventAPIClient.FetchTenantEventsPage(ctx, eventsType, params)
	if err != nil {
		return errors.Wrap(err, "while fetching tenant events page")
	}
	if firstPage == nil {
		return nil
	}

	if err = applyFunc(firstPage); err != nil {
		return fmt.Errorf("while applyfunc on event page: %v", err)
	}

	initialCount := gjson.GetBytes(firstPage.Payload, pageConfig.TotalResultsField).Int()
	totalPages := gjson.GetBytes(firstPage.Payload, pageConfig.TotalPagesField).Int()

	pageStart, err := strconv.ParseInt(params[pageConfig.PageNumField], 10, 64)
	if err != nil {
		return err
	}

	region := getRegionFromCtx(ctx)
	log.C(ctx).Infof("Starting processing for %d events across %d pages for event type %v in region %s", initialCount, totalPages, eventsType, region)

	start := time.Now()
	var m sync.Mutex
	eventPages := make([]*EventsPage, 0)
	queryParamsQueue := make(chan QueryParams)
	wg := sync.WaitGroup{}
	var globalErr safeError

	log.C(ctx).Infof("Initializing %d page fetching workers: starting concurrent retrieval of pages.", pageConfig.PageWorkers)
	for worker := 0; worker < pageConfig.PageWorkers; worker++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for qParams := range queryParamsQueue {
				if globalErr.GetError() != nil {
					continue
				}

				var res *EventsPage
				err := retry.Do(func() error {
					res, err = eventAPIClient.FetchTenantEventsPage(ctx, eventsType, qParams)
					if err != nil {
						return errors.Wrap(err, "while fetching tenant events page")
					}
					if res == nil {
						return apperrors.NewInternalError("next page was expected but response was empty")
					}

					return nil
				}, []retry.Option{retry.Attempts(5)}...)

				if err != nil {
					log.C(ctx).WithError(err).Errorf("Error occurred while fetching page.")
					if globalErr.GetError() == nil {
						globalErr.SetError(err)
					}
					continue
				}

				m.Lock()
				eventPages = append(eventPages, &EventsPage{
					FieldMapping:                 res.FieldMapping,
					MovedSubaccountsFieldMapping: res.MovedSubaccountsFieldMapping,
					ProviderName:                 res.ProviderName,
					Payload:                      res.Payload,
				})
				m.Unlock()
			}
		}()
	}

	wgParams := sync.WaitGroup{}
	wgParams.Add(1)

	log.C(ctx).Infof("Starting a goroutine to prepare query parameters.")
	go createParamsForFetching(&wgParams, pageStart, totalPages, params, pageConfig, queryParamsQueue)

	log.C(ctx).Infof("Waiting for the goroutine to prepare all query parameters.")
	wgParams.Wait()

	close(queryParamsQueue)
	log.C(ctx).Infof("Waiting for all page fetching workers to finish.")
	wg.Wait()

	if globalErr.GetError() != nil {
		return globalErr.GetError()
	}

	log.C(ctx).Infof("Successfully fetched %d event pages for region %s. Starting processing...", len(eventPages), region)
	for _, res := range eventPages {
		if err = applyFunc(res); err != nil {
			return err
		}
	}

	log.C(ctx).Infof("Processing of %d events across %d pages for event type %v in region %s took %v", initialCount, totalPages, eventsType, region, time.Since(start))
	return nil
}

func getRegionFromCtx(ctx context.Context) string {
	region, ok := ctx.Value(TenantRegionCtxKey).(string)
	if !ok {
		return ""
	}
	return region
}

func createParamsForFetching(wg *sync.WaitGroup, pageStart int64, totalPages int64, params QueryParams, config PageConfig, queue chan QueryParams) {
	defer wg.Done()

	for i := pageStart + 1; i <= totalPages; i++ {
		qParams := make(map[string]string)
		for k, v := range params {
			qParams[k] = v
		}
		qParams[config.PageNumField] = strconv.FormatInt(i, 10)
		queue <- qParams
	}
}

func runInChunks(ctx context.Context, maxChunkSize int, tenants []graphql.BusinessTenantMappingInput, storeTenantsFunc func(ctx context.Context, chunk []graphql.BusinessTenantMappingInput) error) error {
	for len(tenants) > 0 {
		chunkSize := int(math.Min(float64(len(tenants)), float64(maxChunkSize)))
		tenantsChunk := tenants[:chunkSize]
		if err := storeTenantsFunc(ctx, tenantsChunk); err != nil {
			return err
		}
		tenants = tenants[chunkSize:]
	}

	return nil
}

type safeError struct {
	mu    sync.Mutex
	Error error
}

func (se *safeError) SetError(err error) {
	se.mu.Lock()
	defer se.mu.Unlock()
	se.Error = err
}

func (se *safeError) GetError() error {
	se.mu.Lock()
	defer se.mu.Unlock()
	return se.Error
}