github.com/kyma-incubator/compass/components/director@v0.0.0-20230623144113-d764f56ff805/cmd/systemfetcher/main.go (about)

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"crypto/tls"
     6  	"encoding/json"
     7  	"net/http"
     8  	"os"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/kyma-incubator/compass/components/director/internal/domain/formationconstraint/operators"
    13  
    14  	"github.com/kyma-incubator/compass/components/director/internal/model"
    15  
    16  	"github.com/kyma-incubator/compass/components/director/internal/domain/systemssync"
    17  
    18  	"github.com/kyma-incubator/compass/components/director/internal/domain/tenantbusinesstype"
    19  
    20  	"github.com/kyma-incubator/compass/components/director/internal/domain/formationconstraint"
    21  	"github.com/kyma-incubator/compass/components/director/internal/domain/formationtemplateconstraintreferences"
    22  
    23  	databuilder "github.com/kyma-incubator/compass/components/director/internal/domain/webhook/datainputbuilder"
    24  
    25  	"github.com/kyma-incubator/compass/components/director/internal/labelfilter"
    26  
    27  	"github.com/kyma-incubator/compass/components/director/internal/domain/formationassignment"
    28  
    29  	webhookclient "github.com/kyma-incubator/compass/components/director/pkg/webhook_client"
    30  
    31  	"github.com/kyma-incubator/compass/components/director/internal/domain/formationtemplate"
    32  
    33  	"github.com/kyma-incubator/compass/components/director/internal/domain/formation"
    34  
    35  	"github.com/kyma-incubator/compass/components/director/internal/domain/api"
    36  	"github.com/kyma-incubator/compass/components/director/internal/domain/application"
    37  	"github.com/kyma-incubator/compass/components/director/internal/domain/apptemplate"
    38  	"github.com/kyma-incubator/compass/components/director/internal/domain/auth"
    39  	bundleutil "github.com/kyma-incubator/compass/components/director/internal/domain/bundle"
    40  	"github.com/kyma-incubator/compass/components/director/internal/domain/bundlereferences"
    41  	"github.com/kyma-incubator/compass/components/director/internal/domain/document"
    42  	"github.com/kyma-incubator/compass/components/director/internal/domain/eventdef"
    43  	"github.com/kyma-incubator/compass/components/director/internal/domain/fetchrequest"
    44  	"github.com/kyma-incubator/compass/components/director/internal/domain/integrationsystem"
    45  	"github.com/kyma-incubator/compass/components/director/internal/domain/label"
    46  	"github.com/kyma-incubator/compass/components/director/internal/domain/labeldef"
    47  	"github.com/kyma-incubator/compass/components/director/internal/domain/runtime"
    48  	runtimectx "github.com/kyma-incubator/compass/components/director/internal/domain/runtime_context"
    49  	"github.com/kyma-incubator/compass/components/director/internal/domain/scenarioassignment"
    50  	"github.com/kyma-incubator/compass/components/director/internal/domain/spec"
    51  	"github.com/kyma-incubator/compass/components/director/internal/domain/tenant"
    52  	"github.com/kyma-incubator/compass/components/director/internal/domain/version"
    53  	"github.com/kyma-incubator/compass/components/director/internal/domain/webhook"
    54  	"github.com/kyma-incubator/compass/components/director/internal/features"
    55  	"github.com/kyma-incubator/compass/components/director/internal/systemfetcher"
    56  	"github.com/kyma-incubator/compass/components/director/internal/uid"
    57  	"github.com/kyma-incubator/compass/components/director/pkg/accessstrategy"
    58  	pkgAuth "github.com/kyma-incubator/compass/components/director/pkg/auth"
    59  	"github.com/kyma-incubator/compass/components/director/pkg/certloader"
    60  	configprovider "github.com/kyma-incubator/compass/components/director/pkg/config"
    61  	"github.com/kyma-incubator/compass/components/director/pkg/executor"
    62  	httputil "github.com/kyma-incubator/compass/components/director/pkg/http"
    63  	"github.com/kyma-incubator/compass/components/director/pkg/log"
    64  
    65  	"github.com/kyma-incubator/compass/components/director/pkg/normalizer"
    66  	oauth "github.com/kyma-incubator/compass/components/director/pkg/oauth"
    67  	"github.com/kyma-incubator/compass/components/director/pkg/persistence"
    68  	gcli "github.com/machinebox/graphql"
    69  	"github.com/pkg/errors"
    70  	"github.com/vrischmann/envconfig"
    71  )
    72  
    73  const discoverSystemsOpMode = "DISCOVER_SYSTEMS"
    74  
    75  type config struct {
    76  	APIConfig      systemfetcher.APIConfig
    77  	OAuth2Config   oauth.Config
    78  	SystemFetcher  systemfetcher.Config
    79  	Database       persistence.DatabaseConfig
    80  	TemplateConfig appTemplateConfig
    81  
    82  	Log log.Config
    83  
    84  	Features features.Config
    85  
    86  	ConfigurationFile string
    87  
    88  	ConfigurationFileReload time.Duration `envconfig:"default=1m"`
    89  	ClientTimeout           time.Duration `envconfig:"default=60s"`
    90  
    91  	CertLoaderConfig certloader.Config
    92  
    93  	SelfRegisterDistinguishLabelKey string `envconfig:"APP_SELF_REGISTER_DISTINGUISH_LABEL_KEY"`
    94  
    95  	ORDWebhookMappings string `envconfig:"APP_ORD_WEBHOOK_MAPPINGS"`
    96  
    97  	ExternalClientCertSecretName string `envconfig:"APP_EXTERNAL_CLIENT_CERT_SECRET_NAME"`
    98  	ExtSvcClientCertSecretName   string `envconfig:"APP_EXT_SVC_CLIENT_CERT_SECRET_NAME"`
    99  }
   100  
   101  type appTemplateConfig struct {
   102  	LabelFilter                    string `envconfig:"APP_TEMPLATE_LABEL_FILTER"`
   103  	OverrideApplicationInput       string `envconfig:"APP_TEMPLATE_OVERRIDE_APPLICATION_INPUT"`
   104  	PlaceholderToSystemKeyMappings string `envconfig:"APP_TEMPLATE_PLACEHOLDER_TO_SYSTEM_KEY_MAPPINGS"`
   105  }
   106  
   107  func main() {
   108  	cfg := config{}
   109  	err := envconfig.InitWithPrefix(&cfg, "APP")
   110  	if err != nil {
   111  		log.D().Fatal(errors.Wrap(err, "failed to load config"))
   112  	}
   113  
   114  	ctx, err := log.Configure(context.Background(), &cfg.Log)
   115  	if err != nil {
   116  		log.D().Fatal(errors.Wrap(err, "failed to configure logger"))
   117  	}
   118  
   119  	cfgProvider := createAndRunConfigProvider(ctx, cfg)
   120  
   121  	transact, closeFunc, err := persistence.Configure(ctx, cfg.Database)
   122  	if err != nil {
   123  		log.D().Fatal(errors.Wrap(err, "failed to connect to the database"))
   124  	}
   125  	defer func() {
   126  		err := closeFunc()
   127  		if err != nil {
   128  			log.D().Fatal(errors.Wrap(err, "failed to close database connection"))
   129  		}
   130  	}()
   131  
   132  	certCache, err := certloader.StartCertLoader(ctx, cfg.CertLoaderConfig)
   133  	if err != nil {
   134  		log.D().Fatal(errors.Wrap(err, "failed to initialize certificate loader"))
   135  	}
   136  
   137  	httpClient := &http.Client{Timeout: cfg.ClientTimeout}
   138  	securedHTTPClient := pkgAuth.PrepareHTTPClient(cfg.ClientTimeout)
   139  	mtlsClient := pkgAuth.PrepareMTLSClient(cfg.ClientTimeout, certCache, cfg.ExternalClientCertSecretName)
   140  	extSvcMtlsClient := pkgAuth.PrepareMTLSClient(cfg.ClientTimeout, certCache, cfg.ExtSvcClientCertSecretName)
   141  
   142  	sf, err := createSystemFetcher(ctx, cfg, cfgProvider, transact, httpClient, securedHTTPClient, mtlsClient, extSvcMtlsClient, certCache)
   143  	if err != nil {
   144  		log.D().Fatal(errors.Wrap(err, "failed to initialize System Fetcher"))
   145  	}
   146  
   147  	if cfg.SystemFetcher.OperationalMode != discoverSystemsOpMode {
   148  		log.C(ctx).Infof("The operatioal mode is set to %q, skipping systems discovery.", cfg.SystemFetcher.OperationalMode)
   149  		return
   150  	}
   151  
   152  	if err = sf.SyncSystems(ctx); err != nil {
   153  		log.D().Fatal(errors.Wrap(err, "failed to sync systems"))
   154  	}
   155  
   156  	if err = sf.UpsertSystemsSyncTimestamps(ctx, transact); err != nil {
   157  		log.D().Fatal(errors.Wrap(err, "failed to upsert systems synchronization timestamps in database"))
   158  	}
   159  }
   160  
   161  func createSystemFetcher(ctx context.Context, cfg config, cfgProvider *configprovider.Provider, tx persistence.Transactioner, httpClient, securedHTTPClient, mtlsClient, extSvcMtlsClient *http.Client, certCache certloader.Cache) (*systemfetcher.SystemFetcher, error) {
   162  	ordWebhookMapping, err := application.UnmarshalMappings(cfg.ORDWebhookMappings)
   163  	if err != nil {
   164  		return nil, errors.Wrap(err, "failed while unmarshalling ord webhook mappings")
   165  	}
   166  
   167  	tenantConverter := tenant.NewConverter()
   168  	tenantBusinessTypeConverter := tenantbusinesstype.NewConverter()
   169  	authConverter := auth.NewConverter()
   170  	frConverter := fetchrequest.NewConverter(authConverter)
   171  	versionConverter := version.NewConverter()
   172  	docConverter := document.NewConverter(frConverter)
   173  	webhookConverter := webhook.NewConverter(authConverter)
   174  	specConverter := spec.NewConverter(frConverter)
   175  	apiConverter := api.NewConverter(versionConverter, specConverter)
   176  	eventAPIConverter := eventdef.NewConverter(versionConverter, specConverter)
   177  	labelDefConverter := labeldef.NewConverter()
   178  	labelConverter := label.NewConverter()
   179  	intSysConverter := integrationsystem.NewConverter()
   180  	bundleConverter := bundleutil.NewConverter(authConverter, apiConverter, eventAPIConverter, docConverter)
   181  	appConverter := application.NewConverter(webhookConverter, bundleConverter)
   182  	runtimeConverter := runtime.NewConverter(webhookConverter)
   183  	bundleReferenceConverter := bundlereferences.NewConverter()
   184  	runtimeContextConverter := runtimectx.NewConverter()
   185  	formationConverter := formation.NewConverter()
   186  	formationTemplateConverter := formationtemplate.NewConverter(webhookConverter)
   187  	assignmentConverter := scenarioassignment.NewConverter()
   188  	appTemplateConverter := apptemplate.NewConverter(appConverter, webhookConverter)
   189  	formationAssignmentConverter := formationassignment.NewConverter()
   190  	formationConstraintConverter := formationconstraint.NewConverter()
   191  	formationTemplateConstraintReferencesConverter := formationtemplateconstraintreferences.NewConverter()
   192  	systemsSyncConverter := systemssync.NewConverter()
   193  
   194  	tenantRepo := tenant.NewRepository(tenantConverter)
   195  	tenantBusinessTypeRepo := tenantbusinesstype.NewRepository(tenantBusinessTypeConverter)
   196  	runtimeRepo := runtime.NewRepository(runtimeConverter)
   197  	applicationRepo := application.NewRepository(appConverter)
   198  	labelRepo := label.NewRepository(labelConverter)
   199  	labelDefRepo := labeldef.NewRepository(labelDefConverter)
   200  	webhookRepo := webhook.NewRepository(webhookConverter)
   201  	apiRepo := api.NewRepository(apiConverter)
   202  	eventAPIRepo := eventdef.NewRepository(eventAPIConverter)
   203  	specRepo := spec.NewRepository(specConverter)
   204  	docRepo := document.NewRepository(docConverter)
   205  	fetchRequestRepo := fetchrequest.NewRepository(frConverter)
   206  	intSysRepo := integrationsystem.NewRepository(intSysConverter)
   207  	bundleRepo := bundleutil.NewRepository(bundleConverter)
   208  	bundleReferenceRepo := bundlereferences.NewRepository(bundleReferenceConverter)
   209  	runtimeContextRepo := runtimectx.NewRepository(runtimeContextConverter)
   210  	formationRepo := formation.NewRepository(formationConverter)
   211  	formationTemplateRepo := formationtemplate.NewRepository(formationTemplateConverter)
   212  	scenarioAssignmentRepo := scenarioassignment.NewRepository(assignmentConverter)
   213  	appTemplateRepo := apptemplate.NewRepository(appTemplateConverter)
   214  	formationAssignmentRepo := formationassignment.NewRepository(formationAssignmentConverter)
   215  	formationConstraintRepo := formationconstraint.NewRepository(formationConstraintConverter)
   216  	formationTemplateConstraintReferencesRepo := formationtemplateconstraintreferences.NewRepository(formationTemplateConstraintReferencesConverter)
   217  	systemsSyncRepo := systemssync.NewRepository(systemsSyncConverter)
   218  
   219  	uidSvc := uid.NewService()
   220  	tenantSvc := tenant.NewService(tenantRepo, uidSvc, tenantConverter)
   221  	tenantBusinessTypeSvc := tenantbusinesstype.NewService(tenantBusinessTypeRepo, uidSvc)
   222  	labelSvc := label.NewLabelService(labelRepo, labelDefRepo, uidSvc)
   223  	intSysSvc := integrationsystem.NewService(intSysRepo, uidSvc)
   224  	scenariosSvc := labeldef.NewService(labelDefRepo, labelRepo, scenarioAssignmentRepo, tenantRepo, uidSvc)
   225  	fetchRequestSvc := fetchrequest.NewService(fetchRequestRepo, httpClient, accessstrategy.NewDefaultExecutorProvider(certCache, cfg.ExternalClientCertSecretName, cfg.ExtSvcClientCertSecretName))
   226  	specSvc := spec.NewService(specRepo, fetchRequestRepo, uidSvc, fetchRequestSvc)
   227  	bundleReferenceSvc := bundlereferences.NewService(bundleReferenceRepo, uidSvc)
   228  	apiSvc := api.NewService(apiRepo, uidSvc, specSvc, bundleReferenceSvc)
   229  	eventAPISvc := eventdef.NewService(eventAPIRepo, uidSvc, specSvc, bundleReferenceSvc)
   230  	docSvc := document.NewService(docRepo, fetchRequestRepo, uidSvc)
   231  	bundleSvc := bundleutil.NewService(bundleRepo, apiSvc, eventAPISvc, docSvc, uidSvc)
   232  	scenarioAssignmentSvc := scenarioassignment.NewService(scenarioAssignmentRepo, scenariosSvc)
   233  	tntSvc := tenant.NewServiceWithLabels(tenantRepo, uidSvc, labelRepo, labelSvc, tenantConverter)
   234  	webhookClient := webhookclient.NewClient(securedHTTPClient, mtlsClient, extSvcMtlsClient)
   235  	appTemplateSvc := apptemplate.NewService(appTemplateRepo, webhookRepo, uidSvc, labelSvc, labelRepo, applicationRepo)
   236  	webhookDataInputBuilder := databuilder.NewWebhookDataInputBuilder(applicationRepo, appTemplateRepo, runtimeRepo, runtimeContextRepo, labelRepo)
   237  	formationConstraintSvc := formationconstraint.NewService(formationConstraintRepo, formationTemplateConstraintReferencesRepo, uidSvc, formationConstraintConverter)
   238  	constraintEngine := operators.NewConstraintEngine(tx, formationConstraintSvc, tenantSvc, scenarioAssignmentSvc, nil, formationRepo, labelRepo, labelSvc, applicationRepo, runtimeContextRepo, formationTemplateRepo, formationAssignmentRepo, cfg.Features.RuntimeTypeLabelKey, cfg.Features.ApplicationTypeLabelKey)
   239  	notificationsBuilder := formation.NewNotificationsBuilder(webhookConverter, constraintEngine, cfg.Features.RuntimeTypeLabelKey, cfg.Features.ApplicationTypeLabelKey)
   240  	notificationsGenerator := formation.NewNotificationsGenerator(applicationRepo, appTemplateRepo, runtimeRepo, runtimeContextRepo, labelRepo, webhookRepo, webhookDataInputBuilder, notificationsBuilder)
   241  	notificationSvc := formation.NewNotificationService(tenantRepo, webhookClient, notificationsGenerator, constraintEngine, webhookConverter, formationTemplateRepo)
   242  	faNotificationSvc := formationassignment.NewFormationAssignmentNotificationService(formationAssignmentRepo, webhookConverter, webhookRepo, tenantRepo, webhookDataInputBuilder, formationRepo, notificationsBuilder, runtimeContextRepo, labelSvc, cfg.Features.RuntimeTypeLabelKey, cfg.Features.ApplicationTypeLabelKey)
   243  	formationAssignmentStatusSvc := formationassignment.NewFormationAssignmentStatusService(formationAssignmentRepo, constraintEngine, faNotificationSvc)
   244  	formationAssignmentSvc := formationassignment.NewService(formationAssignmentRepo, uidSvc, applicationRepo, runtimeRepo, runtimeContextRepo, notificationSvc, faNotificationSvc, labelSvc, formationRepo, formationAssignmentStatusSvc, cfg.Features.RuntimeTypeLabelKey, cfg.Features.ApplicationTypeLabelKey)
   245  	formationStatusSvc := formation.NewFormationStatusService(formationRepo, labelDefRepo, scenariosSvc, notificationSvc, constraintEngine)
   246  	formationSvc := formation.NewService(tx, applicationRepo, labelDefRepo, labelRepo, formationRepo, formationTemplateRepo, labelSvc, uidSvc, scenariosSvc, scenarioAssignmentRepo, scenarioAssignmentSvc, tntSvc, runtimeRepo, runtimeContextRepo, formationAssignmentSvc, faNotificationSvc, notificationSvc, constraintEngine, webhookRepo, formationStatusSvc, cfg.Features.RuntimeTypeLabelKey, cfg.Features.ApplicationTypeLabelKey)
   247  	appSvc := application.NewService(&normalizer.DefaultNormalizator{}, cfgProvider, applicationRepo, webhookRepo, runtimeRepo, labelRepo, intSysRepo, labelSvc, bundleSvc, uidSvc, formationSvc, cfg.SelfRegisterDistinguishLabelKey, ordWebhookMapping)
   248  	systemsSyncSvc := systemssync.NewService(systemsSyncRepo)
   249  
   250  	authProvider := pkgAuth.NewMtlsTokenAuthorizationProvider(cfg.OAuth2Config, cfg.ExternalClientCertSecretName, certCache, pkgAuth.DefaultMtlsClientCreator)
   251  	client := &http.Client{
   252  		Transport: httputil.NewSecuredTransport(httputil.NewHTTPTransportWrapper(http.DefaultTransport.(*http.Transport)), authProvider),
   253  		Timeout:   cfg.APIConfig.Timeout,
   254  	}
   255  	oauthMtlsClient := systemfetcher.NewOauthMtlsClient(cfg.OAuth2Config, certCache, client)
   256  	systemsAPIClient := systemfetcher.NewClient(cfg.APIConfig, oauthMtlsClient)
   257  
   258  	tr := &http.Transport{
   259  		TLSClientConfig: &tls.Config{
   260  			InsecureSkipVerify: cfg.SystemFetcher.DirectorSkipSSLValidation,
   261  		},
   262  	}
   263  
   264  	httpTransport := httputil.NewCorrelationIDTransport(httputil.NewErrorHandlerTransport(httputil.NewHTTPTransportWrapper(tr)))
   265  
   266  	securedClient := &http.Client{
   267  		Transport: httpTransport,
   268  		Timeout:   cfg.SystemFetcher.DirectorRequestTimeout,
   269  	}
   270  
   271  	graphqlClient := gcli.NewClient(cfg.SystemFetcher.DirectorGraphqlURL, gcli.WithHTTPClient(securedClient))
   272  	directorClient := &systemfetcher.DirectorGraphClient{
   273  		Client:        graphqlClient,
   274  		Authenticator: pkgAuth.NewServiceAccountTokenAuthorizationProvider(),
   275  	}
   276  
   277  	dataLoader := systemfetcher.NewDataLoader(tx, cfg.SystemFetcher, appTemplateSvc, intSysSvc)
   278  	if err := dataLoader.LoadData(ctx, os.ReadDir, os.ReadFile); err != nil {
   279  		return nil, err
   280  	}
   281  
   282  	if err := loadSystemsSynchronizationTimestamps(ctx, tx, systemsSyncSvc); err != nil {
   283  		return nil, errors.Wrap(err, "failed while loading systems synchronization timestamps")
   284  	}
   285  
   286  	var placeholdersMapping []systemfetcher.PlaceholderMapping
   287  	if err := json.Unmarshal([]byte(cfg.TemplateConfig.PlaceholderToSystemKeyMappings), &placeholdersMapping); err != nil {
   288  		return nil, errors.Wrapf(err, "while unmarshaling placeholders mapping")
   289  	}
   290  
   291  	if err := calculateTemplateMappings(ctx, cfg, tx, appTemplateSvc, placeholdersMapping); err != nil {
   292  		return nil, errors.Wrap(err, "failed while calculating application templates mappings")
   293  	}
   294  
   295  	templateRenderer, err := systemfetcher.NewTemplateRenderer(appTemplateSvc, appConverter, cfg.TemplateConfig.OverrideApplicationInput, placeholdersMapping)
   296  	if err != nil {
   297  		return nil, errors.Wrapf(err, "while creating template renderer")
   298  	}
   299  
   300  	return systemfetcher.NewSystemFetcher(tx, tenantSvc, appSvc, systemsSyncSvc, tenantBusinessTypeSvc, templateRenderer, systemsAPIClient, directorClient, cfg.SystemFetcher), nil
   301  }
   302  
   303  func createAndRunConfigProvider(ctx context.Context, cfg config) *configprovider.Provider {
   304  	provider := configprovider.NewProvider(cfg.ConfigurationFile)
   305  	err := provider.Load()
   306  	if err != nil {
   307  		log.D().Fatal(errors.Wrap(err, "error on loading configuration file"))
   308  	}
   309  	executor.NewPeriodic(cfg.ConfigurationFileReload, func(ctx context.Context) {
   310  		if err = provider.Load(); err != nil {
   311  			if err != nil {
   312  				log.D().Fatal(errors.Wrap(err, "error from Reloader watch"))
   313  			}
   314  		}
   315  		log.C(ctx).Infof("Successfully reloaded configuration file.")
   316  	}).Run(ctx)
   317  
   318  	return provider
   319  }
   320  
   321  func calculateTemplateMappings(ctx context.Context, cfg config, transact persistence.Transactioner, appTemplateSvc apptemplate.ApplicationTemplateService, placeholdersMapping []systemfetcher.PlaceholderMapping) error {
   322  	applicationTemplates := make([]systemfetcher.TemplateMapping, 0)
   323  
   324  	tx, err := transact.Begin()
   325  	if err != nil {
   326  		return errors.Wrap(err, "failed to begin transaction")
   327  	}
   328  	defer transact.RollbackUnlessCommitted(ctx, tx)
   329  	ctx = persistence.SaveToContext(ctx, tx)
   330  
   331  	appTemplates, err := appTemplateSvc.ListByFilters(ctx, []*labelfilter.LabelFilter{labelfilter.NewForKey(cfg.TemplateConfig.LabelFilter)})
   332  	if err != nil {
   333  		return errors.Wrapf(err, "while listing application templates by label filter %q", cfg.TemplateConfig.LabelFilter)
   334  	}
   335  
   336  	selectFilterProperties := make(map[string]bool, 0)
   337  	for _, appTemplate := range appTemplates {
   338  		lbl, err := appTemplateSvc.ListLabels(ctx, appTemplate.ID)
   339  		if err != nil {
   340  			return errors.Wrapf(err, "while listing labels for application template with ID %q", appTemplate.ID)
   341  		}
   342  
   343  		applicationTemplates = append(applicationTemplates, systemfetcher.TemplateMapping{AppTemplate: appTemplate, Labels: lbl})
   344  
   345  		addPropertiesFromAppTemplatePlaceholders(selectFilterProperties, appTemplate.Placeholders)
   346  	}
   347  
   348  	err = tx.Commit()
   349  	if err != nil {
   350  		return errors.Wrap(err, "failed to commit transaction")
   351  	}
   352  
   353  	systemfetcher.ApplicationTemplates = applicationTemplates
   354  	systemfetcher.SelectFilter = createSelectFilter(selectFilterProperties, placeholdersMapping)
   355  	systemfetcher.ApplicationTemplateLabelFilter = cfg.TemplateConfig.LabelFilter
   356  	systemfetcher.SystemSourceKey = cfg.APIConfig.SystemSourceKey
   357  	return nil
   358  }
   359  
   360  func loadSystemsSynchronizationTimestamps(ctx context.Context, transact persistence.Transactioner, systemSyncSvc systemfetcher.SystemsSyncService) error {
   361  	systemSynchronizationTimestamps := make(map[string]map[string]systemfetcher.SystemSynchronizationTimestamp, 0)
   362  
   363  	tx, err := transact.Begin()
   364  	if err != nil {
   365  		return errors.Wrap(err, "failed to begin transaction")
   366  	}
   367  	defer transact.RollbackUnlessCommitted(ctx, tx)
   368  
   369  	ctx = persistence.SaveToContext(ctx, tx)
   370  
   371  	syncTimestamps, err := systemSyncSvc.List(ctx)
   372  	if err != nil {
   373  		return err
   374  	}
   375  
   376  	for _, s := range syncTimestamps {
   377  		currentTimestamp := systemfetcher.SystemSynchronizationTimestamp{
   378  			ID:                s.ID,
   379  			LastSyncTimestamp: s.LastSyncTimestamp,
   380  		}
   381  
   382  		if _, ok := systemSynchronizationTimestamps[s.TenantID]; !ok {
   383  			systemSynchronizationTimestamps[s.TenantID] = make(map[string]systemfetcher.SystemSynchronizationTimestamp, 0)
   384  		}
   385  
   386  		systemSynchronizationTimestamps[s.TenantID][s.ProductID] = currentTimestamp
   387  	}
   388  
   389  	err = tx.Commit()
   390  	if err != nil {
   391  		return errors.Wrap(err, "failed to commit transaction")
   392  	}
   393  
   394  	systemfetcher.SystemSynchronizationTimestamps = systemSynchronizationTimestamps
   395  
   396  	return nil
   397  }
   398  
   399  func getTopParentFromJSONPath(jsonPath string) string {
   400  	prefix := "$."
   401  	infix := "."
   402  
   403  	topParent := strings.TrimPrefix(jsonPath, prefix)
   404  	firstInfixIndex := strings.Index(topParent, infix)
   405  	if firstInfixIndex == -1 {
   406  		return topParent
   407  	}
   408  
   409  	return topParent[:firstInfixIndex]
   410  }
   411  
   412  func addPropertiesFromAppTemplatePlaceholders(selectFilterProperties map[string]bool, placeholders []model.ApplicationTemplatePlaceholder) {
   413  	for _, placeholder := range placeholders {
   414  		if placeholder.JSONPath != nil && len(*placeholder.JSONPath) > 0 {
   415  			topParent := getTopParentFromJSONPath(*placeholder.JSONPath)
   416  			if _, exists := selectFilterProperties[topParent]; !exists {
   417  				selectFilterProperties[topParent] = true
   418  			}
   419  		}
   420  	}
   421  }
   422  
   423  func createSelectFilter(selectFilterProperties map[string]bool, placeholdersMapping []systemfetcher.PlaceholderMapping) []string {
   424  	selectFilter := make([]string, 0)
   425  
   426  	for _, pm := range placeholdersMapping {
   427  		topParent := getTopParentFromJSONPath(pm.SystemKey)
   428  		if _, exists := selectFilterProperties[topParent]; !exists {
   429  			selectFilterProperties[topParent] = true
   430  		}
   431  	}
   432  
   433  	for property := range selectFilterProperties {
   434  		selectFilter = append(selectFilter, property)
   435  	}
   436  
   437  	return selectFilter
   438  }