go.temporal.io/server@v1.23.0/common/persistence/persistence_metric_clients.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package persistence
    26  
    27  import (
    28  	"context"
    29  	"fmt"
    30  	"time"
    31  
    32  	commonpb "go.temporal.io/api/common/v1"
    33  	"go.temporal.io/api/serviceerror"
    34  	"go.temporal.io/server/common/headers"
    35  	"go.temporal.io/server/common/log"
    36  	"go.temporal.io/server/common/log/tag"
    37  	"go.temporal.io/server/common/metrics"
    38  	"go.temporal.io/server/service/history/tasks"
    39  )
    40  
    41  type (
    42  	metricEmitter struct {
    43  		metricsHandler metrics.Handler
    44  		logger         log.Logger
    45  	}
    46  
    47  	shardPersistenceClient struct {
    48  		metricEmitter
    49  		healthSignals HealthSignalAggregator
    50  		persistence   ShardManager
    51  	}
    52  
    53  	executionPersistenceClient struct {
    54  		metricEmitter
    55  		healthSignals HealthSignalAggregator
    56  		persistence   ExecutionManager
    57  	}
    58  
    59  	taskPersistenceClient struct {
    60  		metricEmitter
    61  		healthSignals HealthSignalAggregator
    62  		persistence   TaskManager
    63  	}
    64  
    65  	metadataPersistenceClient struct {
    66  		metricEmitter
    67  		healthSignals HealthSignalAggregator
    68  		persistence   MetadataManager
    69  	}
    70  
    71  	clusterMetadataPersistenceClient struct {
    72  		metricEmitter
    73  		healthSignals HealthSignalAggregator
    74  		persistence   ClusterMetadataManager
    75  	}
    76  
    77  	queuePersistenceClient struct {
    78  		metricEmitter
    79  		healthSignals HealthSignalAggregator
    80  		persistence   Queue
    81  	}
    82  )
    83  
    84  var _ ShardManager = (*shardPersistenceClient)(nil)
    85  var _ ExecutionManager = (*executionPersistenceClient)(nil)
    86  var _ TaskManager = (*taskPersistenceClient)(nil)
    87  var _ MetadataManager = (*metadataPersistenceClient)(nil)
    88  var _ ClusterMetadataManager = (*clusterMetadataPersistenceClient)(nil)
    89  var _ Queue = (*queuePersistenceClient)(nil)
    90  
    91  // NewShardPersistenceMetricsClient creates a client to manage shards
    92  func NewShardPersistenceMetricsClient(persistence ShardManager, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) ShardManager {
    93  	return &shardPersistenceClient{
    94  		metricEmitter: metricEmitter{
    95  			metricsHandler: metricsHandler,
    96  			logger:         logger,
    97  		},
    98  		healthSignals: healthSignals,
    99  		persistence:   persistence,
   100  	}
   101  }
   102  
   103  // NewExecutionPersistenceMetricsClient creates a client to manage executions
   104  func NewExecutionPersistenceMetricsClient(persistence ExecutionManager, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) ExecutionManager {
   105  	return &executionPersistenceClient{
   106  		metricEmitter: metricEmitter{
   107  			metricsHandler: metricsHandler,
   108  			logger:         logger,
   109  		},
   110  		healthSignals: healthSignals,
   111  		persistence:   persistence,
   112  	}
   113  }
   114  
   115  // NewTaskPersistenceMetricsClient creates a client to manage tasks
   116  func NewTaskPersistenceMetricsClient(persistence TaskManager, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) TaskManager {
   117  	return &taskPersistenceClient{
   118  		metricEmitter: metricEmitter{
   119  			metricsHandler: metricsHandler,
   120  			logger:         logger,
   121  		},
   122  		healthSignals: healthSignals,
   123  		persistence:   persistence,
   124  	}
   125  }
   126  
   127  // NewMetadataPersistenceMetricsClient creates a MetadataManager client to manage metadata
   128  func NewMetadataPersistenceMetricsClient(persistence MetadataManager, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) MetadataManager {
   129  	return &metadataPersistenceClient{
   130  		metricEmitter: metricEmitter{
   131  			metricsHandler: metricsHandler,
   132  			logger:         logger,
   133  		},
   134  		healthSignals: healthSignals,
   135  		persistence:   persistence,
   136  	}
   137  }
   138  
   139  // NewClusterMetadataPersistenceMetricsClient creates a ClusterMetadataManager client to manage cluster metadata
   140  func NewClusterMetadataPersistenceMetricsClient(persistence ClusterMetadataManager, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) ClusterMetadataManager {
   141  	return &clusterMetadataPersistenceClient{
   142  		metricEmitter: metricEmitter{
   143  			metricsHandler: metricsHandler,
   144  			logger:         logger,
   145  		},
   146  		healthSignals: healthSignals,
   147  		persistence:   persistence,
   148  	}
   149  }
   150  
   151  // NewQueuePersistenceMetricsClient creates a client to manage queue
   152  func NewQueuePersistenceMetricsClient(persistence Queue, metricsHandler metrics.Handler, healthSignals HealthSignalAggregator, logger log.Logger) Queue {
   153  	return &queuePersistenceClient{
   154  		metricEmitter: metricEmitter{
   155  			metricsHandler: metricsHandler,
   156  			logger:         logger,
   157  		},
   158  		healthSignals: healthSignals,
   159  		persistence:   persistence,
   160  	}
   161  }
   162  
   163  func (p *shardPersistenceClient) GetName() string {
   164  	return p.persistence.GetName()
   165  }
   166  
   167  func (p *shardPersistenceClient) GetOrCreateShard(
   168  	ctx context.Context,
   169  	request *GetOrCreateShardRequest,
   170  ) (_ *GetOrCreateShardResponse, retErr error) {
   171  	caller := headers.GetCallerInfo(ctx).CallerName
   172  	startTime := time.Now().UTC()
   173  	defer func() {
   174  		latency := time.Since(startTime)
   175  		p.healthSignals.Record(request.ShardID, caller, latency, retErr)
   176  		p.recordRequestMetrics(metrics.PersistenceGetOrCreateShardScope, caller, latency, retErr)
   177  	}()
   178  	return p.persistence.GetOrCreateShard(ctx, request)
   179  }
   180  
   181  func (p *shardPersistenceClient) UpdateShard(
   182  	ctx context.Context,
   183  	request *UpdateShardRequest,
   184  ) (retErr error) {
   185  	caller := headers.GetCallerInfo(ctx).CallerName
   186  	startTime := time.Now().UTC()
   187  	defer func() {
   188  		p.healthSignals.Record(request.ShardInfo.GetShardId(), caller, time.Since(startTime), retErr)
   189  		p.recordRequestMetrics(metrics.PersistenceUpdateShardScope, caller, time.Since(startTime), retErr)
   190  	}()
   191  	return p.persistence.UpdateShard(ctx, request)
   192  }
   193  
   194  func (p *shardPersistenceClient) AssertShardOwnership(
   195  	ctx context.Context,
   196  	request *AssertShardOwnershipRequest,
   197  ) (retErr error) {
   198  	caller := headers.GetCallerInfo(ctx).CallerName
   199  	startTime := time.Now().UTC()
   200  	defer func() {
   201  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   202  		p.recordRequestMetrics(metrics.PersistenceAssertShardOwnershipScope, caller, time.Since(startTime), retErr)
   203  	}()
   204  	return p.persistence.AssertShardOwnership(ctx, request)
   205  }
   206  
   207  func (p *shardPersistenceClient) Close() {
   208  	p.persistence.Close()
   209  }
   210  
   211  func (p *executionPersistenceClient) GetName() string {
   212  	return p.persistence.GetName()
   213  }
   214  
   215  func (p *executionPersistenceClient) GetHistoryBranchUtil() HistoryBranchUtil {
   216  	return p.persistence.GetHistoryBranchUtil()
   217  }
   218  
   219  func (p *executionPersistenceClient) CreateWorkflowExecution(
   220  	ctx context.Context,
   221  	request *CreateWorkflowExecutionRequest,
   222  ) (_ *CreateWorkflowExecutionResponse, retErr error) {
   223  	caller := headers.GetCallerInfo(ctx).CallerName
   224  	startTime := time.Now().UTC()
   225  	defer func() {
   226  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   227  		p.recordRequestMetrics(metrics.PersistenceCreateWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   228  	}()
   229  	return p.persistence.CreateWorkflowExecution(ctx, request)
   230  }
   231  
   232  func (p *executionPersistenceClient) GetWorkflowExecution(
   233  	ctx context.Context,
   234  	request *GetWorkflowExecutionRequest,
   235  ) (_ *GetWorkflowExecutionResponse, retErr error) {
   236  	caller := headers.GetCallerInfo(ctx).CallerName
   237  	startTime := time.Now().UTC()
   238  	defer func() {
   239  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   240  		p.recordRequestMetrics(metrics.PersistenceGetWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   241  	}()
   242  	return p.persistence.GetWorkflowExecution(ctx, request)
   243  }
   244  
   245  func (p *executionPersistenceClient) SetWorkflowExecution(
   246  	ctx context.Context,
   247  	request *SetWorkflowExecutionRequest,
   248  ) (_ *SetWorkflowExecutionResponse, retErr error) {
   249  	caller := headers.GetCallerInfo(ctx).CallerName
   250  	startTime := time.Now().UTC()
   251  	defer func() {
   252  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   253  		p.recordRequestMetrics(metrics.PersistenceSetWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   254  	}()
   255  	return p.persistence.SetWorkflowExecution(ctx, request)
   256  }
   257  
   258  func (p *executionPersistenceClient) UpdateWorkflowExecution(
   259  	ctx context.Context,
   260  	request *UpdateWorkflowExecutionRequest,
   261  ) (_ *UpdateWorkflowExecutionResponse, retErr error) {
   262  	caller := headers.GetCallerInfo(ctx).CallerName
   263  	startTime := time.Now().UTC()
   264  	defer func() {
   265  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   266  		p.recordRequestMetrics(metrics.PersistenceUpdateWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   267  	}()
   268  	return p.persistence.UpdateWorkflowExecution(ctx, request)
   269  }
   270  
   271  func (p *executionPersistenceClient) ConflictResolveWorkflowExecution(
   272  	ctx context.Context,
   273  	request *ConflictResolveWorkflowExecutionRequest,
   274  ) (_ *ConflictResolveWorkflowExecutionResponse, retErr error) {
   275  	caller := headers.GetCallerInfo(ctx).CallerName
   276  	startTime := time.Now().UTC()
   277  	defer func() {
   278  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   279  		p.recordRequestMetrics(metrics.PersistenceConflictResolveWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   280  	}()
   281  	return p.persistence.ConflictResolveWorkflowExecution(ctx, request)
   282  }
   283  
   284  func (p *executionPersistenceClient) DeleteWorkflowExecution(
   285  	ctx context.Context,
   286  	request *DeleteWorkflowExecutionRequest,
   287  ) (retErr error) {
   288  	caller := headers.GetCallerInfo(ctx).CallerName
   289  	startTime := time.Now().UTC()
   290  	defer func() {
   291  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   292  		p.recordRequestMetrics(metrics.PersistenceDeleteWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   293  	}()
   294  	return p.persistence.DeleteWorkflowExecution(ctx, request)
   295  }
   296  
   297  func (p *executionPersistenceClient) DeleteCurrentWorkflowExecution(
   298  	ctx context.Context,
   299  	request *DeleteCurrentWorkflowExecutionRequest,
   300  ) (retErr error) {
   301  	caller := headers.GetCallerInfo(ctx).CallerName
   302  	startTime := time.Now().UTC()
   303  	defer func() {
   304  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   305  		p.recordRequestMetrics(metrics.PersistenceDeleteCurrentWorkflowExecutionScope, caller, time.Since(startTime), retErr)
   306  	}()
   307  	return p.persistence.DeleteCurrentWorkflowExecution(ctx, request)
   308  }
   309  
   310  func (p *executionPersistenceClient) GetCurrentExecution(
   311  	ctx context.Context,
   312  	request *GetCurrentExecutionRequest,
   313  ) (_ *GetCurrentExecutionResponse, retErr error) {
   314  	caller := headers.GetCallerInfo(ctx).CallerName
   315  	startTime := time.Now().UTC()
   316  	defer func() {
   317  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   318  		p.recordRequestMetrics(metrics.PersistenceGetCurrentExecutionScope, caller, time.Since(startTime), retErr)
   319  	}()
   320  	return p.persistence.GetCurrentExecution(ctx, request)
   321  }
   322  
   323  func (p *executionPersistenceClient) ListConcreteExecutions(
   324  	ctx context.Context,
   325  	request *ListConcreteExecutionsRequest,
   326  ) (_ *ListConcreteExecutionsResponse, retErr error) {
   327  	caller := headers.GetCallerInfo(ctx).CallerName
   328  	startTime := time.Now().UTC()
   329  	defer func() {
   330  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   331  		p.recordRequestMetrics(metrics.PersistenceListConcreteExecutionsScope, caller, time.Since(startTime), retErr)
   332  	}()
   333  	return p.persistence.ListConcreteExecutions(ctx, request)
   334  }
   335  
   336  func (p *executionPersistenceClient) RegisterHistoryTaskReader(
   337  	ctx context.Context,
   338  	request *RegisterHistoryTaskReaderRequest,
   339  ) error {
   340  	// hint methods won't go through persistence rate limiter
   341  	// so also not emitting any persistence request/error metrics
   342  	return p.persistence.RegisterHistoryTaskReader(ctx, request)
   343  }
   344  
   345  func (p *executionPersistenceClient) UnregisterHistoryTaskReader(
   346  	ctx context.Context,
   347  	request *UnregisterHistoryTaskReaderRequest,
   348  ) {
   349  	// hint methods won't go through persistence rate limiter
   350  	// so also not emitting any persistence request/error metrics
   351  	p.persistence.UnregisterHistoryTaskReader(ctx, request)
   352  }
   353  
   354  func (p *executionPersistenceClient) UpdateHistoryTaskReaderProgress(
   355  	ctx context.Context,
   356  	request *UpdateHistoryTaskReaderProgressRequest,
   357  ) {
   358  	// hint methods won't go through persistence rate limiter
   359  	// so also not emitting any persistence request/error metrics
   360  	p.persistence.UpdateHistoryTaskReaderProgress(ctx, request)
   361  }
   362  
   363  func (p *executionPersistenceClient) AddHistoryTasks(
   364  	ctx context.Context,
   365  	request *AddHistoryTasksRequest,
   366  ) (retErr error) {
   367  	caller := headers.GetCallerInfo(ctx).CallerName
   368  	startTime := time.Now().UTC()
   369  	defer func() {
   370  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   371  		p.recordRequestMetrics(metrics.PersistenceAddTasksScope, caller, time.Since(startTime), retErr)
   372  	}()
   373  	return p.persistence.AddHistoryTasks(ctx, request)
   374  }
   375  
   376  func (p *executionPersistenceClient) GetHistoryTasks(
   377  	ctx context.Context,
   378  	request *GetHistoryTasksRequest,
   379  ) (_ *GetHistoryTasksResponse, retErr error) {
   380  	var operation string
   381  	switch request.TaskCategory.ID() {
   382  	case tasks.CategoryIDTransfer:
   383  		operation = metrics.PersistenceGetTransferTasksScope
   384  	case tasks.CategoryIDTimer:
   385  		operation = metrics.PersistenceGetTimerTasksScope
   386  	case tasks.CategoryIDVisibility:
   387  		operation = metrics.PersistenceGetVisibilityTasksScope
   388  	case tasks.CategoryIDReplication:
   389  		operation = metrics.PersistenceGetReplicationTasksScope
   390  	case tasks.CategoryIDArchival:
   391  		operation = metrics.PersistenceGetArchivalTasksScope
   392  	default:
   393  		return nil, serviceerror.NewInternal(fmt.Sprintf("unknown task category type: %v", request.TaskCategory))
   394  	}
   395  
   396  	caller := headers.GetCallerInfo(ctx).CallerName
   397  	startTime := time.Now().UTC()
   398  	defer func() {
   399  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   400  		p.recordRequestMetrics(operation, caller, time.Since(startTime), retErr)
   401  	}()
   402  	return p.persistence.GetHistoryTasks(ctx, request)
   403  }
   404  
   405  func (p *executionPersistenceClient) CompleteHistoryTask(
   406  	ctx context.Context,
   407  	request *CompleteHistoryTaskRequest,
   408  ) (retErr error) {
   409  	var operation string
   410  	switch request.TaskCategory.ID() {
   411  	case tasks.CategoryIDTransfer:
   412  		operation = metrics.PersistenceCompleteTransferTaskScope
   413  	case tasks.CategoryIDTimer:
   414  		operation = metrics.PersistenceCompleteTimerTaskScope
   415  	case tasks.CategoryIDVisibility:
   416  		operation = metrics.PersistenceCompleteVisibilityTaskScope
   417  	case tasks.CategoryIDReplication:
   418  		operation = metrics.PersistenceCompleteReplicationTaskScope
   419  	case tasks.CategoryIDArchival:
   420  		operation = metrics.PersistenceCompleteArchivalTaskScope
   421  	default:
   422  		return serviceerror.NewInternal(fmt.Sprintf("unknown task category type: %v", request.TaskCategory))
   423  	}
   424  
   425  	caller := headers.GetCallerInfo(ctx).CallerName
   426  	startTime := time.Now().UTC()
   427  	defer func() {
   428  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   429  		p.recordRequestMetrics(operation, caller, time.Since(startTime), retErr)
   430  	}()
   431  	return p.persistence.CompleteHistoryTask(ctx, request)
   432  }
   433  
   434  func (p *executionPersistenceClient) RangeCompleteHistoryTasks(
   435  	ctx context.Context,
   436  	request *RangeCompleteHistoryTasksRequest,
   437  ) (retErr error) {
   438  	var operation string
   439  	switch request.TaskCategory.ID() {
   440  	case tasks.CategoryIDTransfer:
   441  		operation = metrics.PersistenceRangeCompleteTransferTasksScope
   442  	case tasks.CategoryIDTimer:
   443  		operation = metrics.PersistenceRangeCompleteTimerTasksScope
   444  	case tasks.CategoryIDVisibility:
   445  		operation = metrics.PersistenceRangeCompleteVisibilityTasksScope
   446  	case tasks.CategoryIDReplication:
   447  		operation = metrics.PersistenceRangeCompleteReplicationTasksScope
   448  	case tasks.CategoryIDArchival:
   449  		operation = metrics.PersistenceRangeCompleteArchivalTasksScope
   450  	default:
   451  		return serviceerror.NewInternal(fmt.Sprintf("unknown task category type: %v", request.TaskCategory))
   452  	}
   453  
   454  	caller := headers.GetCallerInfo(ctx).CallerName
   455  	startTime := time.Now().UTC()
   456  	defer func() {
   457  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   458  		p.recordRequestMetrics(operation, caller, time.Since(startTime), retErr)
   459  	}()
   460  	return p.persistence.RangeCompleteHistoryTasks(ctx, request)
   461  }
   462  
   463  func (p *executionPersistenceClient) PutReplicationTaskToDLQ(
   464  	ctx context.Context,
   465  	request *PutReplicationTaskToDLQRequest,
   466  ) (retErr error) {
   467  	caller := headers.GetCallerInfo(ctx).CallerName
   468  	startTime := time.Now().UTC()
   469  	defer func() {
   470  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   471  		p.recordRequestMetrics(metrics.PersistencePutReplicationTaskToDLQScope, caller, time.Since(startTime), retErr)
   472  	}()
   473  	return p.persistence.PutReplicationTaskToDLQ(ctx, request)
   474  }
   475  
   476  func (p *executionPersistenceClient) GetReplicationTasksFromDLQ(
   477  	ctx context.Context,
   478  	request *GetReplicationTasksFromDLQRequest,
   479  ) (_ *GetHistoryTasksResponse, retErr error) {
   480  	caller := headers.GetCallerInfo(ctx).CallerName
   481  	startTime := time.Now().UTC()
   482  	defer func() {
   483  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   484  		p.recordRequestMetrics(metrics.PersistenceGetReplicationTasksFromDLQScope, caller, time.Since(startTime), retErr)
   485  	}()
   486  	return p.persistence.GetReplicationTasksFromDLQ(ctx, request)
   487  }
   488  
   489  func (p *executionPersistenceClient) DeleteReplicationTaskFromDLQ(
   490  	ctx context.Context,
   491  	request *DeleteReplicationTaskFromDLQRequest,
   492  ) (retErr error) {
   493  	caller := headers.GetCallerInfo(ctx).CallerName
   494  	startTime := time.Now().UTC()
   495  	defer func() {
   496  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   497  		p.recordRequestMetrics(metrics.PersistenceDeleteReplicationTaskFromDLQScope, caller, time.Since(startTime), retErr)
   498  	}()
   499  	return p.persistence.DeleteReplicationTaskFromDLQ(ctx, request)
   500  }
   501  
   502  func (p *executionPersistenceClient) RangeDeleteReplicationTaskFromDLQ(
   503  	ctx context.Context,
   504  	request *RangeDeleteReplicationTaskFromDLQRequest,
   505  ) (retErr error) {
   506  	caller := headers.GetCallerInfo(ctx).CallerName
   507  	startTime := time.Now().UTC()
   508  	defer func() {
   509  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   510  		p.recordRequestMetrics(metrics.PersistenceRangeDeleteReplicationTaskFromDLQScope, caller, time.Since(startTime), retErr)
   511  	}()
   512  	return p.persistence.RangeDeleteReplicationTaskFromDLQ(ctx, request)
   513  }
   514  
   515  func (p *executionPersistenceClient) IsReplicationDLQEmpty(
   516  	ctx context.Context,
   517  	request *GetReplicationTasksFromDLQRequest,
   518  ) (_ bool, retErr error) {
   519  	caller := headers.GetCallerInfo(ctx).CallerName
   520  	startTime := time.Now().UTC()
   521  	defer func() {
   522  		p.healthSignals.Record(request.ShardID, caller, time.Since(startTime), retErr)
   523  		p.recordRequestMetrics(metrics.PersistenceGetReplicationTasksFromDLQScope, caller, time.Since(startTime), retErr)
   524  	}()
   525  	return p.persistence.IsReplicationDLQEmpty(ctx, request)
   526  }
   527  
   528  func (p *executionPersistenceClient) Close() {
   529  	p.persistence.Close()
   530  }
   531  
   532  func (p *taskPersistenceClient) GetName() string {
   533  	return p.persistence.GetName()
   534  }
   535  
   536  func (p *taskPersistenceClient) CreateTasks(
   537  	ctx context.Context,
   538  	request *CreateTasksRequest,
   539  ) (_ *CreateTasksResponse, retErr error) {
   540  	caller := headers.GetCallerInfo(ctx).CallerName
   541  	startTime := time.Now().UTC()
   542  	defer func() {
   543  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   544  		p.recordRequestMetrics(metrics.PersistenceCreateTasksScope, caller, time.Since(startTime), retErr)
   545  	}()
   546  	return p.persistence.CreateTasks(ctx, request)
   547  }
   548  
   549  func (p *taskPersistenceClient) GetTasks(
   550  	ctx context.Context,
   551  	request *GetTasksRequest,
   552  ) (_ *GetTasksResponse, retErr error) {
   553  	caller := headers.GetCallerInfo(ctx).CallerName
   554  	startTime := time.Now().UTC()
   555  	defer func() {
   556  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   557  		p.recordRequestMetrics(metrics.PersistenceGetTasksScope, caller, time.Since(startTime), retErr)
   558  	}()
   559  	return p.persistence.GetTasks(ctx, request)
   560  }
   561  
   562  func (p *taskPersistenceClient) CompleteTask(
   563  	ctx context.Context,
   564  	request *CompleteTaskRequest,
   565  ) (retErr error) {
   566  	caller := headers.GetCallerInfo(ctx).CallerName
   567  	startTime := time.Now().UTC()
   568  	defer func() {
   569  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   570  		p.recordRequestMetrics(metrics.PersistenceCompleteTaskScope, caller, time.Since(startTime), retErr)
   571  	}()
   572  	return p.persistence.CompleteTask(ctx, request)
   573  }
   574  
   575  func (p *taskPersistenceClient) CompleteTasksLessThan(
   576  	ctx context.Context,
   577  	request *CompleteTasksLessThanRequest,
   578  ) (_ int, retErr error) {
   579  	caller := headers.GetCallerInfo(ctx).CallerName
   580  	startTime := time.Now().UTC()
   581  	defer func() {
   582  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   583  		p.recordRequestMetrics(metrics.PersistenceCompleteTasksLessThanScope, caller, time.Since(startTime), retErr)
   584  	}()
   585  	return p.persistence.CompleteTasksLessThan(ctx, request)
   586  }
   587  
   588  func (p *taskPersistenceClient) CreateTaskQueue(
   589  	ctx context.Context,
   590  	request *CreateTaskQueueRequest,
   591  ) (_ *CreateTaskQueueResponse, retErr error) {
   592  	caller := headers.GetCallerInfo(ctx).CallerName
   593  	startTime := time.Now().UTC()
   594  	defer func() {
   595  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   596  		p.recordRequestMetrics(metrics.PersistenceCreateTaskQueueScope, caller, time.Since(startTime), retErr)
   597  	}()
   598  	return p.persistence.CreateTaskQueue(ctx, request)
   599  }
   600  
   601  func (p *taskPersistenceClient) UpdateTaskQueue(
   602  	ctx context.Context,
   603  	request *UpdateTaskQueueRequest,
   604  ) (_ *UpdateTaskQueueResponse, retErr error) {
   605  	caller := headers.GetCallerInfo(ctx).CallerName
   606  	startTime := time.Now().UTC()
   607  	defer func() {
   608  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   609  		p.recordRequestMetrics(metrics.PersistenceUpdateTaskQueueScope, caller, time.Since(startTime), retErr)
   610  	}()
   611  	return p.persistence.UpdateTaskQueue(ctx, request)
   612  }
   613  
   614  func (p *taskPersistenceClient) GetTaskQueue(
   615  	ctx context.Context,
   616  	request *GetTaskQueueRequest,
   617  ) (_ *GetTaskQueueResponse, retErr error) {
   618  	caller := headers.GetCallerInfo(ctx).CallerName
   619  	startTime := time.Now().UTC()
   620  	defer func() {
   621  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   622  		p.recordRequestMetrics(metrics.PersistenceGetTaskQueueScope, caller, time.Since(startTime), retErr)
   623  	}()
   624  	return p.persistence.GetTaskQueue(ctx, request)
   625  }
   626  
   627  func (p *taskPersistenceClient) ListTaskQueue(
   628  	ctx context.Context,
   629  	request *ListTaskQueueRequest,
   630  ) (_ *ListTaskQueueResponse, retErr error) {
   631  	caller := headers.GetCallerInfo(ctx).CallerName
   632  	startTime := time.Now().UTC()
   633  	defer func() {
   634  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   635  		p.recordRequestMetrics(metrics.PersistenceListTaskQueueScope, caller, time.Since(startTime), retErr)
   636  	}()
   637  	return p.persistence.ListTaskQueue(ctx, request)
   638  }
   639  
   640  func (p *taskPersistenceClient) DeleteTaskQueue(
   641  	ctx context.Context,
   642  	request *DeleteTaskQueueRequest,
   643  ) (retErr error) {
   644  	caller := headers.GetCallerInfo(ctx).CallerName
   645  	startTime := time.Now().UTC()
   646  	defer func() {
   647  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   648  		p.recordRequestMetrics(metrics.PersistenceDeleteTaskQueueScope, caller, time.Since(startTime), retErr)
   649  	}()
   650  	return p.persistence.DeleteTaskQueue(ctx, request)
   651  }
   652  
   653  func (p *taskPersistenceClient) GetTaskQueueUserData(
   654  	ctx context.Context,
   655  	request *GetTaskQueueUserDataRequest,
   656  ) (_ *GetTaskQueueUserDataResponse, retErr error) {
   657  	caller := headers.GetCallerInfo(ctx).CallerName
   658  	startTime := time.Now().UTC()
   659  	defer func() {
   660  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   661  		p.recordRequestMetrics(metrics.PersistenceGetTaskQueueUserDataScope, caller, time.Since(startTime), retErr)
   662  	}()
   663  	return p.persistence.GetTaskQueueUserData(ctx, request)
   664  }
   665  
   666  func (p *taskPersistenceClient) UpdateTaskQueueUserData(
   667  	ctx context.Context,
   668  	request *UpdateTaskQueueUserDataRequest,
   669  ) (retErr error) {
   670  	caller := headers.GetCallerInfo(ctx).CallerName
   671  	startTime := time.Now().UTC()
   672  	defer func() {
   673  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   674  		p.recordRequestMetrics(metrics.PersistenceUpdateTaskQueueUserDataScope, caller, time.Since(startTime), retErr)
   675  	}()
   676  	return p.persistence.UpdateTaskQueueUserData(ctx, request)
   677  }
   678  
   679  func (p *taskPersistenceClient) ListTaskQueueUserDataEntries(
   680  	ctx context.Context,
   681  	request *ListTaskQueueUserDataEntriesRequest,
   682  ) (_ *ListTaskQueueUserDataEntriesResponse, retErr error) {
   683  	caller := headers.GetCallerInfo(ctx).CallerName
   684  	startTime := time.Now().UTC()
   685  	defer func() {
   686  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   687  		p.recordRequestMetrics(metrics.PersistenceListTaskQueueUserDataEntriesScope, caller, time.Since(startTime), retErr)
   688  	}()
   689  	return p.persistence.ListTaskQueueUserDataEntries(ctx, request)
   690  }
   691  
   692  func (p *taskPersistenceClient) GetTaskQueuesByBuildId(ctx context.Context, request *GetTaskQueuesByBuildIdRequest) (_ []string, retErr error) {
   693  	caller := headers.GetCallerInfo(ctx).CallerName
   694  	startTime := time.Now().UTC()
   695  	defer func() {
   696  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   697  		p.recordRequestMetrics(metrics.PersistenceGetTaskQueuesByBuildIdScope, caller, time.Since(startTime), retErr)
   698  	}()
   699  	return p.persistence.GetTaskQueuesByBuildId(ctx, request)
   700  }
   701  
   702  func (p *taskPersistenceClient) CountTaskQueuesByBuildId(ctx context.Context, request *CountTaskQueuesByBuildIdRequest) (_ int, retErr error) {
   703  	caller := headers.GetCallerInfo(ctx).CallerName
   704  	startTime := time.Now().UTC()
   705  	defer func() {
   706  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   707  		p.recordRequestMetrics(metrics.PersistenceCountTaskQueuesByBuildIdScope, caller, time.Since(startTime), retErr)
   708  	}()
   709  	return p.persistence.CountTaskQueuesByBuildId(ctx, request)
   710  }
   711  
   712  func (p *taskPersistenceClient) Close() {
   713  	p.persistence.Close()
   714  }
   715  
   716  func (p *metadataPersistenceClient) GetName() string {
   717  	return p.persistence.GetName()
   718  }
   719  
   720  func (p *metadataPersistenceClient) CreateNamespace(
   721  	ctx context.Context,
   722  	request *CreateNamespaceRequest,
   723  ) (_ *CreateNamespaceResponse, retErr error) {
   724  	caller := headers.GetCallerInfo(ctx).CallerName
   725  	startTime := time.Now().UTC()
   726  	defer func() {
   727  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   728  		p.recordRequestMetrics(metrics.PersistenceCreateNamespaceScope, caller, time.Since(startTime), retErr)
   729  	}()
   730  	return p.persistence.CreateNamespace(ctx, request)
   731  }
   732  
   733  func (p *metadataPersistenceClient) GetNamespace(
   734  	ctx context.Context,
   735  	request *GetNamespaceRequest,
   736  ) (_ *GetNamespaceResponse, retErr error) {
   737  	caller := headers.GetCallerInfo(ctx).CallerName
   738  	startTime := time.Now().UTC()
   739  	defer func() {
   740  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   741  		p.recordRequestMetrics(metrics.PersistenceGetNamespaceScope, caller, time.Since(startTime), retErr)
   742  	}()
   743  	return p.persistence.GetNamespace(ctx, request)
   744  }
   745  
   746  func (p *metadataPersistenceClient) UpdateNamespace(
   747  	ctx context.Context,
   748  	request *UpdateNamespaceRequest,
   749  ) (retErr error) {
   750  	caller := headers.GetCallerInfo(ctx).CallerName
   751  	startTime := time.Now().UTC()
   752  	defer func() {
   753  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   754  		p.recordRequestMetrics(metrics.PersistenceUpdateNamespaceScope, caller, time.Since(startTime), retErr)
   755  	}()
   756  	return p.persistence.UpdateNamespace(ctx, request)
   757  }
   758  
   759  func (p *metadataPersistenceClient) RenameNamespace(
   760  	ctx context.Context,
   761  	request *RenameNamespaceRequest,
   762  ) (retErr error) {
   763  	caller := headers.GetCallerInfo(ctx).CallerName
   764  	startTime := time.Now().UTC()
   765  	defer func() {
   766  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   767  		p.recordRequestMetrics(metrics.PersistenceRenameNamespaceScope, caller, time.Since(startTime), retErr)
   768  	}()
   769  	return p.persistence.RenameNamespace(ctx, request)
   770  }
   771  
   772  func (p *metadataPersistenceClient) DeleteNamespace(
   773  	ctx context.Context,
   774  	request *DeleteNamespaceRequest,
   775  ) (retErr error) {
   776  	caller := headers.GetCallerInfo(ctx).CallerName
   777  	startTime := time.Now().UTC()
   778  	defer func() {
   779  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   780  		p.recordRequestMetrics(metrics.PersistenceDeleteNamespaceScope, caller, time.Since(startTime), retErr)
   781  	}()
   782  	return p.persistence.DeleteNamespace(ctx, request)
   783  }
   784  
   785  func (p *metadataPersistenceClient) DeleteNamespaceByName(
   786  	ctx context.Context,
   787  	request *DeleteNamespaceByNameRequest,
   788  ) (retErr error) {
   789  	caller := headers.GetCallerInfo(ctx).CallerName
   790  	startTime := time.Now().UTC()
   791  	defer func() {
   792  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   793  		p.recordRequestMetrics(metrics.PersistenceDeleteNamespaceByNameScope, caller, time.Since(startTime), retErr)
   794  	}()
   795  	return p.persistence.DeleteNamespaceByName(ctx, request)
   796  }
   797  
   798  func (p *metadataPersistenceClient) ListNamespaces(
   799  	ctx context.Context,
   800  	request *ListNamespacesRequest,
   801  ) (_ *ListNamespacesResponse, retErr error) {
   802  	caller := headers.GetCallerInfo(ctx).CallerName
   803  	startTime := time.Now().UTC()
   804  	defer func() {
   805  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   806  		p.recordRequestMetrics(metrics.PersistenceListNamespacesScope, caller, time.Since(startTime), retErr)
   807  	}()
   808  	return p.persistence.ListNamespaces(ctx, request)
   809  }
   810  
   811  func (p *metadataPersistenceClient) GetMetadata(
   812  	ctx context.Context,
   813  ) (_ *GetMetadataResponse, retErr error) {
   814  	caller := headers.GetCallerInfo(ctx).CallerName
   815  	startTime := time.Now().UTC()
   816  	defer func() {
   817  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   818  		p.recordRequestMetrics(metrics.PersistenceGetMetadataScope, caller, time.Since(startTime), retErr)
   819  	}()
   820  	return p.persistence.GetMetadata(ctx)
   821  }
   822  
   823  func (p *metadataPersistenceClient) Close() {
   824  	p.persistence.Close()
   825  }
   826  
   827  // AppendHistoryNodes add a node to history node table
   828  func (p *executionPersistenceClient) AppendHistoryNodes(
   829  	ctx context.Context,
   830  	request *AppendHistoryNodesRequest,
   831  ) (_ *AppendHistoryNodesResponse, retErr error) {
   832  	caller := headers.GetCallerInfo(ctx).CallerName
   833  	startTime := time.Now().UTC()
   834  	defer func() {
   835  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   836  		p.recordRequestMetrics(metrics.PersistenceAppendHistoryNodesScope, caller, time.Since(startTime), retErr)
   837  	}()
   838  	return p.persistence.AppendHistoryNodes(ctx, request)
   839  }
   840  
   841  // AppendRawHistoryNodes add a node to history node table
   842  func (p *executionPersistenceClient) AppendRawHistoryNodes(
   843  	ctx context.Context,
   844  	request *AppendRawHistoryNodesRequest,
   845  ) (_ *AppendHistoryNodesResponse, retErr error) {
   846  	caller := headers.GetCallerInfo(ctx).CallerName
   847  	startTime := time.Now().UTC()
   848  	defer func() {
   849  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   850  		p.recordRequestMetrics(metrics.PersistenceAppendRawHistoryNodesScope, caller, time.Since(startTime), retErr)
   851  	}()
   852  	return p.persistence.AppendRawHistoryNodes(ctx, request)
   853  }
   854  
   855  // ReadHistoryBranch returns history node data for a branch
   856  func (p *executionPersistenceClient) ReadHistoryBranch(
   857  	ctx context.Context,
   858  	request *ReadHistoryBranchRequest,
   859  ) (_ *ReadHistoryBranchResponse, retErr error) {
   860  	caller := headers.GetCallerInfo(ctx).CallerName
   861  	startTime := time.Now().UTC()
   862  	defer func() {
   863  		p.recordRequestMetrics(metrics.PersistenceReadHistoryBranchScope, caller, time.Since(startTime), retErr)
   864  	}()
   865  	return p.persistence.ReadHistoryBranch(ctx, request)
   866  }
   867  
   868  func (p *executionPersistenceClient) ReadHistoryBranchReverse(
   869  	ctx context.Context,
   870  	request *ReadHistoryBranchReverseRequest,
   871  ) (_ *ReadHistoryBranchReverseResponse, retErr error) {
   872  	caller := headers.GetCallerInfo(ctx).CallerName
   873  	startTime := time.Now().UTC()
   874  	defer func() {
   875  		p.recordRequestMetrics(metrics.PersistenceReadHistoryBranchReverseScope, caller, time.Since(startTime), retErr)
   876  	}()
   877  	return p.persistence.ReadHistoryBranchReverse(ctx, request)
   878  }
   879  
   880  // ReadHistoryBranchByBatch returns history node data for a branch ByBatch
   881  func (p *executionPersistenceClient) ReadHistoryBranchByBatch(
   882  	ctx context.Context,
   883  	request *ReadHistoryBranchRequest,
   884  ) (_ *ReadHistoryBranchByBatchResponse, retErr error) {
   885  	caller := headers.GetCallerInfo(ctx).CallerName
   886  	startTime := time.Now().UTC()
   887  	defer func() {
   888  		p.recordRequestMetrics(metrics.PersistenceReadHistoryBranchScope, caller, time.Since(startTime), retErr)
   889  	}()
   890  	return p.persistence.ReadHistoryBranchByBatch(ctx, request)
   891  }
   892  
   893  // ReadRawHistoryBranch returns history node raw data for a branch ByBatch
   894  func (p *executionPersistenceClient) ReadRawHistoryBranch(
   895  	ctx context.Context,
   896  	request *ReadHistoryBranchRequest,
   897  ) (_ *ReadRawHistoryBranchResponse, retErr error) {
   898  	caller := headers.GetCallerInfo(ctx).CallerName
   899  	startTime := time.Now().UTC()
   900  	defer func() {
   901  		p.recordRequestMetrics(metrics.PersistenceReadRawHistoryBranchScope, caller, time.Since(startTime), retErr)
   902  	}()
   903  	return p.persistence.ReadRawHistoryBranch(ctx, request)
   904  }
   905  
   906  // ForkHistoryBranch forks a new branch from an old branch
   907  func (p *executionPersistenceClient) ForkHistoryBranch(
   908  	ctx context.Context,
   909  	request *ForkHistoryBranchRequest,
   910  ) (_ *ForkHistoryBranchResponse, retErr error) {
   911  	caller := headers.GetCallerInfo(ctx).CallerName
   912  	startTime := time.Now().UTC()
   913  	defer func() {
   914  		p.recordRequestMetrics(metrics.PersistenceForkHistoryBranchScope, caller, time.Since(startTime), retErr)
   915  	}()
   916  	return p.persistence.ForkHistoryBranch(ctx, request)
   917  }
   918  
   919  // DeleteHistoryBranch removes a branch
   920  func (p *executionPersistenceClient) DeleteHistoryBranch(
   921  	ctx context.Context,
   922  	request *DeleteHistoryBranchRequest,
   923  ) (retErr error) {
   924  	caller := headers.GetCallerInfo(ctx).CallerName
   925  	startTime := time.Now().UTC()
   926  	defer func() {
   927  		p.recordRequestMetrics(metrics.PersistenceDeleteHistoryBranchScope, caller, time.Since(startTime), retErr)
   928  	}()
   929  	return p.persistence.DeleteHistoryBranch(ctx, request)
   930  }
   931  
   932  // TrimHistoryBranch trims a branch
   933  func (p *executionPersistenceClient) TrimHistoryBranch(
   934  	ctx context.Context,
   935  	request *TrimHistoryBranchRequest,
   936  ) (_ *TrimHistoryBranchResponse, retErr error) {
   937  	caller := headers.GetCallerInfo(ctx).CallerName
   938  	startTime := time.Now().UTC()
   939  	defer func() {
   940  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   941  		p.recordRequestMetrics(metrics.PersistenceTrimHistoryBranchScope, caller, time.Since(startTime), retErr)
   942  	}()
   943  	return p.persistence.TrimHistoryBranch(ctx, request)
   944  }
   945  
   946  func (p *executionPersistenceClient) GetAllHistoryTreeBranches(
   947  	ctx context.Context,
   948  	request *GetAllHistoryTreeBranchesRequest,
   949  ) (_ *GetAllHistoryTreeBranchesResponse, retErr error) {
   950  	caller := headers.GetCallerInfo(ctx).CallerName
   951  	startTime := time.Now().UTC()
   952  	defer func() {
   953  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   954  		p.recordRequestMetrics(metrics.PersistenceGetAllHistoryTreeBranchesScope, caller, time.Since(startTime), retErr)
   955  	}()
   956  	return p.persistence.GetAllHistoryTreeBranches(ctx, request)
   957  }
   958  
   959  // GetHistoryTree returns all branch information of a tree
   960  func (p *executionPersistenceClient) GetHistoryTree(
   961  	ctx context.Context,
   962  	request *GetHistoryTreeRequest,
   963  ) (_ *GetHistoryTreeResponse, retErr error) {
   964  	caller := headers.GetCallerInfo(ctx).CallerName
   965  	startTime := time.Now().UTC()
   966  	defer func() {
   967  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   968  		p.recordRequestMetrics(metrics.PersistenceGetHistoryTreeScope, caller, time.Since(startTime), retErr)
   969  	}()
   970  	return p.persistence.GetHistoryTree(ctx, request)
   971  }
   972  
   973  func (p *queuePersistenceClient) Init(
   974  	ctx context.Context,
   975  	blob *commonpb.DataBlob,
   976  ) error {
   977  	return p.persistence.Init(ctx, blob)
   978  }
   979  
   980  func (p *queuePersistenceClient) EnqueueMessage(
   981  	ctx context.Context,
   982  	blob *commonpb.DataBlob,
   983  ) (retErr error) {
   984  	caller := headers.GetCallerInfo(ctx).CallerName
   985  	startTime := time.Now().UTC()
   986  	defer func() {
   987  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
   988  		p.recordRequestMetrics(metrics.PersistenceEnqueueMessageScope, caller, time.Since(startTime), retErr)
   989  	}()
   990  	return p.persistence.EnqueueMessage(ctx, blob)
   991  }
   992  
   993  func (p *queuePersistenceClient) ReadMessages(
   994  	ctx context.Context,
   995  	lastMessageID int64,
   996  	maxCount int,
   997  ) (_ []*QueueMessage, retErr error) {
   998  	caller := headers.GetCallerInfo(ctx).CallerName
   999  	startTime := time.Now().UTC()
  1000  	defer func() {
  1001  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1002  		p.recordRequestMetrics(metrics.PersistenceReadQueueMessagesScope, caller, time.Since(startTime), retErr)
  1003  	}()
  1004  	return p.persistence.ReadMessages(ctx, lastMessageID, maxCount)
  1005  }
  1006  
  1007  func (p *queuePersistenceClient) UpdateAckLevel(
  1008  	ctx context.Context,
  1009  	metadata *InternalQueueMetadata,
  1010  ) (retErr error) {
  1011  	caller := headers.GetCallerInfo(ctx).CallerName
  1012  	startTime := time.Now().UTC()
  1013  	defer func() {
  1014  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1015  		p.recordRequestMetrics(metrics.PersistenceUpdateAckLevelScope, caller, time.Since(startTime), retErr)
  1016  	}()
  1017  	return p.persistence.UpdateAckLevel(ctx, metadata)
  1018  }
  1019  
  1020  func (p *queuePersistenceClient) GetAckLevels(
  1021  	ctx context.Context,
  1022  ) (_ *InternalQueueMetadata, retErr error) {
  1023  	caller := headers.GetCallerInfo(ctx).CallerName
  1024  	startTime := time.Now().UTC()
  1025  	defer func() {
  1026  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1027  		p.recordRequestMetrics(metrics.PersistenceGetAckLevelScope, caller, time.Since(startTime), retErr)
  1028  	}()
  1029  	return p.persistence.GetAckLevels(ctx)
  1030  }
  1031  
  1032  func (p *queuePersistenceClient) DeleteMessagesBefore(
  1033  	ctx context.Context,
  1034  	messageID int64,
  1035  ) (retErr error) {
  1036  	caller := headers.GetCallerInfo(ctx).CallerName
  1037  	startTime := time.Now().UTC()
  1038  	defer func() {
  1039  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1040  		p.recordRequestMetrics(metrics.PersistenceDeleteMessagesBeforeScope, caller, time.Since(startTime), retErr)
  1041  	}()
  1042  	return p.persistence.DeleteMessagesBefore(ctx, messageID)
  1043  }
  1044  
  1045  func (p *queuePersistenceClient) EnqueueMessageToDLQ(
  1046  	ctx context.Context,
  1047  	blob *commonpb.DataBlob,
  1048  ) (_ int64, retErr error) {
  1049  	caller := headers.GetCallerInfo(ctx).CallerName
  1050  	startTime := time.Now().UTC()
  1051  	defer func() {
  1052  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1053  		p.recordRequestMetrics(metrics.PersistenceEnqueueMessageToDLQScope, caller, time.Since(startTime), retErr)
  1054  	}()
  1055  	return p.persistence.EnqueueMessageToDLQ(ctx, blob)
  1056  }
  1057  
  1058  func (p *queuePersistenceClient) ReadMessagesFromDLQ(
  1059  	ctx context.Context,
  1060  	firstMessageID int64,
  1061  	lastMessageID int64,
  1062  	pageSize int,
  1063  	pageToken []byte,
  1064  ) (_ []*QueueMessage, _ []byte, retErr error) {
  1065  	caller := headers.GetCallerInfo(ctx).CallerName
  1066  	startTime := time.Now().UTC()
  1067  	defer func() {
  1068  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1069  		p.recordRequestMetrics(metrics.PersistenceReadMessagesFromDLQScope, caller, time.Since(startTime), retErr)
  1070  	}()
  1071  	return p.persistence.ReadMessagesFromDLQ(ctx, firstMessageID, lastMessageID, pageSize, pageToken)
  1072  }
  1073  
  1074  func (p *queuePersistenceClient) DeleteMessageFromDLQ(
  1075  	ctx context.Context,
  1076  	messageID int64,
  1077  ) (retErr error) {
  1078  	caller := headers.GetCallerInfo(ctx).CallerName
  1079  	startTime := time.Now().UTC()
  1080  	defer func() {
  1081  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1082  		p.recordRequestMetrics(metrics.PersistenceDeleteMessageFromDLQScope, caller, time.Since(startTime), retErr)
  1083  	}()
  1084  	return p.persistence.DeleteMessageFromDLQ(ctx, messageID)
  1085  }
  1086  
  1087  func (p *queuePersistenceClient) RangeDeleteMessagesFromDLQ(
  1088  	ctx context.Context,
  1089  	firstMessageID int64,
  1090  	lastMessageID int64,
  1091  ) (retErr error) {
  1092  	caller := headers.GetCallerInfo(ctx).CallerName
  1093  	startTime := time.Now().UTC()
  1094  	defer func() {
  1095  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1096  		p.recordRequestMetrics(metrics.PersistenceRangeDeleteMessagesFromDLQScope, caller, time.Since(startTime), retErr)
  1097  	}()
  1098  	return p.persistence.RangeDeleteMessagesFromDLQ(ctx, firstMessageID, lastMessageID)
  1099  }
  1100  
  1101  func (p *queuePersistenceClient) UpdateDLQAckLevel(
  1102  	ctx context.Context,
  1103  	metadata *InternalQueueMetadata,
  1104  ) (retErr error) {
  1105  	caller := headers.GetCallerInfo(ctx).CallerName
  1106  	startTime := time.Now().UTC()
  1107  	defer func() {
  1108  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1109  		p.recordRequestMetrics(metrics.PersistenceUpdateDLQAckLevelScope, caller, time.Since(startTime), retErr)
  1110  	}()
  1111  	return p.persistence.UpdateDLQAckLevel(ctx, metadata)
  1112  }
  1113  
  1114  func (p *queuePersistenceClient) GetDLQAckLevels(
  1115  	ctx context.Context,
  1116  ) (_ *InternalQueueMetadata, retErr error) {
  1117  	caller := headers.GetCallerInfo(ctx).CallerName
  1118  	startTime := time.Now().UTC()
  1119  	defer func() {
  1120  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1121  		p.recordRequestMetrics(metrics.PersistenceGetDLQAckLevelScope, caller, time.Since(startTime), retErr)
  1122  	}()
  1123  	return p.persistence.GetDLQAckLevels(ctx)
  1124  }
  1125  
  1126  func (p *queuePersistenceClient) Close() {
  1127  	p.persistence.Close()
  1128  }
  1129  
  1130  func (p *clusterMetadataPersistenceClient) Close() {
  1131  	p.persistence.Close()
  1132  }
  1133  
  1134  func (p *clusterMetadataPersistenceClient) ListClusterMetadata(
  1135  	ctx context.Context,
  1136  	request *ListClusterMetadataRequest,
  1137  ) (_ *ListClusterMetadataResponse, retErr error) {
  1138  	caller := headers.GetCallerInfo(ctx).CallerName
  1139  	startTime := time.Now().UTC()
  1140  	defer func() {
  1141  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1142  		p.recordRequestMetrics(metrics.PersistenceListClusterMetadataScope, caller, time.Since(startTime), retErr)
  1143  	}()
  1144  	return p.persistence.ListClusterMetadata(ctx, request)
  1145  }
  1146  
  1147  func (p *clusterMetadataPersistenceClient) GetCurrentClusterMetadata(
  1148  	ctx context.Context,
  1149  ) (_ *GetClusterMetadataResponse, retErr error) {
  1150  	caller := headers.GetCallerInfo(ctx).CallerName
  1151  	startTime := time.Now().UTC()
  1152  	defer func() {
  1153  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1154  		p.recordRequestMetrics(metrics.PersistenceGetCurrentClusterMetadataScope, caller, time.Since(startTime), retErr)
  1155  	}()
  1156  	return p.persistence.GetCurrentClusterMetadata(ctx)
  1157  }
  1158  
  1159  func (p *clusterMetadataPersistenceClient) GetClusterMetadata(
  1160  	ctx context.Context,
  1161  	request *GetClusterMetadataRequest,
  1162  ) (_ *GetClusterMetadataResponse, retErr error) {
  1163  	caller := headers.GetCallerInfo(ctx).CallerName
  1164  	startTime := time.Now().UTC()
  1165  	defer func() {
  1166  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1167  		p.recordRequestMetrics(metrics.PersistenceGetClusterMetadataScope, caller, time.Since(startTime), retErr)
  1168  	}()
  1169  	return p.persistence.GetClusterMetadata(ctx, request)
  1170  }
  1171  
  1172  func (p *clusterMetadataPersistenceClient) SaveClusterMetadata(
  1173  	ctx context.Context,
  1174  	request *SaveClusterMetadataRequest,
  1175  ) (_ bool, retErr error) {
  1176  	caller := headers.GetCallerInfo(ctx).CallerName
  1177  	startTime := time.Now().UTC()
  1178  	defer func() {
  1179  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1180  		p.recordRequestMetrics(metrics.PersistenceSaveClusterMetadataScope, caller, time.Since(startTime), retErr)
  1181  	}()
  1182  	return p.persistence.SaveClusterMetadata(ctx, request)
  1183  }
  1184  
  1185  func (p *clusterMetadataPersistenceClient) DeleteClusterMetadata(
  1186  	ctx context.Context,
  1187  	request *DeleteClusterMetadataRequest,
  1188  ) (retErr error) {
  1189  	caller := headers.GetCallerInfo(ctx).CallerName
  1190  	startTime := time.Now().UTC()
  1191  	defer func() {
  1192  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1193  		p.recordRequestMetrics(metrics.PersistenceDeleteClusterMetadataScope, caller, time.Since(startTime), retErr)
  1194  	}()
  1195  	return p.persistence.DeleteClusterMetadata(ctx, request)
  1196  }
  1197  
  1198  func (p *clusterMetadataPersistenceClient) GetName() string {
  1199  	return p.persistence.GetName()
  1200  }
  1201  
  1202  func (p *clusterMetadataPersistenceClient) GetClusterMembers(
  1203  	ctx context.Context,
  1204  	request *GetClusterMembersRequest,
  1205  ) (_ *GetClusterMembersResponse, retErr error) {
  1206  	caller := headers.GetCallerInfo(ctx).CallerName
  1207  	startTime := time.Now().UTC()
  1208  	defer func() {
  1209  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1210  		p.recordRequestMetrics(metrics.PersistenceGetClusterMembersScope, caller, time.Since(startTime), retErr)
  1211  	}()
  1212  	return p.persistence.GetClusterMembers(ctx, request)
  1213  }
  1214  
  1215  func (p *clusterMetadataPersistenceClient) UpsertClusterMembership(
  1216  	ctx context.Context,
  1217  	request *UpsertClusterMembershipRequest,
  1218  ) (retErr error) {
  1219  	caller := headers.GetCallerInfo(ctx).CallerName
  1220  	startTime := time.Now().UTC()
  1221  	defer func() {
  1222  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1223  		p.recordRequestMetrics(metrics.PersistenceUpsertClusterMembershipScope, caller, time.Since(startTime), retErr)
  1224  	}()
  1225  	return p.persistence.UpsertClusterMembership(ctx, request)
  1226  }
  1227  
  1228  func (p *clusterMetadataPersistenceClient) PruneClusterMembership(
  1229  	ctx context.Context,
  1230  	request *PruneClusterMembershipRequest,
  1231  ) (retErr error) {
  1232  	caller := headers.GetCallerInfo(ctx).CallerName
  1233  	startTime := time.Now().UTC()
  1234  	defer func() {
  1235  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1236  		p.recordRequestMetrics(metrics.PersistencePruneClusterMembershipScope, caller, time.Since(startTime), retErr)
  1237  	}()
  1238  	return p.persistence.PruneClusterMembership(ctx, request)
  1239  }
  1240  
  1241  func (p *metadataPersistenceClient) InitializeSystemNamespaces(
  1242  	ctx context.Context,
  1243  	currentClusterName string,
  1244  ) (retErr error) {
  1245  	caller := headers.GetCallerInfo(ctx).CallerName
  1246  	startTime := time.Now().UTC()
  1247  	defer func() {
  1248  		p.healthSignals.Record(CallerSegmentMissing, caller, time.Since(startTime), retErr)
  1249  		p.recordRequestMetrics(metrics.PersistenceInitializeSystemNamespaceScope, caller, time.Since(startTime), retErr)
  1250  	}()
  1251  	return p.persistence.InitializeSystemNamespaces(ctx, currentClusterName)
  1252  }
  1253  
  1254  func (p *metricEmitter) recordRequestMetrics(operation string, caller string, latency time.Duration, err error) {
  1255  	handler := p.metricsHandler.WithTags(metrics.OperationTag(operation), metrics.NamespaceTag(caller))
  1256  	handler.Counter(metrics.PersistenceRequests.Name()).Record(1)
  1257  	handler.Timer(metrics.PersistenceLatency.Name()).Record(latency)
  1258  	updateErrorMetric(handler, p.logger, operation, err)
  1259  }
  1260  
  1261  func updateErrorMetric(handler metrics.Handler, logger log.Logger, operation string, err error) {
  1262  	if err != nil {
  1263  		handler.Counter(metrics.PersistenceErrorWithType.Name()).Record(1, metrics.ServiceErrorTypeTag(err))
  1264  		switch err := err.(type) {
  1265  		case *ShardAlreadyExistError,
  1266  			*ShardOwnershipLostError,
  1267  			*AppendHistoryTimeoutError,
  1268  			*CurrentWorkflowConditionFailedError,
  1269  			*WorkflowConditionFailedError,
  1270  			*ConditionFailedError,
  1271  			*TimeoutError,
  1272  			*serviceerror.InvalidArgument,
  1273  			*serviceerror.NamespaceAlreadyExists,
  1274  			*serviceerror.NotFound,
  1275  			*serviceerror.NamespaceNotFound:
  1276  			// no-op
  1277  
  1278  		case *serviceerror.ResourceExhausted:
  1279  			handler.Counter(metrics.PersistenceErrResourceExhaustedCounter.Name()).Record(1, metrics.ResourceExhaustedCauseTag(err.Cause))
  1280  		default:
  1281  			logger.Error("Operation failed with internal error.", tag.Error(err), tag.Operation(operation))
  1282  			handler.Counter(metrics.PersistenceFailures.Name()).Record(1)
  1283  		}
  1284  	}
  1285  }