go.temporal.io/server@v1.23.0/common/persistence/history_task_queue_manager.go (about)

     1  // The MIT License
     2  //
     3  // Copyright (c) 2020 Temporal Technologies Inc.  All rights reserved.
     4  //
     5  // Copyright (c) 2020 Uber Technologies, Inc.
     6  //
     7  // Permission is hereby granted, free of charge, to any person obtaining a copy
     8  // of this software and associated documentation files (the "Software"), to deal
     9  // in the Software without restriction, including without limitation the rights
    10  // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    11  // copies of the Software, and to permit persons to whom the Software is
    12  // furnished to do so, subject to the following conditions:
    13  //
    14  // The above copyright notice and this permission notice shall be included in
    15  // all copies or substantial portions of the Software.
    16  //
    17  // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    18  // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    19  // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    20  // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    21  // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    22  // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    23  // THE SOFTWARE.
    24  
    25  package persistence
    26  
    27  import (
    28  	"context"
    29  	"crypto/sha256"
    30  	"encoding/base64"
    31  	"errors"
    32  	"fmt"
    33  
    34  	commonpb "go.temporal.io/api/common/v1"
    35  	"go.temporal.io/api/enums/v1"
    36  	persistencespb "go.temporal.io/server/api/persistence/v1"
    37  	"go.temporal.io/server/common/persistence/serialization"
    38  )
    39  
    40  const (
    41  	// clusterNamesHashSuffixLength is the number of characters to use from the hash of the cluster names when forming
    42  	// the queue name. This is used to avoid name collisions when a cluster name contains the separator character.
    43  	clusterNamesHashSuffixLength = 8
    44  
    45  	ErrMsgSerializeTaskToEnqueue = "failed to serialize history task for task queue"
    46  	// ErrMsgDeserializeRawHistoryTask is returned when the raw task cannot be deserialized from the task queue. This error
    47  	// is returned when this whole top-level proto cannot be deserialized.
    48  	//  Raw Task (a proto): <-- when this cannot be deserialized
    49  	//	- ShardID
    50  	//	- Blob (a serialized task)
    51  	ErrMsgDeserializeRawHistoryTask = "failed to deserialize raw history task from task queue"
    52  	// ErrMsgDeserializeHistoryTask is returned when the history task cannot be deserialized from the task queue. This
    53  	// error is returned when the blob inside the raw task cannot be deserialized.
    54  	//  Raw Task (a proto):
    55  	//	- ShardID
    56  	//	- Blob (a serialized task) <-- when this cannot be deserialized
    57  	ErrMsgDeserializeHistoryTask = "failed to deserialize history task blob"
    58  )
    59  
    60  var (
    61  	ErrReadTasksNonPositivePageSize = errors.New("page size to read history tasks must be positive")
    62  	ErrHistoryTaskBlobIsNil         = errors.New("history task from queue has nil blob")
    63  	ErrEnqueueTaskRequestTaskIsNil  = errors.New("enqueue task request task is nil")
    64  	ErrQueueAlreadyExists           = errors.New("queue already exists")
    65  	ErrShardIDInvalid               = errors.New("shard ID must be greater than 0")
    66  )
    67  
    68  func NewHistoryTaskQueueManager(queue QueueV2) *HistoryTaskQueueManagerImpl {
    69  	return &HistoryTaskQueueManagerImpl{
    70  		queue:      queue,
    71  		serializer: serialization.NewTaskSerializer(),
    72  	}
    73  }
    74  
    75  func (m *HistoryTaskQueueManagerImpl) EnqueueTask(
    76  	ctx context.Context,
    77  	request *EnqueueTaskRequest,
    78  ) (*EnqueueTaskResponse, error) {
    79  	if request.Task == nil {
    80  		return nil, ErrEnqueueTaskRequestTaskIsNil
    81  	}
    82  	blob, err := m.serializer.SerializeTask(request.Task)
    83  	if err != nil {
    84  		return nil, fmt.Errorf("%v: %w", ErrMsgSerializeTaskToEnqueue, err)
    85  	}
    86  	if request.SourceShardID <= 0 {
    87  		return nil, fmt.Errorf("%w: shardID = %d", ErrShardIDInvalid, request.SourceShardID)
    88  	}
    89  
    90  	taskCategory := request.Task.GetCategory()
    91  	task := persistencespb.HistoryTask{
    92  		ShardId: int32(request.SourceShardID),
    93  		Blob:    blob,
    94  	}
    95  	taskBytes, _ := task.Marshal()
    96  	blob = &commonpb.DataBlob{
    97  		EncodingType: enums.ENCODING_TYPE_PROTO3,
    98  		Data:         taskBytes,
    99  	}
   100  	queueKey := QueueKey{
   101  		QueueType:     request.QueueType,
   102  		Category:      taskCategory,
   103  		SourceCluster: request.SourceCluster,
   104  		TargetCluster: request.TargetCluster,
   105  	}
   106  
   107  	message, err := m.queue.EnqueueMessage(ctx, &InternalEnqueueMessageRequest{
   108  		QueueType: request.QueueType,
   109  		QueueName: queueKey.GetQueueName(),
   110  		Blob:      blob,
   111  	})
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  
   116  	return &EnqueueTaskResponse{
   117  		Metadata: message.Metadata,
   118  	}, nil
   119  }
   120  
   121  // ReadRawTasks returns a page of "raw" tasks from the queue. Here's a quick disambiguation of the different types of
   122  // tasks:
   123  //
   124  //   - [go.temporal.io/server/api/history/v1.Task]: the proto that is serialized and stored in the database which
   125  //     contains a shard ID and a blob of the serialized history task. This is also called a "raw" task.
   126  //   - [go.temporal.io/server/service/history/tasks.Task]: the interface that is implemented by all history tasks.
   127  //     This is the primary type used in code to represent a history task since it is the most structured.
   128  func (m *HistoryTaskQueueManagerImpl) ReadRawTasks(
   129  	ctx context.Context,
   130  	request *ReadRawTasksRequest,
   131  ) (*ReadRawTasksResponse, error) {
   132  	if request.PageSize <= 0 {
   133  		return nil, fmt.Errorf("%w: %v", ErrReadTasksNonPositivePageSize, request.PageSize)
   134  	}
   135  
   136  	response, err := m.queue.ReadMessages(ctx, &InternalReadMessagesRequest{
   137  		QueueType:     request.QueueKey.QueueType,
   138  		QueueName:     request.QueueKey.GetQueueName(),
   139  		PageSize:      request.PageSize,
   140  		NextPageToken: request.NextPageToken,
   141  	})
   142  	if err != nil {
   143  		return nil, err
   144  	}
   145  
   146  	responseTasks := make([]RawHistoryTask, len(response.Messages))
   147  	for i, message := range response.Messages {
   148  		var task persistencespb.HistoryTask
   149  		err := serialization.Proto3Decode(message.Data.Data, message.Data.EncodingType, &task)
   150  		if err != nil {
   151  			return nil, fmt.Errorf("%v: %w", ErrMsgDeserializeRawHistoryTask, err)
   152  		}
   153  		responseTasks[i].MessageMetadata = message.MetaData
   154  		responseTasks[i].Payload = &task
   155  	}
   156  
   157  	return &ReadRawTasksResponse{
   158  		Tasks:         responseTasks,
   159  		NextPageToken: response.NextPageToken,
   160  	}, nil
   161  }
   162  
   163  // ReadTasks is a convenience method on top of ReadRawTasks that deserializes the tasks into the [tasks.Task] type.
   164  func (m *HistoryTaskQueueManagerImpl) ReadTasks(ctx context.Context, request *ReadTasksRequest) (*ReadTasksResponse, error) {
   165  	response, err := m.ReadRawTasks(ctx, request)
   166  	if err != nil {
   167  		return nil, err
   168  	}
   169  
   170  	resTasks := make([]HistoryTask, len(response.Tasks))
   171  
   172  	for i, rawTask := range response.Tasks {
   173  		blob := rawTask.Payload.Blob
   174  		if blob == nil {
   175  			return nil, serialization.NewDeserializationError(enums.ENCODING_TYPE_PROTO3, ErrHistoryTaskBlobIsNil)
   176  		}
   177  
   178  		task, err := m.serializer.DeserializeTask(request.QueueKey.Category, blob)
   179  		if err != nil {
   180  			return nil, fmt.Errorf("%v: %w", ErrMsgDeserializeHistoryTask, err)
   181  		}
   182  
   183  		resTasks[i] = HistoryTask{
   184  			MessageMetadata: rawTask.MessageMetadata,
   185  			Task:            task,
   186  		}
   187  	}
   188  
   189  	return &ReadTasksResponse{
   190  		Tasks:         resTasks,
   191  		NextPageToken: response.NextPageToken,
   192  	}, nil
   193  }
   194  
   195  func (m *HistoryTaskQueueManagerImpl) CreateQueue(
   196  	ctx context.Context,
   197  	request *CreateQueueRequest,
   198  ) (*CreateQueueResponse, error) {
   199  	_, err := m.queue.CreateQueue(ctx, &InternalCreateQueueRequest{
   200  		QueueType: request.QueueKey.QueueType,
   201  		QueueName: request.QueueKey.GetQueueName(),
   202  	})
   203  	if err != nil {
   204  		return nil, err
   205  	}
   206  	return &CreateQueueResponse{}, nil
   207  }
   208  
   209  func (m *HistoryTaskQueueManagerImpl) DeleteTasks(
   210  	ctx context.Context,
   211  	request *DeleteTasksRequest,
   212  ) (*DeleteTasksResponse, error) {
   213  	resp, err := m.queue.RangeDeleteMessages(ctx, &InternalRangeDeleteMessagesRequest{
   214  		QueueType:                   request.QueueKey.QueueType,
   215  		QueueName:                   request.QueueKey.GetQueueName(),
   216  		InclusiveMaxMessageMetadata: request.InclusiveMaxMessageMetadata,
   217  	})
   218  	if err != nil {
   219  		return nil, err
   220  	}
   221  	return &DeleteTasksResponse{MessagesDeleted: resp.MessagesDeleted}, nil
   222  }
   223  
   224  func (m HistoryTaskQueueManagerImpl) ListQueues(
   225  	ctx context.Context,
   226  	request *ListQueuesRequest,
   227  ) (*ListQueuesResponse, error) {
   228  	resp, err := m.queue.ListQueues(ctx, &InternalListQueuesRequest{
   229  		QueueType:     request.QueueType,
   230  		PageSize:      request.PageSize,
   231  		NextPageToken: request.NextPageToken,
   232  	})
   233  	if err != nil {
   234  		return nil, err
   235  	}
   236  	return &ListQueuesResponse{
   237  		Queues:        resp.Queues,
   238  		NextPageToken: resp.NextPageToken,
   239  	}, nil
   240  }
   241  
   242  // combineUnique combines the given strings into a single string by hashing the length of each string and the string
   243  // itself. This is used to generate a unique suffix for the queue name.
   244  func combineUnique(strs ...string) string {
   245  	h := sha256.New()
   246  	for _, str := range strs {
   247  		b := sha256.Sum256([]byte(str))
   248  		_, _ = h.Write(b[:])
   249  	}
   250  	return base64.StdEncoding.EncodeToString(h.Sum(nil))
   251  }
   252  
   253  func (k QueueKey) GetQueueName() string {
   254  	hash := combineUnique(k.SourceCluster, k.TargetCluster)[:clusterNamesHashSuffixLength]
   255  	return fmt.Sprintf("%d_%s_%s_%s", k.Category.ID(), k.SourceCluster, k.TargetCluster, hash)
   256  }