go.temporal.io/server@v1.23.0/common/persistence/history_task_queue_manager.go (about) 1 // The MIT License 2 // 3 // Copyright (c) 2020 Temporal Technologies Inc. All rights reserved. 4 // 5 // Copyright (c) 2020 Uber Technologies, Inc. 6 // 7 // Permission is hereby granted, free of charge, to any person obtaining a copy 8 // of this software and associated documentation files (the "Software"), to deal 9 // in the Software without restriction, including without limitation the rights 10 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 // copies of the Software, and to permit persons to whom the Software is 12 // furnished to do so, subject to the following conditions: 13 // 14 // The above copyright notice and this permission notice shall be included in 15 // all copies or substantial portions of the Software. 16 // 17 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 // THE SOFTWARE. 24 25 package persistence 26 27 import ( 28 "context" 29 "crypto/sha256" 30 "encoding/base64" 31 "errors" 32 "fmt" 33 34 commonpb "go.temporal.io/api/common/v1" 35 "go.temporal.io/api/enums/v1" 36 persistencespb "go.temporal.io/server/api/persistence/v1" 37 "go.temporal.io/server/common/persistence/serialization" 38 ) 39 40 const ( 41 // clusterNamesHashSuffixLength is the number of characters to use from the hash of the cluster names when forming 42 // the queue name. This is used to avoid name collisions when a cluster name contains the separator character. 43 clusterNamesHashSuffixLength = 8 44 45 ErrMsgSerializeTaskToEnqueue = "failed to serialize history task for task queue" 46 // ErrMsgDeserializeRawHistoryTask is returned when the raw task cannot be deserialized from the task queue. This error 47 // is returned when this whole top-level proto cannot be deserialized. 48 // Raw Task (a proto): <-- when this cannot be deserialized 49 // - ShardID 50 // - Blob (a serialized task) 51 ErrMsgDeserializeRawHistoryTask = "failed to deserialize raw history task from task queue" 52 // ErrMsgDeserializeHistoryTask is returned when the history task cannot be deserialized from the task queue. This 53 // error is returned when the blob inside the raw task cannot be deserialized. 54 // Raw Task (a proto): 55 // - ShardID 56 // - Blob (a serialized task) <-- when this cannot be deserialized 57 ErrMsgDeserializeHistoryTask = "failed to deserialize history task blob" 58 ) 59 60 var ( 61 ErrReadTasksNonPositivePageSize = errors.New("page size to read history tasks must be positive") 62 ErrHistoryTaskBlobIsNil = errors.New("history task from queue has nil blob") 63 ErrEnqueueTaskRequestTaskIsNil = errors.New("enqueue task request task is nil") 64 ErrQueueAlreadyExists = errors.New("queue already exists") 65 ErrShardIDInvalid = errors.New("shard ID must be greater than 0") 66 ) 67 68 func NewHistoryTaskQueueManager(queue QueueV2) *HistoryTaskQueueManagerImpl { 69 return &HistoryTaskQueueManagerImpl{ 70 queue: queue, 71 serializer: serialization.NewTaskSerializer(), 72 } 73 } 74 75 func (m *HistoryTaskQueueManagerImpl) EnqueueTask( 76 ctx context.Context, 77 request *EnqueueTaskRequest, 78 ) (*EnqueueTaskResponse, error) { 79 if request.Task == nil { 80 return nil, ErrEnqueueTaskRequestTaskIsNil 81 } 82 blob, err := m.serializer.SerializeTask(request.Task) 83 if err != nil { 84 return nil, fmt.Errorf("%v: %w", ErrMsgSerializeTaskToEnqueue, err) 85 } 86 if request.SourceShardID <= 0 { 87 return nil, fmt.Errorf("%w: shardID = %d", ErrShardIDInvalid, request.SourceShardID) 88 } 89 90 taskCategory := request.Task.GetCategory() 91 task := persistencespb.HistoryTask{ 92 ShardId: int32(request.SourceShardID), 93 Blob: blob, 94 } 95 taskBytes, _ := task.Marshal() 96 blob = &commonpb.DataBlob{ 97 EncodingType: enums.ENCODING_TYPE_PROTO3, 98 Data: taskBytes, 99 } 100 queueKey := QueueKey{ 101 QueueType: request.QueueType, 102 Category: taskCategory, 103 SourceCluster: request.SourceCluster, 104 TargetCluster: request.TargetCluster, 105 } 106 107 message, err := m.queue.EnqueueMessage(ctx, &InternalEnqueueMessageRequest{ 108 QueueType: request.QueueType, 109 QueueName: queueKey.GetQueueName(), 110 Blob: blob, 111 }) 112 if err != nil { 113 return nil, err 114 } 115 116 return &EnqueueTaskResponse{ 117 Metadata: message.Metadata, 118 }, nil 119 } 120 121 // ReadRawTasks returns a page of "raw" tasks from the queue. Here's a quick disambiguation of the different types of 122 // tasks: 123 // 124 // - [go.temporal.io/server/api/history/v1.Task]: the proto that is serialized and stored in the database which 125 // contains a shard ID and a blob of the serialized history task. This is also called a "raw" task. 126 // - [go.temporal.io/server/service/history/tasks.Task]: the interface that is implemented by all history tasks. 127 // This is the primary type used in code to represent a history task since it is the most structured. 128 func (m *HistoryTaskQueueManagerImpl) ReadRawTasks( 129 ctx context.Context, 130 request *ReadRawTasksRequest, 131 ) (*ReadRawTasksResponse, error) { 132 if request.PageSize <= 0 { 133 return nil, fmt.Errorf("%w: %v", ErrReadTasksNonPositivePageSize, request.PageSize) 134 } 135 136 response, err := m.queue.ReadMessages(ctx, &InternalReadMessagesRequest{ 137 QueueType: request.QueueKey.QueueType, 138 QueueName: request.QueueKey.GetQueueName(), 139 PageSize: request.PageSize, 140 NextPageToken: request.NextPageToken, 141 }) 142 if err != nil { 143 return nil, err 144 } 145 146 responseTasks := make([]RawHistoryTask, len(response.Messages)) 147 for i, message := range response.Messages { 148 var task persistencespb.HistoryTask 149 err := serialization.Proto3Decode(message.Data.Data, message.Data.EncodingType, &task) 150 if err != nil { 151 return nil, fmt.Errorf("%v: %w", ErrMsgDeserializeRawHistoryTask, err) 152 } 153 responseTasks[i].MessageMetadata = message.MetaData 154 responseTasks[i].Payload = &task 155 } 156 157 return &ReadRawTasksResponse{ 158 Tasks: responseTasks, 159 NextPageToken: response.NextPageToken, 160 }, nil 161 } 162 163 // ReadTasks is a convenience method on top of ReadRawTasks that deserializes the tasks into the [tasks.Task] type. 164 func (m *HistoryTaskQueueManagerImpl) ReadTasks(ctx context.Context, request *ReadTasksRequest) (*ReadTasksResponse, error) { 165 response, err := m.ReadRawTasks(ctx, request) 166 if err != nil { 167 return nil, err 168 } 169 170 resTasks := make([]HistoryTask, len(response.Tasks)) 171 172 for i, rawTask := range response.Tasks { 173 blob := rawTask.Payload.Blob 174 if blob == nil { 175 return nil, serialization.NewDeserializationError(enums.ENCODING_TYPE_PROTO3, ErrHistoryTaskBlobIsNil) 176 } 177 178 task, err := m.serializer.DeserializeTask(request.QueueKey.Category, blob) 179 if err != nil { 180 return nil, fmt.Errorf("%v: %w", ErrMsgDeserializeHistoryTask, err) 181 } 182 183 resTasks[i] = HistoryTask{ 184 MessageMetadata: rawTask.MessageMetadata, 185 Task: task, 186 } 187 } 188 189 return &ReadTasksResponse{ 190 Tasks: resTasks, 191 NextPageToken: response.NextPageToken, 192 }, nil 193 } 194 195 func (m *HistoryTaskQueueManagerImpl) CreateQueue( 196 ctx context.Context, 197 request *CreateQueueRequest, 198 ) (*CreateQueueResponse, error) { 199 _, err := m.queue.CreateQueue(ctx, &InternalCreateQueueRequest{ 200 QueueType: request.QueueKey.QueueType, 201 QueueName: request.QueueKey.GetQueueName(), 202 }) 203 if err != nil { 204 return nil, err 205 } 206 return &CreateQueueResponse{}, nil 207 } 208 209 func (m *HistoryTaskQueueManagerImpl) DeleteTasks( 210 ctx context.Context, 211 request *DeleteTasksRequest, 212 ) (*DeleteTasksResponse, error) { 213 resp, err := m.queue.RangeDeleteMessages(ctx, &InternalRangeDeleteMessagesRequest{ 214 QueueType: request.QueueKey.QueueType, 215 QueueName: request.QueueKey.GetQueueName(), 216 InclusiveMaxMessageMetadata: request.InclusiveMaxMessageMetadata, 217 }) 218 if err != nil { 219 return nil, err 220 } 221 return &DeleteTasksResponse{MessagesDeleted: resp.MessagesDeleted}, nil 222 } 223 224 func (m HistoryTaskQueueManagerImpl) ListQueues( 225 ctx context.Context, 226 request *ListQueuesRequest, 227 ) (*ListQueuesResponse, error) { 228 resp, err := m.queue.ListQueues(ctx, &InternalListQueuesRequest{ 229 QueueType: request.QueueType, 230 PageSize: request.PageSize, 231 NextPageToken: request.NextPageToken, 232 }) 233 if err != nil { 234 return nil, err 235 } 236 return &ListQueuesResponse{ 237 Queues: resp.Queues, 238 NextPageToken: resp.NextPageToken, 239 }, nil 240 } 241 242 // combineUnique combines the given strings into a single string by hashing the length of each string and the string 243 // itself. This is used to generate a unique suffix for the queue name. 244 func combineUnique(strs ...string) string { 245 h := sha256.New() 246 for _, str := range strs { 247 b := sha256.Sum256([]byte(str)) 248 _, _ = h.Write(b[:]) 249 } 250 return base64.StdEncoding.EncodeToString(h.Sum(nil)) 251 } 252 253 func (k QueueKey) GetQueueName() string { 254 hash := combineUnique(k.SourceCluster, k.TargetCluster)[:clusterNamesHashSuffixLength] 255 return fmt.Sprintf("%d_%s_%s_%s", k.Category.ID(), k.SourceCluster, k.TargetCluster, hash) 256 }