github.com/thiagoyeds/go-cloud@v0.26.0/pubsub/gcppubsub/gcppubsub.go (about) 1 // Copyright 2018 The Go Cloud Development Kit Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 // Package gcppubsub provides a pubsub implementation that uses GCP 16 // PubSub. Use OpenTopic to construct a *pubsub.Topic, and/or OpenSubscription 17 // to construct a *pubsub.Subscription. 18 // 19 // URLs 20 // 21 // For pubsub.OpenTopic and pubsub.OpenSubscription, gcppubsub registers 22 // for the scheme "gcppubsub". 23 // The default URL opener will creating a connection using use default 24 // credentials from the environment, as described in 25 // https://cloud.google.com/docs/authentication/production. 26 // To customize the URL opener, or for more details on the URL format, 27 // see URLOpener. 28 // See https://gocloud.dev/concepts/urls/ for background information. 29 // 30 // GCP Pub/Sub emulator is supported as per https://cloud.google.com/pubsub/docs/emulator 31 // So, when environment variable 'PUBSUB_EMULATOR_HOST' is set 32 // driver connects to the specified emulator host by default. 33 // 34 // Message Delivery Semantics 35 // 36 // GCP Pub/Sub supports at-least-once semantics; applications must 37 // call Message.Ack after processing a message, or it will be redelivered. 38 // See https://godoc.org/gocloud.dev/pubsub#hdr-At_most_once_and_At_least_once_Delivery 39 // for more background. 40 // 41 // As 42 // 43 // gcppubsub exposes the following types for As: 44 // - Topic: *raw.PublisherClient 45 // - Subscription: *raw.SubscriberClient 46 // - Message.BeforeSend: *pb.PubsubMessage 47 // - Message.AfterSend: *string for the pb.PublishResponse.MessageIds entry corresponding to the message. 48 // - Message: *pb.PubsubMessage 49 // - Error: *google.golang.org/grpc/status.Status 50 package gcppubsub // import "gocloud.dev/pubsub/gcppubsub" 51 52 import ( 53 "context" 54 "fmt" 55 "net/url" 56 "os" 57 "path" 58 "regexp" 59 "strconv" 60 "strings" 61 "sync" 62 "time" 63 64 raw "cloud.google.com/go/pubsub/apiv1" 65 "github.com/google/wire" 66 "gocloud.dev/gcerrors" 67 "gocloud.dev/gcp" 68 "gocloud.dev/internal/gcerr" 69 "gocloud.dev/internal/useragent" 70 "gocloud.dev/pubsub" 71 "gocloud.dev/pubsub/batcher" 72 "gocloud.dev/pubsub/driver" 73 "google.golang.org/api/option" 74 pb "google.golang.org/genproto/googleapis/pubsub/v1" 75 "google.golang.org/grpc" 76 "google.golang.org/grpc/credentials" 77 "google.golang.org/grpc/credentials/oauth" 78 "google.golang.org/grpc/status" 79 ) 80 81 var endPoint = "pubsub.googleapis.com:443" 82 83 var sendBatcherOpts = &batcher.Options{ 84 MaxBatchSize: 1000, // The PubSub service limits the number of messages in a single Publish RPC 85 MaxHandlers: 2, 86 // The PubSub service limits the size of the request body in a single Publish RPC. 87 // The limit is currently documented as "10MB (total size)" and "10MB (data field)" per message. 88 // We are enforcing 9MiB to give ourselves some headroom for message attributes since those 89 // are currently not considered when computing the byte size of a message. 90 MaxBatchByteSize: 9 * 1024 * 1024, 91 } 92 93 var defaultRecvBatcherOpts = &batcher.Options{ 94 // GCP Pub/Sub returns at most 1000 messages per RPC. 95 MaxBatchSize: 1000, 96 MaxHandlers: 10, 97 } 98 99 var ackBatcherOpts = &batcher.Options{ 100 // The PubSub service limits the size of Acknowledge/ModifyAckDeadline RPCs. 101 // (E.g., "Request payload size exceeds the limit: 524288 bytes."). 102 MaxBatchSize: 1000, 103 MaxHandlers: 2, 104 } 105 106 func init() { 107 o := new(lazyCredsOpener) 108 pubsub.DefaultURLMux().RegisterTopic(Scheme, o) 109 pubsub.DefaultURLMux().RegisterSubscription(Scheme, o) 110 } 111 112 // Set holds Wire providers for this package. 113 var Set = wire.NewSet( 114 Dial, 115 PublisherClient, 116 SubscriberClient, 117 wire.Struct(new(SubscriptionOptions)), 118 wire.Struct(new(TopicOptions)), 119 wire.Struct(new(URLOpener), "Conn", "TopicOptions", "SubscriptionOptions"), 120 ) 121 122 // lazyCredsOpener obtains Application Default Credentials on the first call 123 // to OpenTopicURL/OpenSubscriptionURL. 124 type lazyCredsOpener struct { 125 init sync.Once 126 opener *URLOpener 127 err error 128 } 129 130 func (o *lazyCredsOpener) defaultConn(ctx context.Context) (*URLOpener, error) { 131 o.init.Do(func() { 132 var conn *grpc.ClientConn 133 var err error 134 if e := os.Getenv("PUBSUB_EMULATOR_HOST"); e != "" { 135 // Connect to the GCP pubsub emulator by overriding the default endpoint 136 // if the 'PUBSUB_EMULATOR_HOST' environment variable is set. 137 // Check https://cloud.google.com/pubsub/docs/emulator for more info. 138 endPoint = e 139 conn, err = dialEmulator(ctx, e) 140 if err != nil { 141 o.err = err 142 return 143 } 144 } else { 145 creds, err := gcp.DefaultCredentials(ctx) 146 if err != nil { 147 o.err = err 148 return 149 } 150 151 conn, _, err = Dial(ctx, creds.TokenSource) 152 if err != nil { 153 o.err = err 154 return 155 } 156 } 157 o.opener = &URLOpener{Conn: conn} 158 }) 159 return o.opener, o.err 160 } 161 162 func (o *lazyCredsOpener) OpenTopicURL(ctx context.Context, u *url.URL) (*pubsub.Topic, error) { 163 opener, err := o.defaultConn(ctx) 164 if err != nil { 165 return nil, fmt.Errorf("open topic %v: failed to open default connection: %v", u, err) 166 } 167 return opener.OpenTopicURL(ctx, u) 168 } 169 170 func (o *lazyCredsOpener) OpenSubscriptionURL(ctx context.Context, u *url.URL) (*pubsub.Subscription, error) { 171 opener, err := o.defaultConn(ctx) 172 if err != nil { 173 return nil, fmt.Errorf("open subscription %v: failed to open default connection: %v", u, err) 174 } 175 return opener.OpenSubscriptionURL(ctx, u) 176 } 177 178 // Scheme is the URL scheme gcppubsub registers its URLOpeners under on pubsub.DefaultMux. 179 const Scheme = "gcppubsub" 180 181 // URLOpener opens GCP Pub/Sub URLs like "gcppubsub://projects/myproject/topics/mytopic" for 182 // topics or "gcppubsub://projects/myproject/subscriptions/mysub" for subscriptions. 183 // 184 // The shortened forms "gcppubsub://myproject/mytopic" for topics or 185 // "gcppubsub://myproject/mysub" for subscriptions are also supported. 186 // 187 // The following query parameters are supported: 188 // 189 // - max_recv_batch_size: sets SubscriptionOptions.MaxBatchSize 190 // 191 // Currently their use is limited to subscribers. 192 type URLOpener struct { 193 // Conn must be set to a non-nil ClientConn authenticated with 194 // Cloud Pub/Sub scope or equivalent. 195 Conn *grpc.ClientConn 196 197 // TopicOptions specifies the options to pass to OpenTopic. 198 TopicOptions TopicOptions 199 // SubscriptionOptions specifies the options to pass to OpenSubscription. 200 SubscriptionOptions SubscriptionOptions 201 } 202 203 // OpenTopicURL opens a pubsub.Topic based on u. 204 func (o *URLOpener) OpenTopicURL(ctx context.Context, u *url.URL) (*pubsub.Topic, error) { 205 for param := range u.Query() { 206 return nil, fmt.Errorf("open topic %v: invalid query parameter %q", u, param) 207 } 208 pc, err := PublisherClient(ctx, o.Conn) 209 if err != nil { 210 return nil, err 211 } 212 topicPath := path.Join(u.Host, u.Path) 213 if topicPathRE.MatchString(topicPath) { 214 return OpenTopicByPath(pc, topicPath, &o.TopicOptions) 215 } 216 // Shortened form? 217 topicName := strings.TrimPrefix(u.Path, "/") 218 return OpenTopic(pc, gcp.ProjectID(u.Host), topicName, &o.TopicOptions), nil 219 } 220 221 // OpenSubscriptionURL opens a pubsub.Subscription based on u. 222 func (o *URLOpener) OpenSubscriptionURL(ctx context.Context, u *url.URL) (*pubsub.Subscription, error) { 223 // Set subscription options to use defaults 224 opts := o.SubscriptionOptions 225 226 for param, value := range u.Query() { 227 switch param { 228 case "max_recv_batch_size": 229 maxBatchSize, err := queryParameterInt(value) 230 if err != nil { 231 return nil, fmt.Errorf("open subscription %v: invalid query parameter %q: %v", u, param, err) 232 } 233 234 if maxBatchSize <= 0 || maxBatchSize > 1000 { 235 return nil, fmt.Errorf("open subscription %v: invalid query parameter %q: must be between 1 and 1000", u, param) 236 } 237 238 opts.MaxBatchSize = maxBatchSize 239 default: 240 return nil, fmt.Errorf("open subscription %v: invalid query parameter %q", u, param) 241 } 242 } 243 sc, err := SubscriberClient(ctx, o.Conn) 244 if err != nil { 245 return nil, err 246 } 247 subPath := path.Join(u.Host, u.Path) 248 if subscriptionPathRE.MatchString(subPath) { 249 return OpenSubscriptionByPath(sc, subPath, &opts) 250 } 251 // Shortened form? 252 subName := strings.TrimPrefix(u.Path, "/") 253 return OpenSubscription(sc, gcp.ProjectID(u.Host), subName, &opts), nil 254 } 255 256 type topic struct { 257 path string 258 client *raw.PublisherClient 259 } 260 261 // Dial opens a gRPC connection to the GCP Pub Sub API. 262 // 263 // The second return value is a function that can be called to clean up 264 // the connection opened by Dial. 265 func Dial(ctx context.Context, ts gcp.TokenSource) (*grpc.ClientConn, func(), error) { 266 conn, err := grpc.DialContext(ctx, endPoint, 267 grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")), 268 grpc.WithPerRPCCredentials(oauth.TokenSource{TokenSource: ts}), 269 // The default message size limit for gRPC is 4MB, while GCP 270 // PubSub supports messages up to 10MB. Aside from the message itself 271 // there is also other data in the gRPC response, bringing the maximum 272 // response size above 10MB. Tell gRPC to support up to 11MB. 273 // https://github.com/googleapis/google-cloud-node/issues/1991 274 grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024*1024*11)), 275 useragent.GRPCDialOption("pubsub"), 276 ) 277 278 if err != nil { 279 return nil, nil, err 280 } 281 return conn, func() { conn.Close() }, nil 282 } 283 284 // dialEmulator opens a gRPC connection to the GCP Pub Sub API. 285 func dialEmulator(ctx context.Context, e string) (*grpc.ClientConn, error) { 286 conn, err := grpc.DialContext(ctx, e, grpc.WithInsecure(), useragent.GRPCDialOption("pubsub")) 287 if err != nil { 288 return nil, err 289 } 290 return conn, nil 291 } 292 293 // PublisherClient returns a *raw.PublisherClient that can be used in OpenTopic. 294 func PublisherClient(ctx context.Context, conn *grpc.ClientConn) (*raw.PublisherClient, error) { 295 return raw.NewPublisherClient(ctx, option.WithGRPCConn(conn)) 296 } 297 298 // SubscriberClient returns a *raw.SubscriberClient that can be used in OpenSubscription. 299 func SubscriberClient(ctx context.Context, conn *grpc.ClientConn) (*raw.SubscriberClient, error) { 300 return raw.NewSubscriberClient(ctx, option.WithGRPCConn(conn)) 301 } 302 303 // TopicOptions will contain configuration for topics. 304 type TopicOptions struct{} 305 306 // OpenTopic returns a *pubsub.Topic backed by an existing GCP PubSub topic 307 // in the given projectID. topicName is the last part of the full topic 308 // path, e.g., "foo" from "projects/<projectID>/topic/foo". 309 // See the package documentation for an example. 310 func OpenTopic(client *raw.PublisherClient, projectID gcp.ProjectID, topicName string, opts *TopicOptions) *pubsub.Topic { 311 topicPath := fmt.Sprintf("projects/%s/topics/%s", projectID, topicName) 312 return pubsub.NewTopic(openTopic(client, topicPath), sendBatcherOpts) 313 } 314 315 var topicPathRE = regexp.MustCompile("^projects/.+/topics/.+$") 316 317 // OpenTopicByPath returns a *pubsub.Topic backed by an existing GCP PubSub 318 // topic. topicPath must be of the form "projects/<projectID>/topic/<topic>". 319 // See the package documentation for an example. 320 func OpenTopicByPath(client *raw.PublisherClient, topicPath string, opts *TopicOptions) (*pubsub.Topic, error) { 321 if !topicPathRE.MatchString(topicPath) { 322 return nil, fmt.Errorf("invalid topicPath %q; must match %v", topicPath, topicPathRE) 323 } 324 return pubsub.NewTopic(openTopic(client, topicPath), sendBatcherOpts), nil 325 } 326 327 // openTopic returns the driver for OpenTopic. This function exists so the test 328 // harness can get the driver interface implementation if it needs to. 329 func openTopic(client *raw.PublisherClient, topicPath string) driver.Topic { 330 return &topic{topicPath, client} 331 } 332 333 // SendBatch implements driver.Topic.SendBatch. 334 func (t *topic) SendBatch(ctx context.Context, dms []*driver.Message) error { 335 var ms []*pb.PubsubMessage 336 for _, dm := range dms { 337 psm := &pb.PubsubMessage{Data: dm.Body, Attributes: dm.Metadata} 338 if dm.BeforeSend != nil { 339 asFunc := func(i interface{}) bool { 340 if p, ok := i.(**pb.PubsubMessage); ok { 341 *p = psm 342 return true 343 } 344 return false 345 } 346 if err := dm.BeforeSend(asFunc); err != nil { 347 return err 348 } 349 } 350 ms = append(ms, psm) 351 } 352 req := &pb.PublishRequest{Topic: t.path, Messages: ms} 353 pr, err := t.client.Publish(ctx, req) 354 if err != nil { 355 return err 356 } 357 if len(pr.MessageIds) == len(dms) { 358 for n, dm := range dms { 359 if dm.AfterSend != nil { 360 asFunc := func(i interface{}) bool { 361 if p, ok := i.(*string); ok { 362 *p = pr.MessageIds[n] 363 return true 364 } 365 return false 366 } 367 if err := dm.AfterSend(asFunc); err != nil { 368 return err 369 } 370 } 371 } 372 } 373 return nil 374 } 375 376 // IsRetryable implements driver.Topic.IsRetryable. 377 func (t *topic) IsRetryable(error) bool { 378 // The client handles retries. 379 return false 380 } 381 382 // As implements driver.Topic.As. 383 func (t *topic) As(i interface{}) bool { 384 c, ok := i.(**raw.PublisherClient) 385 if !ok { 386 return false 387 } 388 *c = t.client 389 return true 390 } 391 392 // ErrorAs implements driver.Topic.ErrorAs 393 func (*topic) ErrorAs(err error, i interface{}) bool { 394 return errorAs(err, i) 395 } 396 397 func errorAs(err error, i interface{}) bool { 398 s, ok := status.FromError(err) 399 if !ok { 400 return false 401 } 402 p, ok := i.(**status.Status) 403 if !ok { 404 return false 405 } 406 *p = s 407 return true 408 } 409 410 func (*topic) ErrorCode(err error) gcerrors.ErrorCode { 411 return gcerr.GRPCCode(err) 412 } 413 414 // Close implements driver.Topic.Close. 415 func (*topic) Close() error { return nil } 416 417 type subscription struct { 418 client *raw.SubscriberClient 419 path string 420 options *SubscriptionOptions 421 } 422 423 // SubscriptionOptions will contain configuration for subscriptions. 424 type SubscriptionOptions struct { 425 // MaxBatchSize caps the maximum batch size used when retrieving messages. It defaults to 1000. 426 MaxBatchSize int 427 } 428 429 // OpenSubscription returns a *pubsub.Subscription backed by an existing GCP 430 // PubSub subscription subscriptionName in the given projectID. See the package 431 // documentation for an example. 432 func OpenSubscription(client *raw.SubscriberClient, projectID gcp.ProjectID, subscriptionName string, opts *SubscriptionOptions) *pubsub.Subscription { 433 path := fmt.Sprintf("projects/%s/subscriptions/%s", projectID, subscriptionName) 434 435 dsub := openSubscription(client, path, opts) 436 recvOpts := *defaultRecvBatcherOpts 437 recvOpts.MaxBatchSize = dsub.options.MaxBatchSize 438 return pubsub.NewSubscription(dsub, &recvOpts, ackBatcherOpts) 439 } 440 441 var subscriptionPathRE = regexp.MustCompile("^projects/.+/subscriptions/.+$") 442 443 // OpenSubscriptionByPath returns a *pubsub.Subscription backed by an existing 444 // GCP PubSub subscription. subscriptionPath must be of the form 445 // "projects/<projectID>/subscriptions/<subscription>". 446 // See the package documentation for an example. 447 func OpenSubscriptionByPath(client *raw.SubscriberClient, subscriptionPath string, opts *SubscriptionOptions) (*pubsub.Subscription, error) { 448 if !subscriptionPathRE.MatchString(subscriptionPath) { 449 return nil, fmt.Errorf("invalid subscriptionPath %q; must match %v", subscriptionPath, subscriptionPathRE) 450 } 451 452 dsub := openSubscription(client, subscriptionPath, opts) 453 recvOpts := *defaultRecvBatcherOpts 454 recvOpts.MaxBatchSize = dsub.options.MaxBatchSize 455 return pubsub.NewSubscription(dsub, &recvOpts, ackBatcherOpts), nil 456 } 457 458 // openSubscription returns a driver.Subscription. 459 func openSubscription(client *raw.SubscriberClient, subscriptionPath string, opts *SubscriptionOptions) *subscription { 460 if opts == nil { 461 opts = &SubscriptionOptions{} 462 } 463 464 if opts.MaxBatchSize == 0 { 465 opts.MaxBatchSize = defaultRecvBatcherOpts.MaxBatchSize 466 } 467 468 return &subscription{client, subscriptionPath, opts} 469 } 470 471 // ReceiveBatch implements driver.Subscription.ReceiveBatch. 472 func (s *subscription) ReceiveBatch(ctx context.Context, maxMessages int) ([]*driver.Message, error) { 473 // Whether to ask Pull to return immediately, or wait for some messages to 474 // arrive. If we're making multiple RPCs, we don't want any of them to wait; 475 // we might have gotten messages from one of the other RPCs. 476 // maxMessages will only be high enough to set this to true in high-throughput 477 // situations, so the likelihood of getting 0 messages is small anyway. 478 returnImmediately := maxMessages == s.options.MaxBatchSize 479 480 req := &pb.PullRequest{ 481 Subscription: s.path, 482 ReturnImmediately: returnImmediately, 483 MaxMessages: int32(maxMessages), 484 } 485 resp, err := s.client.Pull(ctx, req) 486 if err != nil { 487 return nil, err 488 } 489 if len(resp.ReceivedMessages) == 0 { 490 // If we did happen to get 0 messages, and we didn't ask the server to wait 491 // for messages, sleep a bit to avoid spinning. 492 if returnImmediately { 493 time.Sleep(100 * time.Millisecond) 494 } 495 return nil, nil 496 } 497 498 ms := make([]*driver.Message, 0, len(resp.ReceivedMessages)) 499 for _, rm := range resp.ReceivedMessages { 500 rmm := rm.Message 501 m := &driver.Message{ 502 LoggableID: rmm.MessageId, 503 Body: rmm.Data, 504 Metadata: rmm.Attributes, 505 AckID: rm.AckId, 506 AsFunc: messageAsFunc(rmm), 507 } 508 ms = append(ms, m) 509 } 510 return ms, nil 511 } 512 513 func messageAsFunc(pm *pb.PubsubMessage) func(interface{}) bool { 514 return func(i interface{}) bool { 515 p, ok := i.(**pb.PubsubMessage) 516 if !ok { 517 return false 518 } 519 *p = pm 520 return true 521 } 522 } 523 524 // SendAcks implements driver.Subscription.SendAcks. 525 func (s *subscription) SendAcks(ctx context.Context, ids []driver.AckID) error { 526 ids2 := make([]string, 0, len(ids)) 527 for _, id := range ids { 528 ids2 = append(ids2, id.(string)) 529 } 530 return s.client.Acknowledge(ctx, &pb.AcknowledgeRequest{Subscription: s.path, AckIds: ids2}) 531 } 532 533 // CanNack implements driver.CanNack. 534 func (s *subscription) CanNack() bool { return true } 535 536 // SendNacks implements driver.Subscription.SendNacks. 537 func (s *subscription) SendNacks(ctx context.Context, ids []driver.AckID) error { 538 ids2 := make([]string, 0, len(ids)) 539 for _, id := range ids { 540 ids2 = append(ids2, id.(string)) 541 } 542 return s.client.ModifyAckDeadline(ctx, &pb.ModifyAckDeadlineRequest{ 543 Subscription: s.path, 544 AckIds: ids2, 545 AckDeadlineSeconds: 0, 546 }) 547 } 548 549 // IsRetryable implements driver.Subscription.IsRetryable. 550 func (s *subscription) IsRetryable(err error) bool { 551 // The client mostly handles retries, but does not 552 // include DeadlineExceeded for some reason. 553 if s.ErrorCode(err) == gcerrors.DeadlineExceeded { 554 return true 555 } 556 return false 557 } 558 559 // As implements driver.Subscription.As. 560 func (s *subscription) As(i interface{}) bool { 561 c, ok := i.(**raw.SubscriberClient) 562 if !ok { 563 return false 564 } 565 *c = s.client 566 return true 567 } 568 569 // ErrorAs implements driver.Subscription.ErrorAs 570 func (*subscription) ErrorAs(err error, i interface{}) bool { 571 return errorAs(err, i) 572 } 573 574 func (*subscription) ErrorCode(err error) gcerrors.ErrorCode { 575 return gcerr.GRPCCode(err) 576 } 577 578 // Close implements driver.Subscription.Close. 579 func (*subscription) Close() error { return nil } 580 581 func queryParameterInt(value []string) (int, error) { 582 if len(value) > 1 { 583 return 0, fmt.Errorf("expected only one parameter value, got: %v", len(value)) 584 } 585 586 return strconv.Atoi(value[0]) 587 }