go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/server/tq/doc.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package tq provides a task queue implementation on top of Cloud Tasks.
    16  //
    17  // It exposes a high-level API that operates with proto messages and hides
    18  // gory details such as serialization, routing, authentication, etc.
    19  //
    20  // # Transactional tasks
    21  //
    22  // Tasks can be submitted as part of a database transaction. This is controlled
    23  // by Kind field of TaskClass. A transactional task is guaranteed to be
    24  // delivered (at least once) if and only if the database transaction was
    25  // committed successfully. The are some prerequisites for using this mechanism.
    26  //
    27  // First, the sweeper must be running somewhere. The sweeper is responsible for
    28  // discovering tasks that were successfully committed into the database, but
    29  // were failed to be dispatched to Cloud Tasks (for example if the client that
    30  // was submitting the task crashed right after committing the transaction). The
    31  // sweeper can run either as a standalone service (the most convenient option
    32  // for Kubernetes deployments) or as a cron job (the most convenient option for
    33  // Appengine deployments).
    34  //
    35  // Second, the core server/tq library needs to "know" how to talk to the
    36  // database that implements transactions. This is achieved by blank-importing
    37  // a corresponding package.
    38  //
    39  // For Datastore:
    40  //
    41  //	import _ "go.chromium.org/luci/server/tq/txn/datastore"
    42  //
    43  // For Spanner:
    44  //
    45  //	import _ "go.chromium.org/luci/server/tq/txn/spanner"
    46  //
    47  // The exact location of the import doesn't matter as long as the package is
    48  // present in the import tree of the binary. If your tests use transactional
    49  // tasks, they'll need to import the corresponding packages as well.
    50  //
    51  // # Enabling the sweeper on Appengine
    52  //
    53  // In cron.yaml:
    54  //   - url: /internal/tasks/c/sweep
    55  //     schedule: every 1 minutes
    56  //
    57  // In queue.yaml:
    58  //   - name: tq-sweep
    59  //     rate: 500/s
    60  //
    61  // # Using the sweeper with Spanner
    62  //
    63  // You need to Create below tables in your database:
    64  //
    65  //	CREATE TABLE TQReminders (
    66  //	  ID STRING(MAX) NOT NULL,
    67  //	  FreshUntil TIMESTAMP NOT NULL,
    68  //	  Payload BYTES(102400) NOT NULL,
    69  //	) PRIMARY KEY (ID ASC);
    70  //
    71  //	CREATE TABLE TQLeases (
    72  //	  SectionID STRING(MAX) NOT NULL,
    73  //	  LeaseID INT64 NOT NULL,
    74  //	  SerializedParts ARRAY<STRING(MAX)>,
    75  //	  ExpiresAt TIMESTAMP NOT NULL,
    76  //	) PRIMARY KEY (SectionID ASC, LeaseID ASC);
    77  package tq