github.com/bartle-stripe/trillian@v1.2.1/storage/mysql/queue_batching.go (about)

     1  // +build batched_queue
     2  
     3  // Copyright 2017 Google Inc. All Rights Reserved.
     4  //
     5  // Licensed under the Apache License, Version 2.0 (the "License");
     6  // you may not use this file except in compliance with the License.
     7  // You may obtain a copy of the License at
     8  //
     9  //     http://www.apache.org/licenses/LICENSE-2.0
    10  //
    11  // Unless required by applicable law or agreed to in writing, software
    12  // distributed under the License is distributed on an "AS IS" BASIS,
    13  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  // See the License for the specific language governing permissions and
    15  // limitations under the License.
    16  
    17  package mysql
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"database/sql"
    23  	"encoding/binary"
    24  	"fmt"
    25  	"strings"
    26  	"time"
    27  
    28  	"github.com/golang/glog"
    29  	"github.com/golang/protobuf/ptypes"
    30  	"github.com/google/trillian"
    31  )
    32  
    33  const (
    34  	// If this statement ORDER BY clause is changed refer to the comment in removeSequencedLeaves
    35  	selectQueuedLeavesSQL = `SELECT LeafIdentityHash,MerkleLeafHash,QueueTimestampNanos,QueueID
    36  			FROM Unsequenced
    37  			WHERE TreeID=?
    38  			AND Bucket=0
    39  			AND QueueTimestampNanos<=?
    40  			ORDER BY QueueTimestampNanos,LeafIdentityHash ASC LIMIT ?`
    41  	insertUnsequencedEntrySQL = `INSERT INTO Unsequenced(TreeId,Bucket,LeafIdentityHash,MerkleLeafHash,QueueTimestampNanos,QueueID) VALUES(?,0,?,?,?,?)`
    42  	deleteUnsequencedSQL      = "DELETE FROM Unsequenced WHERE QueueID IN (<placeholder>)"
    43  )
    44  
    45  type dequeuedLeaf []byte
    46  
    47  func dequeueInfo(_ []byte, queueID []byte) dequeuedLeaf {
    48  	return dequeuedLeaf(queueID)
    49  }
    50  
    51  func (t *logTreeTX) dequeueLeaf(rows *sql.Rows) (*trillian.LogLeaf, dequeuedLeaf, error) {
    52  	var leafIDHash []byte
    53  	var merkleHash []byte
    54  	var queueTimestamp int64
    55  	var queueID []byte
    56  
    57  	err := rows.Scan(&leafIDHash, &merkleHash, &queueTimestamp, &queueID)
    58  	if err != nil {
    59  		glog.Warningf("Error scanning work rows: %s", err)
    60  		return nil, nil, err
    61  	}
    62  
    63  	queueTimestampProto, err := ptypes.TimestampProto(time.Unix(0, queueTimestamp))
    64  	if err != nil {
    65  		return nil, dequeuedLeaf{}, fmt.Errorf("got invalid queue timestamp: %v", err)
    66  	}
    67  	// Note: the LeafData and ExtraData being nil here is OK as this is only used by the
    68  	// sequencer. The sequencer only writes to the SequencedLeafData table and the client
    69  	// supplied data was already written to LeafData as part of queueing the leaf.
    70  	leaf := &trillian.LogLeaf{
    71  		LeafIdentityHash: leafIDHash,
    72  		MerkleLeafHash:   merkleHash,
    73  		QueueTimestamp:   queueTimestampProto,
    74  	}
    75  	return leaf, dequeueInfo(leafIDHash, queueID), nil
    76  }
    77  
    78  func generateQueueID(treeID int64, leafIdentityHash []byte, timestamp int64) []byte {
    79  	h := sha256.New()
    80  	b := make([]byte, 10)
    81  	binary.PutVarint(b, treeID)
    82  	h.Write(b)
    83  	b = make([]byte, 10)
    84  	binary.PutVarint(b, timestamp)
    85  	h.Write(b)
    86  	h.Write(leafIdentityHash)
    87  	return h.Sum(nil)
    88  }
    89  
    90  func queueArgs(treeID int64, identityHash []byte, queueTimestamp time.Time) []interface{} {
    91  	timestamp := queueTimestamp.UnixNano()
    92  	return []interface{}{timestamp, generateQueueID(treeID, identityHash, timestamp)}
    93  }
    94  
    95  func (t *logTreeTX) UpdateSequencedLeaves(ctx context.Context, leaves []*trillian.LogLeaf) error {
    96  	querySuffix := []string{}
    97  	args := []interface{}{}
    98  	for _, leaf := range leaves {
    99  		iTimestamp, err := ptypes.Timestamp(leaf.IntegrateTimestamp)
   100  		if err != nil {
   101  			return fmt.Errorf("got invalid integrate timestamp: %v", err)
   102  		}
   103  		querySuffix = append(querySuffix, valuesPlaceholder5)
   104  		args = append(args, t.treeID, leaf.LeafIdentityHash, leaf.MerkleLeafHash, leaf.LeafIndex, iTimestamp.UnixNano())
   105  	}
   106  	result, err := t.tx.ExecContext(ctx, insertSequencedLeafSQL+strings.Join(querySuffix, ","), args...)
   107  	if err != nil {
   108  		glog.Warningf("Failed to update sequenced leaves: %s", err)
   109  	}
   110  	return checkResultOkAndRowCountIs(result, err, int64(len(leaves)))
   111  }
   112  
   113  func (m *mySQLLogStorage) getDeleteUnsequencedStmt(ctx context.Context, num int) (*sql.Stmt, error) {
   114  	return m.getStmt(ctx, deleteUnsequencedSQL, num, "?", "?")
   115  }
   116  
   117  // removeSequencedLeaves removes the passed in leaves slice (which may be
   118  // modified as part of the operation).
   119  func (t *logTreeTX) removeSequencedLeaves(ctx context.Context, queueIDs []dequeuedLeaf) error {
   120  	// Don't need to re-sort because the query ordered by leaf hash. If that changes because
   121  	// the query is expensive then the sort will need to be done here. See comment in
   122  	// QueueLeaves.
   123  	tmpl, err := t.ls.getDeleteUnsequencedStmt(ctx, len(queueIDs))
   124  	if err != nil {
   125  		glog.Warningf("Failed to get delete statement for sequenced work: %s", err)
   126  		return err
   127  	}
   128  	stx := t.tx.StmtContext(ctx, tmpl)
   129  	args := make([]interface{}, len(queueIDs))
   130  	for i, q := range queueIDs {
   131  		args[i] = []byte(q)
   132  	}
   133  	result, err := stx.ExecContext(ctx, args...)
   134  	if err != nil {
   135  		// Error is handled by checkResultOkAndRowCountIs() below
   136  		glog.Warningf("Failed to delete sequenced work: %s", err)
   137  	}
   138  	return checkResultOkAndRowCountIs(result, err, int64(len(queueIDs)))
   139  }