github.com/argoproj/argo-events@v1.9.1/eventbus/kafka/sensor/kafka_transaction.go (about)

     1  package kafka
     2  
     3  import (
     4  	"github.com/IBM/sarama"
     5  	"go.uber.org/zap"
     6  )
     7  
     8  type KafkaTransaction struct {
     9  	Logger *zap.SugaredLogger
    10  
    11  	// kafka details
    12  	Producer  sarama.AsyncProducer
    13  	GroupName string
    14  	Topic     string
    15  	Partition int32
    16  
    17  	// used to reset the offset and metadata if transaction fails
    18  	ResetOffset   int64
    19  	ResetMetadata string
    20  }
    21  
    22  func (t *KafkaTransaction) Commit(session sarama.ConsumerGroupSession, messages []*sarama.ProducerMessage, offset int64, metadata string) error {
    23  	// No need for a transaction if no messages, just update the
    24  	// offset and metadata
    25  	if len(messages) == 0 {
    26  		session.MarkOffset(t.Topic, t.Partition, offset, metadata)
    27  		session.Commit()
    28  		return nil
    29  	}
    30  
    31  	t.Logger.Infow("Begin transaction",
    32  		zap.String("topic", t.Topic),
    33  		zap.Int32("partition", t.Partition),
    34  		zap.Int("messages", len(messages)))
    35  
    36  	if err := t.Producer.BeginTxn(); err != nil {
    37  		return err
    38  	}
    39  
    40  	for _, msg := range messages {
    41  		t.Producer.Input() <- msg
    42  	}
    43  
    44  	offsets := map[string][]*sarama.PartitionOffsetMetadata{
    45  		t.Topic: {{
    46  			Partition: t.Partition,
    47  			Offset:    offset,
    48  			Metadata:  &metadata,
    49  		}},
    50  	}
    51  
    52  	if err := t.Producer.AddOffsetsToTxn(offsets, t.GroupName); err != nil {
    53  		t.Logger.Errorw("Kafka transaction error", zap.Error(err))
    54  		t.handleTxnError(session, func() error {
    55  			return t.Producer.AddOffsetsToTxn(offsets, t.GroupName)
    56  		})
    57  	}
    58  
    59  	if err := t.Producer.CommitTxn(); err != nil {
    60  		t.Logger.Errorw("Kafka transaction error", zap.Error(err))
    61  		t.handleTxnError(session, func() error {
    62  			return t.Producer.CommitTxn()
    63  		})
    64  	}
    65  
    66  	t.Logger.Infow("Finished transaction",
    67  		zap.String("topic", t.Topic),
    68  		zap.Int32("partition", t.Partition))
    69  
    70  	return nil
    71  }
    72  
    73  func (t *KafkaTransaction) handleTxnError(session sarama.ConsumerGroupSession, defaulthandler func() error) {
    74  	for {
    75  		if t.Producer.TxnStatus()&sarama.ProducerTxnFlagFatalError != 0 {
    76  			// reset current consumer offset to retry consume this record
    77  			session.ResetOffset(t.Topic, t.Partition, t.ResetOffset, t.ResetMetadata)
    78  			// fatal error, need to restart
    79  			t.Logger.Fatal("Message consumer: t.Producer is in a fatal state.")
    80  			return
    81  		}
    82  		if t.Producer.TxnStatus()&sarama.ProducerTxnFlagAbortableError != 0 {
    83  			if err := t.Producer.AbortTxn(); err != nil {
    84  				t.Logger.Errorw("Message consumer: unable to abort transaction.", zap.Error(err))
    85  				continue
    86  			}
    87  			// reset current consumer offset to retry consume this record
    88  			session.ResetOffset(t.Topic, t.Partition, t.ResetOffset, t.ResetMetadata)
    89  			// fatal error, need to restart
    90  			t.Logger.Fatal("Message consumer: t.Producer is in a fatal state, aborted transaction.")
    91  			return
    92  		}
    93  
    94  		// attempt retry
    95  		if err := defaulthandler(); err == nil {
    96  			return
    97  		}
    98  	}
    99  }