github.com/zuoyebang/bitalostable@v1.0.1-0.20240229032404-e3b99a834294/event.go (about)

     1  // Copyright 2018 The LevelDB-Go and Pebble and Bitalostored Authors. All rights reserved. Use
     2  // of this source code is governed by a BSD-style license that can be found in
     3  // the LICENSE file.
     4  
     5  package bitalostable
     6  
     7  import (
     8  	"strings"
     9  	"time"
    10  
    11  	"github.com/cockroachdb/redact"
    12  	"github.com/zuoyebang/bitalostable/internal/humanize"
    13  	"github.com/zuoyebang/bitalostable/internal/manifest"
    14  )
    15  
    16  // TableInfo exports the manifest.TableInfo type.
    17  type TableInfo = manifest.TableInfo
    18  
    19  func tablesTotalSize(tables []TableInfo) uint64 {
    20  	var size uint64
    21  	for i := range tables {
    22  		size += tables[i].Size
    23  	}
    24  	return size
    25  }
    26  
    27  func formatFileNums(tables []TableInfo) string {
    28  	var buf strings.Builder
    29  	for i := range tables {
    30  		if i > 0 {
    31  			buf.WriteString(" ")
    32  		}
    33  		buf.WriteString(tables[i].FileNum.String())
    34  	}
    35  	return buf.String()
    36  }
    37  
    38  // LevelInfo contains info pertaining to a partificular level.
    39  type LevelInfo struct {
    40  	Level  int
    41  	Tables []TableInfo
    42  }
    43  
    44  func (i LevelInfo) String() string {
    45  	return redact.StringWithoutMarkers(i)
    46  }
    47  
    48  // SafeFormat implements redact.SafeFormatter.
    49  func (i LevelInfo) SafeFormat(w redact.SafePrinter, _ rune) {
    50  	w.Printf("L%d [%s] (%s)", redact.Safe(i.Level), redact.Safe(formatFileNums(i.Tables)),
    51  		redact.Safe(humanize.Uint64(tablesTotalSize(i.Tables))))
    52  }
    53  
    54  // CompactionInfo contains the info for a compaction event.
    55  type CompactionInfo struct {
    56  	// JobID is the ID of the compaction job.
    57  	JobID int
    58  	// Reason is the reason for the compaction.
    59  	Reason string
    60  	// Input contains the input tables for the compaction organized by level.
    61  	Input []LevelInfo
    62  	// Output contains the output tables generated by the compaction. The output
    63  	// tables are empty for the compaction begin event.
    64  	Output LevelInfo
    65  	// Duration is the time spent compacting, including reading and writing
    66  	// sstables.
    67  	Duration time.Duration
    68  	// TotalDuration is the total wall-time duration of the compaction,
    69  	// including applying the compaction to the database. TotalDuration is
    70  	// always ≥ Duration.
    71  	TotalDuration time.Duration
    72  	Done          bool
    73  	Err           error
    74  }
    75  
    76  func (i CompactionInfo) String() string {
    77  	return redact.StringWithoutMarkers(i)
    78  }
    79  
    80  // SafeFormat implements redact.SafeFormatter.
    81  func (i CompactionInfo) SafeFormat(w redact.SafePrinter, _ rune) {
    82  	if i.Err != nil {
    83  		w.Printf("[JOB %d] compaction(%s) to L%d error: %s",
    84  			redact.Safe(i.JobID), redact.SafeString(i.Reason), redact.Safe(i.Output.Level), i.Err)
    85  		return
    86  	}
    87  
    88  	if !i.Done {
    89  		w.Printf("[JOB %d] compacting(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
    90  		w.Print(levelInfos(i.Input))
    91  		return
    92  	}
    93  	outputSize := tablesTotalSize(i.Output.Tables)
    94  	w.Printf("[JOB %d] compacted(%s) ", redact.Safe(i.JobID), redact.SafeString(i.Reason))
    95  	w.Print(levelInfos(i.Input))
    96  	w.Printf(" -> L%d [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
    97  		redact.Safe(i.Output.Level),
    98  		redact.Safe(formatFileNums(i.Output.Tables)),
    99  		redact.Safe(humanize.Uint64(outputSize)),
   100  		redact.Safe(i.Duration.Seconds()),
   101  		redact.Safe(i.TotalDuration.Seconds()),
   102  		redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
   103  }
   104  
   105  type levelInfos []LevelInfo
   106  
   107  func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) {
   108  	for j, levelInfo := range i {
   109  		if j > 0 {
   110  			w.Printf(" + ")
   111  		}
   112  		w.Print(levelInfo)
   113  	}
   114  }
   115  
   116  // DiskSlowInfo contains the info for a disk slowness event when writing to a
   117  // file.
   118  type DiskSlowInfo struct {
   119  	// Path of file being written to.
   120  	Path string
   121  	// Duration that has elapsed since this disk operation started.
   122  	Duration time.Duration
   123  }
   124  
   125  func (i DiskSlowInfo) String() string {
   126  	return redact.StringWithoutMarkers(i)
   127  }
   128  
   129  // SafeFormat implements redact.SafeFormatter.
   130  func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   131  	w.Printf("disk slowness detected: write to file %s has been ongoing for %0.1fs",
   132  		i.Path, redact.Safe(i.Duration.Seconds()))
   133  }
   134  
   135  // FlushInfo contains the info for a flush event.
   136  type FlushInfo struct {
   137  	// JobID is the ID of the flush job.
   138  	JobID int
   139  	// Reason is the reason for the flush.
   140  	Reason string
   141  	// Input contains the count of input memtables that were flushed.
   142  	Input int
   143  	// Output contains the ouptut table generated by the flush. The output info
   144  	// is empty for the flush begin event.
   145  	Output []TableInfo
   146  	// Duration is the time spent flushing. This duration includes writing and
   147  	// syncing all of the flushed keys to sstables.
   148  	Duration time.Duration
   149  	// TotalDuration is the total wall-time duration of the flush, including
   150  	// applying the flush to the database. TotalDuration is always ≥ Duration.
   151  	TotalDuration time.Duration
   152  	Done          bool
   153  	Err           error
   154  }
   155  
   156  func (i FlushInfo) String() string {
   157  	return redact.StringWithoutMarkers(i)
   158  }
   159  
   160  // SafeFormat implements redact.SafeFormatter.
   161  func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   162  	if i.Err != nil {
   163  		w.Printf("[JOB %d] flush error: %s", redact.Safe(i.JobID), i.Err)
   164  		return
   165  	}
   166  
   167  	plural := redact.SafeString("s")
   168  	if i.Input == 1 {
   169  		plural = ""
   170  	}
   171  	if !i.Done {
   172  		w.Printf("[JOB %d] flushing %d memtable", redact.Safe(i.JobID), redact.Safe(i.Input))
   173  		w.SafeString(plural)
   174  		w.Printf(" to L0")
   175  		return
   176  	}
   177  
   178  	outputSize := tablesTotalSize(i.Output)
   179  	w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s",
   180  		redact.Safe(i.JobID), redact.Safe(i.Input), plural,
   181  		redact.Safe(formatFileNums(i.Output)),
   182  		redact.Safe(humanize.Uint64(outputSize)),
   183  		redact.Safe(i.Duration.Seconds()),
   184  		redact.Safe(i.TotalDuration.Seconds()),
   185  		redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds()))))
   186  }
   187  
   188  // ManifestCreateInfo contains info about a manifest creation event.
   189  type ManifestCreateInfo struct {
   190  	// JobID is the ID of the job the caused the manifest to be created.
   191  	JobID int
   192  	Path  string
   193  	// The file number of the new Manifest.
   194  	FileNum FileNum
   195  	Err     error
   196  }
   197  
   198  func (i ManifestCreateInfo) String() string {
   199  	return redact.StringWithoutMarkers(i)
   200  }
   201  
   202  // SafeFormat implements redact.SafeFormatter.
   203  func (i ManifestCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   204  	if i.Err != nil {
   205  		w.Printf("[JOB %d] MANIFEST create error: %s", redact.Safe(i.JobID), i.Err)
   206  		return
   207  	}
   208  	w.Printf("[JOB %d] MANIFEST created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
   209  }
   210  
   211  // ManifestDeleteInfo contains the info for a Manifest deletion event.
   212  type ManifestDeleteInfo struct {
   213  	// JobID is the ID of the job the caused the Manifest to be deleted.
   214  	JobID   int
   215  	Path    string
   216  	FileNum FileNum
   217  	Err     error
   218  }
   219  
   220  func (i ManifestDeleteInfo) String() string {
   221  	return redact.StringWithoutMarkers(i)
   222  }
   223  
   224  // SafeFormat implements redact.SafeFormatter.
   225  func (i ManifestDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   226  	if i.Err != nil {
   227  		w.Printf("[JOB %d] MANIFEST delete error: %s", redact.Safe(i.JobID), i.Err)
   228  		return
   229  	}
   230  	w.Printf("[JOB %d] MANIFEST deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
   231  }
   232  
   233  // TableCreateInfo contains the info for a table creation event.
   234  type TableCreateInfo struct {
   235  	JobID int
   236  	// Reason is the reason for the table creation: "compacting", "flushing", or
   237  	// "ingesting".
   238  	Reason  string
   239  	Path    string
   240  	FileNum FileNum
   241  }
   242  
   243  func (i TableCreateInfo) String() string {
   244  	return redact.StringWithoutMarkers(i)
   245  }
   246  
   247  // SafeFormat implements redact.SafeFormatter.
   248  func (i TableCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   249  	w.Printf("[JOB %d] %s: sstable created %s",
   250  		redact.Safe(i.JobID), redact.Safe(i.Reason), redact.Safe(i.FileNum))
   251  }
   252  
   253  // TableDeleteInfo contains the info for a table deletion event.
   254  type TableDeleteInfo struct {
   255  	JobID   int
   256  	Path    string
   257  	FileNum FileNum
   258  	Err     error
   259  }
   260  
   261  func (i TableDeleteInfo) String() string {
   262  	return redact.StringWithoutMarkers(i)
   263  }
   264  
   265  // SafeFormat implements redact.SafeFormatter.
   266  func (i TableDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   267  	if i.Err != nil {
   268  		w.Printf("[JOB %d] sstable delete error %s: %s",
   269  			redact.Safe(i.JobID), redact.Safe(i.FileNum), i.Err)
   270  		return
   271  	}
   272  	w.Printf("[JOB %d] sstable deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
   273  }
   274  
   275  // TableIngestInfo contains the info for a table ingestion event.
   276  type TableIngestInfo struct {
   277  	// JobID is the ID of the job the caused the table to be ingested.
   278  	JobID  int
   279  	Tables []struct {
   280  		TableInfo
   281  		Level int
   282  	}
   283  	// GlobalSeqNum is the sequence number that was assigned to all entries in
   284  	// the ingested table.
   285  	GlobalSeqNum uint64
   286  	Err          error
   287  }
   288  
   289  func (i TableIngestInfo) String() string {
   290  	return redact.StringWithoutMarkers(i)
   291  }
   292  
   293  // SafeFormat implements redact.SafeFormatter.
   294  func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   295  	if i.Err != nil {
   296  		w.Printf("[JOB %d] ingest error: %s", redact.Safe(i.JobID), i.Err)
   297  		return
   298  	}
   299  
   300  	w.Printf("[JOB %d] ingested", redact.Safe(i.JobID))
   301  	for j := range i.Tables {
   302  		t := &i.Tables[j]
   303  		if j > 0 {
   304  			w.Printf(",")
   305  		}
   306  		w.Printf(" L%d:%s (%s)", redact.Safe(t.Level), redact.Safe(t.FileNum),
   307  			redact.Safe(humanize.Uint64(t.Size)))
   308  	}
   309  }
   310  
   311  // TableStatsInfo contains the info for a table stats loaded event.
   312  type TableStatsInfo struct {
   313  	// JobID is the ID of the job that finished loading the initial tables'
   314  	// stats.
   315  	JobID int
   316  }
   317  
   318  func (i TableStatsInfo) String() string {
   319  	return redact.StringWithoutMarkers(i)
   320  }
   321  
   322  // SafeFormat implements redact.SafeFormatter.
   323  func (i TableStatsInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   324  	w.Printf("[JOB %d] all initial table stats loaded", redact.Safe(i.JobID))
   325  }
   326  
   327  // TableValidatedInfo contains information on the result of a validation run
   328  // on an sstable.
   329  type TableValidatedInfo struct {
   330  	JobID int
   331  	Meta  *fileMetadata
   332  }
   333  
   334  func (i TableValidatedInfo) String() string {
   335  	return redact.StringWithoutMarkers(i)
   336  }
   337  
   338  // SafeFormat implements redact.SafeFormatter.
   339  func (i TableValidatedInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   340  	w.Printf("[JOB %d] validated table: %s", redact.Safe(i.JobID), i.Meta)
   341  }
   342  
   343  // WALCreateInfo contains info about a WAL creation event.
   344  type WALCreateInfo struct {
   345  	// JobID is the ID of the job the caused the WAL to be created.
   346  	JobID int
   347  	Path  string
   348  	// The file number of the new WAL.
   349  	FileNum FileNum
   350  	// The file number of a previous WAL which was recycled to create this
   351  	// one. Zero if recycling did not take place.
   352  	RecycledFileNum FileNum
   353  	Err             error
   354  }
   355  
   356  func (i WALCreateInfo) String() string {
   357  	return redact.StringWithoutMarkers(i)
   358  }
   359  
   360  // SafeFormat implements redact.SafeFormatter.
   361  func (i WALCreateInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   362  	if i.Err != nil {
   363  		w.Printf("[JOB %d] WAL create error: %s", redact.Safe(i.JobID), i.Err)
   364  		return
   365  	}
   366  
   367  	if i.RecycledFileNum == 0 {
   368  		w.Printf("[JOB %d] WAL created %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
   369  		return
   370  	}
   371  
   372  	w.Printf("[JOB %d] WAL created %s (recycled %s)",
   373  		redact.Safe(i.JobID), redact.Safe(i.FileNum), redact.Safe(i.RecycledFileNum))
   374  }
   375  
   376  // WALDeleteInfo contains the info for a WAL deletion event.
   377  type WALDeleteInfo struct {
   378  	// JobID is the ID of the job the caused the WAL to be deleted.
   379  	JobID   int
   380  	Path    string
   381  	FileNum FileNum
   382  	Err     error
   383  }
   384  
   385  func (i WALDeleteInfo) String() string {
   386  	return redact.StringWithoutMarkers(i)
   387  }
   388  
   389  // SafeFormat implements redact.SafeFormatter.
   390  func (i WALDeleteInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   391  	if i.Err != nil {
   392  		w.Printf("[JOB %d] WAL delete error: %s", redact.Safe(i.JobID), i.Err)
   393  		return
   394  	}
   395  	w.Printf("[JOB %d] WAL deleted %s", redact.Safe(i.JobID), redact.Safe(i.FileNum))
   396  }
   397  
   398  // WriteStallBeginInfo contains the info for a write stall begin event.
   399  type WriteStallBeginInfo struct {
   400  	Reason string
   401  }
   402  
   403  func (i WriteStallBeginInfo) String() string {
   404  	return redact.StringWithoutMarkers(i)
   405  }
   406  
   407  // SafeFormat implements redact.SafeFormatter.
   408  func (i WriteStallBeginInfo) SafeFormat(w redact.SafePrinter, _ rune) {
   409  	w.Printf("write stall beginning: %s", redact.Safe(i.Reason))
   410  }
   411  
   412  // EventListener contains a set of functions that will be invoked when various
   413  // significant DB events occur. Note that the functions should not run for an
   414  // excessive amount of time as they are invoked synchronously by the DB and may
   415  // block continued DB work. For a similar reason it is advisable to not perform
   416  // any synchronous calls back into the DB.
   417  type EventListener struct {
   418  	// BackgroundError is invoked whenever an error occurs during a background
   419  	// operation such as flush or compaction.
   420  	BackgroundError func(error)
   421  
   422  	// CompactionBegin is invoked after the inputs to a compaction have been
   423  	// determined, but before the compaction has produced any output.
   424  	CompactionBegin func(CompactionInfo)
   425  
   426  	// CompactionEnd is invoked after a compaction has completed and the result
   427  	// has been installed.
   428  	CompactionEnd func(CompactionInfo)
   429  
   430  	// DiskSlow is invoked after a disk write operation on a file created
   431  	// with a disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks)
   432  	// is observed to exceed the specified disk slowness threshold duration.
   433  	DiskSlow func(DiskSlowInfo)
   434  
   435  	// FlushBegin is invoked after the inputs to a flush have been determined,
   436  	// but before the flush has produced any output.
   437  	FlushBegin func(FlushInfo)
   438  
   439  	// FlushEnd is invoked after a flush has complated and the result has been
   440  	// installed.
   441  	FlushEnd func(FlushInfo)
   442  
   443  	// FormatUpgrade is invoked after the database's FormatMajorVersion
   444  	// is upgraded.
   445  	FormatUpgrade func(FormatMajorVersion)
   446  
   447  	// ManifestCreated is invoked after a manifest has been created.
   448  	ManifestCreated func(ManifestCreateInfo)
   449  
   450  	// ManifestDeleted is invoked after a manifest has been deleted.
   451  	ManifestDeleted func(ManifestDeleteInfo)
   452  
   453  	// TableCreated is invoked when a table has been created.
   454  	TableCreated func(TableCreateInfo)
   455  
   456  	// TableDeleted is invoked after a table has been deleted.
   457  	TableDeleted func(TableDeleteInfo)
   458  
   459  	// TableIngested is invoked after an externally created table has been
   460  	// ingested via a call to DB.Ingest().
   461  	TableIngested func(TableIngestInfo)
   462  
   463  	// TableStatsLoaded is invoked at most once, when the table stats
   464  	// collector has loaded statistics for all tables that existed at Open.
   465  	TableStatsLoaded func(TableStatsInfo)
   466  
   467  	// TableValidated is invoked after validation runs on an sstable.
   468  	TableValidated func(TableValidatedInfo)
   469  
   470  	// WALCreated is invoked after a WAL has been created.
   471  	WALCreated func(WALCreateInfo)
   472  
   473  	// WALDeleted is invoked after a WAL has been deleted.
   474  	WALDeleted func(WALDeleteInfo)
   475  
   476  	// WriteStallBegin is invoked when writes are intentionally delayed.
   477  	WriteStallBegin func(WriteStallBeginInfo)
   478  
   479  	// WriteStallEnd is invoked when delayed writes are released.
   480  	WriteStallEnd func()
   481  }
   482  
   483  // EnsureDefaults ensures that background error events are logged to the
   484  // specified logger if a handler for those events hasn't been otherwise
   485  // specified. Ensure all handlers are non-nil so that we don't have to check
   486  // for nil-ness before invoking.
   487  func (l *EventListener) EnsureDefaults(logger Logger) {
   488  	if l.BackgroundError == nil {
   489  		if logger != nil {
   490  			l.BackgroundError = func(err error) {
   491  				logger.Infof("background error: %s", err)
   492  			}
   493  		} else {
   494  			l.BackgroundError = func(error) {}
   495  		}
   496  	}
   497  	if l.CompactionBegin == nil {
   498  		l.CompactionBegin = func(info CompactionInfo) {}
   499  	}
   500  	if l.CompactionEnd == nil {
   501  		l.CompactionEnd = func(info CompactionInfo) {}
   502  	}
   503  	if l.DiskSlow == nil {
   504  		l.DiskSlow = func(info DiskSlowInfo) {}
   505  	}
   506  	if l.FlushBegin == nil {
   507  		l.FlushBegin = func(info FlushInfo) {}
   508  	}
   509  	if l.FlushEnd == nil {
   510  		l.FlushEnd = func(info FlushInfo) {}
   511  	}
   512  	if l.FormatUpgrade == nil {
   513  		l.FormatUpgrade = func(v FormatMajorVersion) {}
   514  	}
   515  	if l.ManifestCreated == nil {
   516  		l.ManifestCreated = func(info ManifestCreateInfo) {}
   517  	}
   518  	if l.ManifestDeleted == nil {
   519  		l.ManifestDeleted = func(info ManifestDeleteInfo) {}
   520  	}
   521  	if l.TableCreated == nil {
   522  		l.TableCreated = func(info TableCreateInfo) {}
   523  	}
   524  	if l.TableDeleted == nil {
   525  		l.TableDeleted = func(info TableDeleteInfo) {}
   526  	}
   527  	if l.TableIngested == nil {
   528  		l.TableIngested = func(info TableIngestInfo) {}
   529  	}
   530  	if l.TableStatsLoaded == nil {
   531  		l.TableStatsLoaded = func(info TableStatsInfo) {}
   532  	}
   533  	if l.TableValidated == nil {
   534  		l.TableValidated = func(validated TableValidatedInfo) {}
   535  	}
   536  	if l.WALCreated == nil {
   537  		l.WALCreated = func(info WALCreateInfo) {}
   538  	}
   539  	if l.WALDeleted == nil {
   540  		l.WALDeleted = func(info WALDeleteInfo) {}
   541  	}
   542  	if l.WriteStallBegin == nil {
   543  		l.WriteStallBegin = func(info WriteStallBeginInfo) {}
   544  	}
   545  	if l.WriteStallEnd == nil {
   546  		l.WriteStallEnd = func() {}
   547  	}
   548  }
   549  
   550  // MakeLoggingEventListener creates an EventListener that logs all events to the
   551  // specified logger.
   552  func MakeLoggingEventListener(logger Logger) EventListener {
   553  	if logger == nil {
   554  		logger = DefaultLogger
   555  	}
   556  
   557  	return EventListener{
   558  		BackgroundError: func(err error) {
   559  			logger.Infof("background error: %s", err)
   560  		},
   561  		CompactionBegin: func(info CompactionInfo) {
   562  			logger.Infof("%s", info)
   563  		},
   564  		CompactionEnd: func(info CompactionInfo) {
   565  			logger.Infof("%s", info)
   566  		},
   567  		DiskSlow: func(info DiskSlowInfo) {
   568  			logger.Infof("%s", info)
   569  		},
   570  		FlushBegin: func(info FlushInfo) {
   571  			logger.Infof("%s", info)
   572  		},
   573  		FlushEnd: func(info FlushInfo) {
   574  			logger.Infof("%s", info)
   575  		},
   576  		FormatUpgrade: func(v FormatMajorVersion) {
   577  			logger.Infof("upgraded to format version: %s", v)
   578  		},
   579  		ManifestCreated: func(info ManifestCreateInfo) {
   580  			logger.Infof("%s", info)
   581  		},
   582  		ManifestDeleted: func(info ManifestDeleteInfo) {
   583  			logger.Infof("%s", info)
   584  		},
   585  		TableCreated: func(info TableCreateInfo) {
   586  			logger.Infof("%s", info)
   587  		},
   588  		TableDeleted: func(info TableDeleteInfo) {
   589  			logger.Infof("%s", info)
   590  		},
   591  		TableIngested: func(info TableIngestInfo) {
   592  			logger.Infof("%s", info)
   593  		},
   594  		TableStatsLoaded: func(info TableStatsInfo) {
   595  			logger.Infof("%s", info)
   596  		},
   597  		TableValidated: func(info TableValidatedInfo) {
   598  			logger.Infof("%s", info)
   599  		},
   600  		WALCreated: func(info WALCreateInfo) {
   601  			logger.Infof("%s", info)
   602  		},
   603  		WALDeleted: func(info WALDeleteInfo) {
   604  			logger.Infof("%s", info)
   605  		},
   606  		WriteStallBegin: func(info WriteStallBeginInfo) {
   607  			logger.Infof("%s", info)
   608  		},
   609  		WriteStallEnd: func() {
   610  			logger.Infof("write stall ending")
   611  		},
   612  	}
   613  }
   614  
   615  // TeeEventListener wraps two EventListeners, forwarding all events to both.
   616  func TeeEventListener(a, b EventListener) EventListener {
   617  	a.EnsureDefaults(nil)
   618  	b.EnsureDefaults(nil)
   619  	return EventListener{
   620  		BackgroundError: func(err error) {
   621  			a.BackgroundError(err)
   622  			b.BackgroundError(err)
   623  		},
   624  		CompactionBegin: func(info CompactionInfo) {
   625  			a.CompactionBegin(info)
   626  			b.CompactionBegin(info)
   627  		},
   628  		CompactionEnd: func(info CompactionInfo) {
   629  			a.CompactionEnd(info)
   630  			b.CompactionEnd(info)
   631  		},
   632  		DiskSlow: func(info DiskSlowInfo) {
   633  			a.DiskSlow(info)
   634  			b.DiskSlow(info)
   635  		},
   636  		FlushBegin: func(info FlushInfo) {
   637  			a.FlushBegin(info)
   638  			b.FlushBegin(info)
   639  		},
   640  		FlushEnd: func(info FlushInfo) {
   641  			a.FlushEnd(info)
   642  			b.FlushEnd(info)
   643  		},
   644  		FormatUpgrade: func(v FormatMajorVersion) {
   645  			a.FormatUpgrade(v)
   646  			b.FormatUpgrade(v)
   647  		},
   648  		ManifestCreated: func(info ManifestCreateInfo) {
   649  			a.ManifestCreated(info)
   650  			b.ManifestCreated(info)
   651  		},
   652  		ManifestDeleted: func(info ManifestDeleteInfo) {
   653  			a.ManifestDeleted(info)
   654  			b.ManifestDeleted(info)
   655  		},
   656  		TableCreated: func(info TableCreateInfo) {
   657  			a.TableCreated(info)
   658  			b.TableCreated(info)
   659  		},
   660  		TableDeleted: func(info TableDeleteInfo) {
   661  			a.TableDeleted(info)
   662  			b.TableDeleted(info)
   663  		},
   664  		TableIngested: func(info TableIngestInfo) {
   665  			a.TableIngested(info)
   666  			b.TableIngested(info)
   667  		},
   668  		TableStatsLoaded: func(info TableStatsInfo) {
   669  			a.TableStatsLoaded(info)
   670  			b.TableStatsLoaded(info)
   671  		},
   672  		TableValidated: func(info TableValidatedInfo) {
   673  			a.TableValidated(info)
   674  			b.TableValidated(info)
   675  		},
   676  		WALCreated: func(info WALCreateInfo) {
   677  			a.WALCreated(info)
   678  			b.WALCreated(info)
   679  		},
   680  		WALDeleted: func(info WALDeleteInfo) {
   681  			a.WALDeleted(info)
   682  			b.WALDeleted(info)
   683  		},
   684  		WriteStallBegin: func(info WriteStallBeginInfo) {
   685  			a.WriteStallBegin(info)
   686  			b.WriteStallBegin(info)
   687  		},
   688  		WriteStallEnd: func() {
   689  			a.WriteStallEnd()
   690  			b.WriteStallEnd()
   691  		},
   692  	}
   693  }