github.com/hasnat/dolt/go@v0.0.0-20210628190320-9eb5d843fbb7/libraries/doltcore/sqle/async_indexed_lookups.go (about)

     1  // Copyright 2020 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package sqle
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"runtime"
    21  
    22  	"github.com/dolthub/go-mysql-server/sql"
    23  
    24  	"github.com/dolthub/dolt/go/libraries/utils/async"
    25  	"github.com/dolthub/dolt/go/store/types"
    26  )
    27  
    28  type lookupResult struct {
    29  	idx uint64
    30  	r   sql.Row
    31  	err error
    32  }
    33  
    34  // toLookup represents an table lookup that should be performed by one of the global asyncLookups instance's worker routines
    35  type toLookup struct {
    36  	idx        uint64
    37  	t          types.Tuple
    38  	tupleToRow func(types.Tuple) (sql.Row, error)
    39  	resBuf     *async.RingBuffer
    40  	epoch      int
    41  }
    42  
    43  // asyncLookups is a pool of worker routines reading from a channel doing table lookups
    44  type asyncLookups struct {
    45  	ctx        context.Context
    46  	toLookupCh chan toLookup
    47  }
    48  
    49  // newAsyncLookups kicks off a number of worker routines and creates a channel for sending lookups to workers.  The
    50  // routines live for the life of the program
    51  func newAsyncLookups(bufferSize int) *asyncLookups {
    52  	toLookupCh := make(chan toLookup, bufferSize)
    53  	art := &asyncLookups{toLookupCh: toLookupCh}
    54  
    55  	return art
    56  }
    57  
    58  func (art *asyncLookups) start(numWorkers int) {
    59  	for i := 0; i < numWorkers; i++ {
    60  		go func() {
    61  			art.workerFunc()
    62  		}()
    63  	}
    64  }
    65  
    66  func (art *asyncLookups) workerFunc() {
    67  	f := func() {
    68  		var curr toLookup
    69  		var ok bool
    70  
    71  		defer func() {
    72  			if r := recover(); r != nil {
    73  				// Attempt to write a failure to the channel and discard any additional errors
    74  				if err, ok := r.(error); ok {
    75  					_ = curr.resBuf.Push(lookupResult{idx: curr.idx, r: nil, err: err}, curr.epoch)
    76  				} else {
    77  					_ = curr.resBuf.Push(lookupResult{idx: curr.idx, r: nil, err: fmt.Errorf("%v", r)}, curr.epoch)
    78  				}
    79  			}
    80  
    81  			// if the channel used for lookups is closed then fail spectacularly
    82  			if !ok {
    83  				panic("toLookup channel closed.  All lookups will fail and will result in a deadlock")
    84  			}
    85  		}()
    86  
    87  		for {
    88  			curr, ok = <-art.toLookupCh
    89  
    90  			if !ok {
    91  				break
    92  			}
    93  
    94  			r, err := curr.tupleToRow(curr.t)
    95  			_ = curr.resBuf.Push(lookupResult{idx: curr.idx, r: r, err: err}, curr.epoch)
    96  		}
    97  	}
    98  
    99  	// these routines will run forever unless f is allowed to panic which only happens when the lookup channel is closed
   100  	for {
   101  		f()
   102  	}
   103  }
   104  
   105  // lookups is a global asyncLookups instance which is used by the indexLookupRowIterAdapter
   106  var lookups *asyncLookups
   107  
   108  func init() {
   109  	lookups = newAsyncLookups(runtime.NumCPU() * 256)
   110  	lookups.start(runtime.NumCPU())
   111  }