github.com/puellanivis/breton@v0.2.16/lib/mapreduce/options.go (about)

     1  package mapreduce
     2  
     3  type config struct {
     4  	threadCount int
     5  	mapperCount int
     6  
     7  	// We can only have one bound on stripe size: maximum or minimum.
     8  	// So we encode them to the same integer as such:
     9  	// * If zero, there is no minimum or maximum limit,
    10  	// * If negative, the magnitude is the minimum stripe size to be used.
    11  	// * If positive, the magnitude is the maximum stripe size to be used.
    12  	stripeSize int
    13  
    14  	ordered bool
    15  }
    16  
    17  // Option defines a function that applies a setting or value to a MapReduce configuration.
    18  type Option func(mr *MapReduce) Option
    19  
    20  // WithThreadCount sets the number of threads (concurrently executing goroutines) that the MapReduce should use.
    21  func WithThreadCount(num int) Option {
    22  	return func(mr *MapReduce) Option {
    23  		mr.mu.Lock()
    24  		defer mr.mu.Unlock()
    25  
    26  		save := mr.conf.threadCount
    27  
    28  		mr.conf.threadCount = num
    29  
    30  		return WithThreadCount(save)
    31  	}
    32  }
    33  
    34  // WithMapperCount sets the number of Mappers (total number of goroutine tasks) that the MapReduce should use.
    35  //
    36  // This value MAY BE overridden, if stripe size values are also specified.
    37  func WithMapperCount(num int) Option {
    38  	return func(mr *MapReduce) Option {
    39  		mr.mu.Lock()
    40  		defer mr.mu.Unlock()
    41  
    42  		save := mr.conf.mapperCount
    43  
    44  		mr.conf.mapperCount = num
    45  
    46  		return WithMapperCount(save)
    47  	}
    48  }
    49  
    50  func withStripeSize(size int) Option {
    51  	return func(mr *MapReduce) Option {
    52  		mr.mu.Lock()
    53  		defer mr.mu.Unlock()
    54  
    55  		save := mr.conf.stripeSize
    56  
    57  		// We encode a minimum limit as a negative value.
    58  		mr.conf.stripeSize = size
    59  
    60  		return withStripeSize(save)
    61  	}
    62  }
    63  
    64  // WithMinStripeSize sets the minimum number of elements that each Mapper will receive.
    65  //
    66  // One cannot set both a minimum and maximum stripe size setting,
    67  // otherwise we could end up in a case where neither constraint could be met.
    68  // Therefore, setting a minimum stripe size will unset any maximum stripe size setting.
    69  //
    70  // If after calculating the work for each Mapper,
    71  // the number of elements to be handled per Mapper is less than this value,
    72  // then the number of Mappers will be set to a value,
    73  // where the number of elements handled by each Mapper is greater than or equal to this value.
    74  //
    75  // A value less than 1 resets all stripe size settings, both minimum and maximum.
    76  func WithMinStripeSize(size int) Option {
    77  	if size < 1 {
    78  		size = 0
    79  	}
    80  
    81  	return withStripeSize(-size)
    82  }
    83  
    84  // WithMaxStripeSize sets the maximum number of elements that each Mapper will receive.
    85  //
    86  // One cannot set both a minimum and maximum stripe size setting,
    87  // otherwise we could end up in a case where neither constraint could be met.
    88  // Therefore, setting a maximum stripe size will unset any minimum stripe size setting.
    89  //
    90  // If after calculating the work for each Mapper,
    91  // the number of elements to be handled per Mapper is greater than this value,
    92  // then the number of Mappers will be set to a value,
    93  // where the number of elements handled by each Mapper is less than or equal to this value.
    94  //
    95  // A value less than 1 resets all stripe size settings, both minimum and maximum.
    96  func WithMaxStripeSize(size int) Option {
    97  	if size < 1 {
    98  		size = 0
    99  	}
   100  
   101  	return withStripeSize(size)
   102  }
   103  
   104  // WithOrdering sets the ordering state of the Reduce phase.
   105  //
   106  // When the Reduce phase is ordered,
   107  // then the Reduce phase of each Mapper will happen in Order,
   108  // that is, given a < b < c,
   109  // the end of the Reduce for range [a,b) HAPPENS BEFORE the start of the Reduce for range [b,c).
   110  //
   111  // So, when the Reduce phase is ordered,
   112  // even if all other Mappers complete before the first Mapper completes,
   113  // the Reduce for the first Mapper WILL execute first.
   114  func WithOrdering(ordered bool) Option {
   115  	return func(mr *MapReduce) Option {
   116  		mr.mu.Lock()
   117  		defer mr.mu.Unlock()
   118  
   119  		save := mr.conf.ordered
   120  
   121  		mr.conf.ordered = ordered
   122  
   123  		return WithOrdering(save)
   124  	}
   125  }