github.com/biogo/biogo@v1.0.4/concurrent/map.go (about)

     1  // Copyright ©2011-2013 The bíogo Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package concurrent
     6  
     7  import (
     8  	"github.com/biogo/biogo/util"
     9  
    10  	"fmt"
    11  	"math"
    12  )
    13  
    14  // A Mapper is an Operator that can subdivide itself.
    15  type Mapper interface {
    16  	Operator
    17  	Slice(i, j int) Mapper
    18  	Len() int
    19  }
    20  
    21  // Map routines to iterate a function over an array, potentially splitting the array slice into
    22  // chunks so that each chunk is processed concurrently. When using concurrent processing the
    23  // Chunk size is either the nearest even division of the total array over the chosen concurrent
    24  // processing goroutines or a specified maximum chunk size, whichever is smaller. Reducing
    25  // chunk size can reduce the impact of divergence in time for processing chunks, but may add
    26  // to overhead.
    27  func Map(set Mapper, threads, maxChunkSize int) (results []interface{}, err error) {
    28  	queue := make(chan Operator, 1)
    29  	p := NewProcessor(queue, 0, threads)
    30  	defer p.Stop()
    31  
    32  	chunkSize := util.Min(int(math.Ceil(float64(set.Len())/float64(threads))), maxChunkSize)
    33  
    34  	quit := make(chan struct{})
    35  
    36  	go func() {
    37  		for s := 0; s*chunkSize < set.Len(); s++ {
    38  			select {
    39  			case <-quit:
    40  				break
    41  			default:
    42  				endChunk := util.Min(chunkSize*(s+1), set.Len())
    43  				queue <- set.Slice(chunkSize*s, endChunk)
    44  			}
    45  		}
    46  	}()
    47  
    48  	for r := 0; r*chunkSize < set.Len(); r++ {
    49  		result := <-p.out
    50  		if result.Err != nil {
    51  			err = fmt.Errorf("concurrent: map failed: %v", err)
    52  			close(quit)
    53  			break
    54  		}
    55  		results = append(results, result.Value)
    56  	}
    57  
    58  	return
    59  }
    60  
    61  // A future Map function - synchronisation is via a Promise.
    62  func PromiseMap(set Mapper, threads, maxChunkSize int) *Promise {
    63  	promise := NewPromise(false, false, false)
    64  
    65  	go func() {
    66  		result, err := Map(set, threads, maxChunkSize)
    67  		if err == nil {
    68  			promise.Fulfill(result)
    69  		} else {
    70  			promise.Fail(result, err)
    71  		}
    72  	}()
    73  
    74  	return promise
    75  }