github.com/minio/mc@v0.0.0-20240503112107-b471de8d1882/cmd/pipechan.go (about)

     1  // Copyright (c) 2015-2022 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package cmd
    19  
    20  import "github.com/rjeczalik/notify"
    21  
    22  // Dynamically sized logical channel: a pipe which never blocks even when
    23  // it receives too many elements. Memory consumption is increased and decresed
    24  // on the fly following the number of elements. Here is the benchmark which
    25  // compares a regular channel which a large capacity (like 1 Million) in
    26  // contrast to a PipeChan with an realtime increasing/decreasing capacity.
    27  //
    28  // BenchmarkRegular1M-4                 200         172809399 ns/op
    29  // BenchmarkPipeChan1M-4                100         316668338 ns/op
    30  // BenchmarkRegular100K-4              2000          16540648 ns/op
    31  // BenchmarkPipeChan100K-4             1000          31905966 ns/op
    32  // BenchmarkRegular10K-4              20000           1637665 ns/op
    33  // BenchmarkPipeChan10K-4             10000           3324329 ns/op
    34  // BenchmarkRegular1K-4              200000            168512 ns/op
    35  // BenchmarkPipeChan1K-4             100000            550623 ns/op
    36  
    37  // PipeChan builds a new dynamically sized channel
    38  func PipeChan(capacity int) (inputCh, outputCh chan notify.EventInfo) {
    39  	// A set of channels which store all elements received from input
    40  	channels := make(chan chan notify.EventInfo, 1000)
    41  
    42  	inputCh = make(chan notify.EventInfo, capacity)
    43  
    44  	// A goroutine which receives elements from inputCh and creates
    45  	// new channels when needed.
    46  	go func() {
    47  		// Create the first channel
    48  		currCh := make(chan notify.EventInfo, capacity)
    49  		channels <- currCh
    50  
    51  		for elem := range inputCh {
    52  			// Prepare next channel with a double capacity when
    53  			// half of the current channel is already filled.
    54  			if len(currCh) >= cap(currCh)/2 {
    55  				close(currCh)
    56  				currCh = make(chan notify.EventInfo, cap(currCh)*2)
    57  				channels <- currCh
    58  			}
    59  			// Prepare next channel with half capacity when
    60  			// current channel is 1/4 filled
    61  			if len(currCh) >= capacity && len(currCh) <= cap(currCh)/4 {
    62  				close(currCh)
    63  				currCh = make(chan notify.EventInfo, cap(currCh)/2)
    64  				channels <- currCh
    65  			}
    66  			// Send element to current channel
    67  			currCh <- elem
    68  		}
    69  
    70  		close(currCh)
    71  		close(channels)
    72  	}()
    73  
    74  	// Copy elements from infinite channel set to the output
    75  	outputCh = make(chan notify.EventInfo, capacity)
    76  	go func() {
    77  		for {
    78  			currCh, ok := <-channels
    79  			if !ok {
    80  				break
    81  			}
    82  			for v := range currCh {
    83  				outputCh <- v
    84  			}
    85  		}
    86  		close(outputCh)
    87  	}()
    88  	return inputCh, outputCh
    89  }