github.com/15mga/kiwi@v0.0.2-0.20240324021231-b95d5c3ac751/worker/parallel.go (about)

     1  package worker
     2  
     3  import (
     4  	"runtime"
     5  	"sync"
     6  	"sync/atomic"
     7  
     8  	"github.com/15mga/kiwi/ds"
     9  	"github.com/15mga/kiwi/util"
    10  )
    11  
    12  var (
    13  	_Parallel         *parallel
    14  	_ParallelNum      int
    15  	_WorkerNum        int
    16  	_WorkerNum32      uint32
    17  	_JobParallelCount int
    18  )
    19  
    20  func init() {
    21  	_ParallelNum = runtime.NumCPU()
    22  	if _ParallelNum < 8 {
    23  		_ParallelNum = 8
    24  	}
    25  	_WorkerNum = _ParallelNum - 1
    26  	_WorkerNum32 = uint32(_WorkerNum)
    27  	_JobParallelCount = _JobUnit * _WorkerNum
    28  }
    29  
    30  const (
    31  	_JobUnit = 128
    32  )
    33  
    34  type parallel struct {
    35  	workers []*parallelWorker
    36  }
    37  
    38  func InitParallel() {
    39  	if _Parallel != nil {
    40  		return
    41  	}
    42  	_Parallel = &parallel{
    43  		workers: make([]*parallelWorker, _WorkerNum),
    44  	}
    45  	for i := 0; i < _WorkerNum; i++ {
    46  		w := newParallelWorker()
    47  		_Parallel.workers[i] = w
    48  		go w.start()
    49  	}
    50  }
    51  
    52  var (
    53  	_WorkerIdx uint32
    54  )
    55  
    56  func PushPJob(job IJob) {
    57  	idx := atomic.AddUint32(&_WorkerIdx, 1)
    58  	_Parallel.workers[idx%_WorkerNum32].PushJob(job)
    59  }
    60  
    61  func getAvgCount(l int) int {
    62  	if l < _JobParallelCount {
    63  		return _JobUnit
    64  	}
    65  	num := _ParallelNum
    66  	count := l / num
    67  	if l%num != 0 {
    68  		count++
    69  	}
    70  	return count
    71  }
    72  
    73  func PFn(fns []util.FnAnySlc, params ...any) {
    74  	l := len(fns)
    75  	if l <= _JobUnit {
    76  		for _, fn := range fns {
    77  			fn(params)
    78  		}
    79  		return
    80  	}
    81  	var wg sync.WaitGroup
    82  	avg := getAvgCount(l)
    83  	var end int
    84  	for start := avg; end < l; start += avg {
    85  		end = start + avg
    86  		if end > l {
    87  			end = l
    88  		}
    89  		wg.Add(1)
    90  		PushPJob(&fnJob{
    91  			fns:    fns,
    92  			start:  start,
    93  			end:    end,
    94  			wg:     &wg,
    95  			params: params,
    96  		})
    97  	}
    98  	for idx := 0; idx < avg; idx++ {
    99  		fns[idx](params)
   100  	}
   101  	wg.Wait()
   102  }
   103  
   104  func P[DT any](data []DT, fn func(DT)) {
   105  	l := len(data)
   106  	if l < _JobUnit {
   107  		for _, d := range data {
   108  			fn(d)
   109  		}
   110  		return
   111  	}
   112  	var wg sync.WaitGroup
   113  	avg := getAvgCount(l)
   114  	var end int
   115  	for start := avg; end < l; start += avg {
   116  		end = start + avg
   117  		if end > l {
   118  			end = l
   119  		}
   120  		wg.Add(1)
   121  		PushPJob(&slcJob[DT]{
   122  			slcJobBase: slcJobBase[DT]{
   123  				data:  data,
   124  				start: start,
   125  				end:   end,
   126  				wg:    &wg,
   127  			},
   128  			fn: fn,
   129  		})
   130  	}
   131  	for idx := 0; idx < avg; idx++ {
   132  		fn(data[idx])
   133  	}
   134  	wg.Wait()
   135  }
   136  
   137  func PFilter[DT1 any, DT2 comparable](data []DT1, fn func(DT1) (DT2, bool), complete func([]DT2)) {
   138  	l := len(data)
   139  	if l < _JobUnit {
   140  		slc := make([]DT2, 0, l)
   141  		for _, d := range data {
   142  			item, ok := fn(d)
   143  			if ok {
   144  				slc = append(slc, item)
   145  			}
   146  		}
   147  		if len(slc) > 0 {
   148  			complete(slc)
   149  		}
   150  		return
   151  	}
   152  	var wg sync.WaitGroup
   153  	all := make([]*ds.Array[DT2], 0, _WorkerNum)
   154  	avg := getAvgCount(l)
   155  	var end int
   156  	for start := avg; end < l; start += avg {
   157  		end = start + avg
   158  		if end > l {
   159  			end = l
   160  		}
   161  		wg.Add(1)
   162  		arr := ds.NewArray[DT2](end - start)
   163  		all = append(all, arr)
   164  		PushPJob(&slcJob[DT1]{
   165  			slcJobBase: slcJobBase[DT1]{
   166  				data:  data,
   167  				start: start,
   168  				end:   end,
   169  				wg:    &wg,
   170  			},
   171  			fn: func(d DT1) {
   172  				item, ok := fn(d)
   173  				if ok {
   174  					arr.Add(item)
   175  				}
   176  			},
   177  		})
   178  	}
   179  	slc := make([]DT2, 0, avg)
   180  	for idx := 0; idx < avg; idx++ {
   181  		item, ok := fn(data[idx])
   182  		if ok {
   183  			slc = append(slc, item)
   184  		}
   185  	}
   186  	if len(slc) > 0 {
   187  		complete(slc)
   188  	}
   189  	wg.Wait()
   190  	for _, arr := range all {
   191  		if arr.Count() > 0 {
   192  			complete(arr.Values())
   193  		}
   194  	}
   195  }
   196  
   197  func PParams[DT any](data []DT, fn func(DT, []any), params ...any) {
   198  	l := len(data)
   199  	if l < _JobUnit {
   200  		for _, d := range data {
   201  			fn(d, params)
   202  		}
   203  		return
   204  	}
   205  	var wg sync.WaitGroup
   206  	avg := getAvgCount(l)
   207  	var end int
   208  	for start := avg; end < l; start += avg {
   209  		end = start + avg
   210  		if end > l {
   211  			end = l
   212  		}
   213  		wg.Add(1)
   214  		PushPJob(&slcJobParams[DT]{
   215  			slcJobBase: slcJobBase[DT]{
   216  				data:  data,
   217  				start: start,
   218  				end:   end,
   219  				wg:    &wg,
   220  			},
   221  			fn:     fn,
   222  			params: params,
   223  		})
   224  	}
   225  	for idx := 0; idx < avg; idx++ {
   226  		fn(data[idx], params)
   227  	}
   228  	wg.Wait()
   229  }
   230  
   231  func PToFnLink[DT any](data []DT, fn func(DT, *ds.FnLink)) {
   232  	l := len(data)
   233  	if l <= _JobUnit {
   234  		buffer := ds.NewFnLink()
   235  		for _, d := range data {
   236  			fn(d, buffer)
   237  		}
   238  		buffer.Invoke()
   239  		buffer.Dispose()
   240  		return
   241  	}
   242  	var wg sync.WaitGroup
   243  	avg := getAvgCount(l)
   244  	var end int
   245  	buffers := make([]*ds.FnLink, 0, _WorkerNum)
   246  	for start := avg; end < l; start += avg {
   247  		end = start + avg
   248  		if end > l {
   249  			end = l
   250  		}
   251  		wg.Add(1)
   252  		buffer := ds.NewFnLink()
   253  		buffers = append(buffers, buffer)
   254  		PushPJob(&slcToFnJob[DT]{
   255  			slcToFnJobBase: slcToFnJobBase[DT]{
   256  				buffer: buffer,
   257  				data:   data,
   258  				start:  start,
   259  				end:    end,
   260  				wg:     &wg,
   261  			},
   262  			fn: fn,
   263  		})
   264  	}
   265  	buffer := ds.NewFnLink()
   266  	for idx := 0; idx < avg; idx++ {
   267  		fn(data[idx], buffer)
   268  	}
   269  	buffer.Invoke()
   270  	buffer.Dispose()
   271  	wg.Wait()
   272  	for _, b := range buffers {
   273  		b.Invoke()
   274  		b.Dispose()
   275  	}
   276  }
   277  
   278  func PParamsToFnLink[DT any](data []DT, fn func(DT, []any, *ds.FnLink), params ...any) {
   279  	l := len(data)
   280  	if l <= _JobUnit {
   281  		buffer := ds.NewFnLink()
   282  		for _, d := range data {
   283  			fn(d, params, buffer)
   284  		}
   285  		buffer.Invoke()
   286  		buffer.Dispose()
   287  		return
   288  	}
   289  	var wg sync.WaitGroup
   290  	avg := getAvgCount(l)
   291  	var end int
   292  	buffers := make([]*ds.FnLink, 0, _WorkerNum)
   293  	for start := avg; end < l; start += avg {
   294  		end = start + avg
   295  		if end > l {
   296  			end = l
   297  		}
   298  		wg.Add(1)
   299  		buffer := ds.NewFnLink()
   300  		buffers = append(buffers, buffer)
   301  		PushPJob(&slcToFnJobParams[DT]{
   302  			slcToFnJobBase: slcToFnJobBase[DT]{
   303  				buffer: buffer,
   304  				data:   data,
   305  				start:  start,
   306  				end:    end,
   307  				wg:     &wg,
   308  			},
   309  			fn:     fn,
   310  			params: params,
   311  		})
   312  	}
   313  	buffer := ds.NewFnLink()
   314  	for idx := 0; idx < avg; idx++ {
   315  		fn(data[idx], params, buffer)
   316  	}
   317  	buffer.Invoke()
   318  	buffer.Dispose()
   319  	wg.Wait()
   320  	for _, b := range buffers {
   321  		b.Invoke()
   322  		b.Dispose()
   323  	}
   324  }
   325  
   326  func PToLink[InT, OutT any](data []InT, fn func(InT, *ds.Link[OutT]),
   327  	pcr func(*ds.Link[OutT])) {
   328  	l := len(data)
   329  	if l <= _JobUnit {
   330  		buffer := ds.NewLink[OutT]()
   331  		for _, d := range data {
   332  			fn(d, buffer)
   333  		}
   334  		pcr(buffer)
   335  		return
   336  	}
   337  	var wg sync.WaitGroup
   338  	avg := getAvgCount(l)
   339  	var end int
   340  	buffers := make([]*ds.Link[OutT], 0, _WorkerNum)
   341  	for start := avg; end < l; start += avg {
   342  		end = start + avg
   343  		if end > l {
   344  			end = l
   345  		}
   346  		wg.Add(1)
   347  		buffer := ds.NewLink[OutT]()
   348  		buffers = append(buffers, buffer)
   349  		PushPJob(&slcToLnkJob[InT, OutT]{
   350  			buffer: buffer,
   351  			data:   data,
   352  			start:  start,
   353  			end:    end,
   354  			fn:     fn,
   355  			wg:     &wg,
   356  		})
   357  	}
   358  	buffer := ds.NewLink[OutT]()
   359  	for idx := 0; idx < avg; idx++ {
   360  		fn(data[idx], buffer)
   361  	}
   362  	pcr(buffer)
   363  	wg.Wait()
   364  	for _, b := range buffers {
   365  		pcr(b)
   366  	}
   367  }
   368  
   369  func PParamsToToLink[InT, OutT any](data []InT, fn func(InT, []any, *ds.Link[OutT]),
   370  	pcr func(*ds.Link[OutT]), params ...any) {
   371  	l := len(data)
   372  	if l <= _JobUnit {
   373  		buffer := ds.NewLink[OutT]()
   374  		for _, d := range data {
   375  			fn(d, params, buffer)
   376  		}
   377  		pcr(buffer)
   378  		return
   379  	}
   380  	count := l / _WorkerNum
   381  	if l%_WorkerNum != 0 {
   382  		count++
   383  	}
   384  	var wg sync.WaitGroup
   385  	avg := getAvgCount(l)
   386  	var end int
   387  	buffers := make([]*ds.Link[OutT], 0, _WorkerNum)
   388  	for start := avg; end < l; start += avg {
   389  		end = start + avg
   390  		if end > l {
   391  			end = l
   392  		}
   393  		wg.Add(1)
   394  		buffer := ds.NewLink[OutT]()
   395  		buffers = append(buffers, buffer)
   396  		PushPJob(&slcToLnkJobParams[InT, OutT]{
   397  			buffer: buffer,
   398  			data:   data,
   399  			start:  start,
   400  			end:    end,
   401  			fn:     fn,
   402  			wg:     &wg,
   403  			params: params,
   404  		})
   405  	}
   406  	buffer := ds.NewLink[OutT]()
   407  	for idx := 0; idx < avg; idx++ {
   408  		fn(data[idx], params, buffer)
   409  	}
   410  	pcr(buffer)
   411  	wg.Wait()
   412  	for _, b := range buffers {
   413  		pcr(b)
   414  	}
   415  }
   416  
   417  func newParallelWorker() *parallelWorker {
   418  	return &parallelWorker{
   419  		jobCh: make(chan IJob, 32),
   420  	}
   421  }
   422  
   423  type parallelWorker struct {
   424  	jobCh chan IJob
   425  }
   426  
   427  func (w *parallelWorker) PushJob(job IJob) {
   428  	w.jobCh <- job
   429  }
   430  
   431  func (w *parallelWorker) start() {
   432  	for j := range w.jobCh {
   433  		j.Do()
   434  	}
   435  }
   436  
   437  type IJob interface {
   438  	Do()
   439  }
   440  
   441  type fnJob struct {
   442  	fns        []util.FnAnySlc
   443  	start, end int
   444  	wg         *sync.WaitGroup
   445  	params     []any
   446  }
   447  
   448  func (j *fnJob) Do() {
   449  	for i := j.start; i < j.end; i++ {
   450  		j.fns[i](j.params)
   451  	}
   452  	j.wg.Done()
   453  }
   454  
   455  type slcJobBase[DT any] struct {
   456  	data       []DT
   457  	start, end int
   458  	fn         func(int, DT)
   459  	wg         *sync.WaitGroup
   460  }
   461  
   462  type slcJob[DT any] struct {
   463  	slcJobBase[DT]
   464  	fn func(DT)
   465  }
   466  
   467  func (j *slcJob[DT]) Do() {
   468  	for i := j.start; i < j.end; i++ {
   469  		j.fn(j.data[i])
   470  	}
   471  	j.wg.Done()
   472  }
   473  
   474  type slcJobParams[DT any] struct {
   475  	slcJobBase[DT]
   476  	fn     func(DT, []any)
   477  	params []any
   478  }
   479  
   480  func (j *slcJobParams[DT]) Do() {
   481  	for i := j.start; i < j.end; i++ {
   482  		j.fn(j.data[i], j.params)
   483  	}
   484  	j.wg.Done()
   485  }
   486  
   487  type slcToLnkJob[DT, BT any] struct {
   488  	buffer     *ds.Link[BT]
   489  	data       []DT
   490  	start, end int
   491  	fn         func(DT, *ds.Link[BT])
   492  	wg         *sync.WaitGroup
   493  }
   494  
   495  func (j *slcToLnkJob[DT, BT]) Do() {
   496  	for i := j.start; i < j.end; i++ {
   497  		j.fn(j.data[i], j.buffer)
   498  	}
   499  	j.wg.Done()
   500  }
   501  
   502  type slcToLnkJobParams[DT, BT any] struct {
   503  	buffer     *ds.Link[BT]
   504  	data       []DT
   505  	start, end int
   506  	fn         func(DT, []any, *ds.Link[BT])
   507  	wg         *sync.WaitGroup
   508  	params     []any
   509  }
   510  
   511  func (j *slcToLnkJobParams[DT, BT]) Do() {
   512  	for i := j.start; i < j.end; i++ {
   513  		j.fn(j.data[i], j.params, j.buffer)
   514  	}
   515  	j.wg.Done()
   516  }
   517  
   518  type slcToFnJobBase[DT any] struct {
   519  	buffer     *ds.FnLink
   520  	data       []DT
   521  	start, end int
   522  	wg         *sync.WaitGroup
   523  }
   524  
   525  type slcToFnJob[DT any] struct {
   526  	slcToFnJobBase[DT]
   527  	fn func(DT, *ds.FnLink)
   528  }
   529  
   530  func (j *slcToFnJob[DT]) Do() {
   531  	for i := j.start; i < j.end; i++ {
   532  		j.fn(j.data[i], j.buffer)
   533  	}
   534  	j.wg.Done()
   535  }
   536  
   537  type slcToFnJobParams[DT any] struct {
   538  	slcToFnJobBase[DT]
   539  	fn     func(DT, []any, *ds.FnLink)
   540  	params []any
   541  }
   542  
   543  func (j *slcToFnJobParams[DT]) Do() {
   544  	for i := j.start; i < j.end; i++ {
   545  		j.fn(j.data[i], j.params, j.buffer)
   546  	}
   547  	j.wg.Done()
   548  }