github.com/15mga/kiwi@v0.0.2-0.20240324021231-b95d5c3ac751/worker/parallel.go (about) 1 package worker 2 3 import ( 4 "runtime" 5 "sync" 6 "sync/atomic" 7 8 "github.com/15mga/kiwi/ds" 9 "github.com/15mga/kiwi/util" 10 ) 11 12 var ( 13 _Parallel *parallel 14 _ParallelNum int 15 _WorkerNum int 16 _WorkerNum32 uint32 17 _JobParallelCount int 18 ) 19 20 func init() { 21 _ParallelNum = runtime.NumCPU() 22 if _ParallelNum < 8 { 23 _ParallelNum = 8 24 } 25 _WorkerNum = _ParallelNum - 1 26 _WorkerNum32 = uint32(_WorkerNum) 27 _JobParallelCount = _JobUnit * _WorkerNum 28 } 29 30 const ( 31 _JobUnit = 128 32 ) 33 34 type parallel struct { 35 workers []*parallelWorker 36 } 37 38 func InitParallel() { 39 if _Parallel != nil { 40 return 41 } 42 _Parallel = ¶llel{ 43 workers: make([]*parallelWorker, _WorkerNum), 44 } 45 for i := 0; i < _WorkerNum; i++ { 46 w := newParallelWorker() 47 _Parallel.workers[i] = w 48 go w.start() 49 } 50 } 51 52 var ( 53 _WorkerIdx uint32 54 ) 55 56 func PushPJob(job IJob) { 57 idx := atomic.AddUint32(&_WorkerIdx, 1) 58 _Parallel.workers[idx%_WorkerNum32].PushJob(job) 59 } 60 61 func getAvgCount(l int) int { 62 if l < _JobParallelCount { 63 return _JobUnit 64 } 65 num := _ParallelNum 66 count := l / num 67 if l%num != 0 { 68 count++ 69 } 70 return count 71 } 72 73 func PFn(fns []util.FnAnySlc, params ...any) { 74 l := len(fns) 75 if l <= _JobUnit { 76 for _, fn := range fns { 77 fn(params) 78 } 79 return 80 } 81 var wg sync.WaitGroup 82 avg := getAvgCount(l) 83 var end int 84 for start := avg; end < l; start += avg { 85 end = start + avg 86 if end > l { 87 end = l 88 } 89 wg.Add(1) 90 PushPJob(&fnJob{ 91 fns: fns, 92 start: start, 93 end: end, 94 wg: &wg, 95 params: params, 96 }) 97 } 98 for idx := 0; idx < avg; idx++ { 99 fns[idx](params) 100 } 101 wg.Wait() 102 } 103 104 func P[DT any](data []DT, fn func(DT)) { 105 l := len(data) 106 if l < _JobUnit { 107 for _, d := range data { 108 fn(d) 109 } 110 return 111 } 112 var wg sync.WaitGroup 113 avg := getAvgCount(l) 114 var end int 115 for start := avg; end < l; start += avg { 116 end = start + avg 117 if end > l { 118 end = l 119 } 120 wg.Add(1) 121 PushPJob(&slcJob[DT]{ 122 slcJobBase: slcJobBase[DT]{ 123 data: data, 124 start: start, 125 end: end, 126 wg: &wg, 127 }, 128 fn: fn, 129 }) 130 } 131 for idx := 0; idx < avg; idx++ { 132 fn(data[idx]) 133 } 134 wg.Wait() 135 } 136 137 func PFilter[DT1 any, DT2 comparable](data []DT1, fn func(DT1) (DT2, bool), complete func([]DT2)) { 138 l := len(data) 139 if l < _JobUnit { 140 slc := make([]DT2, 0, l) 141 for _, d := range data { 142 item, ok := fn(d) 143 if ok { 144 slc = append(slc, item) 145 } 146 } 147 if len(slc) > 0 { 148 complete(slc) 149 } 150 return 151 } 152 var wg sync.WaitGroup 153 all := make([]*ds.Array[DT2], 0, _WorkerNum) 154 avg := getAvgCount(l) 155 var end int 156 for start := avg; end < l; start += avg { 157 end = start + avg 158 if end > l { 159 end = l 160 } 161 wg.Add(1) 162 arr := ds.NewArray[DT2](end - start) 163 all = append(all, arr) 164 PushPJob(&slcJob[DT1]{ 165 slcJobBase: slcJobBase[DT1]{ 166 data: data, 167 start: start, 168 end: end, 169 wg: &wg, 170 }, 171 fn: func(d DT1) { 172 item, ok := fn(d) 173 if ok { 174 arr.Add(item) 175 } 176 }, 177 }) 178 } 179 slc := make([]DT2, 0, avg) 180 for idx := 0; idx < avg; idx++ { 181 item, ok := fn(data[idx]) 182 if ok { 183 slc = append(slc, item) 184 } 185 } 186 if len(slc) > 0 { 187 complete(slc) 188 } 189 wg.Wait() 190 for _, arr := range all { 191 if arr.Count() > 0 { 192 complete(arr.Values()) 193 } 194 } 195 } 196 197 func PParams[DT any](data []DT, fn func(DT, []any), params ...any) { 198 l := len(data) 199 if l < _JobUnit { 200 for _, d := range data { 201 fn(d, params) 202 } 203 return 204 } 205 var wg sync.WaitGroup 206 avg := getAvgCount(l) 207 var end int 208 for start := avg; end < l; start += avg { 209 end = start + avg 210 if end > l { 211 end = l 212 } 213 wg.Add(1) 214 PushPJob(&slcJobParams[DT]{ 215 slcJobBase: slcJobBase[DT]{ 216 data: data, 217 start: start, 218 end: end, 219 wg: &wg, 220 }, 221 fn: fn, 222 params: params, 223 }) 224 } 225 for idx := 0; idx < avg; idx++ { 226 fn(data[idx], params) 227 } 228 wg.Wait() 229 } 230 231 func PToFnLink[DT any](data []DT, fn func(DT, *ds.FnLink)) { 232 l := len(data) 233 if l <= _JobUnit { 234 buffer := ds.NewFnLink() 235 for _, d := range data { 236 fn(d, buffer) 237 } 238 buffer.Invoke() 239 buffer.Dispose() 240 return 241 } 242 var wg sync.WaitGroup 243 avg := getAvgCount(l) 244 var end int 245 buffers := make([]*ds.FnLink, 0, _WorkerNum) 246 for start := avg; end < l; start += avg { 247 end = start + avg 248 if end > l { 249 end = l 250 } 251 wg.Add(1) 252 buffer := ds.NewFnLink() 253 buffers = append(buffers, buffer) 254 PushPJob(&slcToFnJob[DT]{ 255 slcToFnJobBase: slcToFnJobBase[DT]{ 256 buffer: buffer, 257 data: data, 258 start: start, 259 end: end, 260 wg: &wg, 261 }, 262 fn: fn, 263 }) 264 } 265 buffer := ds.NewFnLink() 266 for idx := 0; idx < avg; idx++ { 267 fn(data[idx], buffer) 268 } 269 buffer.Invoke() 270 buffer.Dispose() 271 wg.Wait() 272 for _, b := range buffers { 273 b.Invoke() 274 b.Dispose() 275 } 276 } 277 278 func PParamsToFnLink[DT any](data []DT, fn func(DT, []any, *ds.FnLink), params ...any) { 279 l := len(data) 280 if l <= _JobUnit { 281 buffer := ds.NewFnLink() 282 for _, d := range data { 283 fn(d, params, buffer) 284 } 285 buffer.Invoke() 286 buffer.Dispose() 287 return 288 } 289 var wg sync.WaitGroup 290 avg := getAvgCount(l) 291 var end int 292 buffers := make([]*ds.FnLink, 0, _WorkerNum) 293 for start := avg; end < l; start += avg { 294 end = start + avg 295 if end > l { 296 end = l 297 } 298 wg.Add(1) 299 buffer := ds.NewFnLink() 300 buffers = append(buffers, buffer) 301 PushPJob(&slcToFnJobParams[DT]{ 302 slcToFnJobBase: slcToFnJobBase[DT]{ 303 buffer: buffer, 304 data: data, 305 start: start, 306 end: end, 307 wg: &wg, 308 }, 309 fn: fn, 310 params: params, 311 }) 312 } 313 buffer := ds.NewFnLink() 314 for idx := 0; idx < avg; idx++ { 315 fn(data[idx], params, buffer) 316 } 317 buffer.Invoke() 318 buffer.Dispose() 319 wg.Wait() 320 for _, b := range buffers { 321 b.Invoke() 322 b.Dispose() 323 } 324 } 325 326 func PToLink[InT, OutT any](data []InT, fn func(InT, *ds.Link[OutT]), 327 pcr func(*ds.Link[OutT])) { 328 l := len(data) 329 if l <= _JobUnit { 330 buffer := ds.NewLink[OutT]() 331 for _, d := range data { 332 fn(d, buffer) 333 } 334 pcr(buffer) 335 return 336 } 337 var wg sync.WaitGroup 338 avg := getAvgCount(l) 339 var end int 340 buffers := make([]*ds.Link[OutT], 0, _WorkerNum) 341 for start := avg; end < l; start += avg { 342 end = start + avg 343 if end > l { 344 end = l 345 } 346 wg.Add(1) 347 buffer := ds.NewLink[OutT]() 348 buffers = append(buffers, buffer) 349 PushPJob(&slcToLnkJob[InT, OutT]{ 350 buffer: buffer, 351 data: data, 352 start: start, 353 end: end, 354 fn: fn, 355 wg: &wg, 356 }) 357 } 358 buffer := ds.NewLink[OutT]() 359 for idx := 0; idx < avg; idx++ { 360 fn(data[idx], buffer) 361 } 362 pcr(buffer) 363 wg.Wait() 364 for _, b := range buffers { 365 pcr(b) 366 } 367 } 368 369 func PParamsToToLink[InT, OutT any](data []InT, fn func(InT, []any, *ds.Link[OutT]), 370 pcr func(*ds.Link[OutT]), params ...any) { 371 l := len(data) 372 if l <= _JobUnit { 373 buffer := ds.NewLink[OutT]() 374 for _, d := range data { 375 fn(d, params, buffer) 376 } 377 pcr(buffer) 378 return 379 } 380 count := l / _WorkerNum 381 if l%_WorkerNum != 0 { 382 count++ 383 } 384 var wg sync.WaitGroup 385 avg := getAvgCount(l) 386 var end int 387 buffers := make([]*ds.Link[OutT], 0, _WorkerNum) 388 for start := avg; end < l; start += avg { 389 end = start + avg 390 if end > l { 391 end = l 392 } 393 wg.Add(1) 394 buffer := ds.NewLink[OutT]() 395 buffers = append(buffers, buffer) 396 PushPJob(&slcToLnkJobParams[InT, OutT]{ 397 buffer: buffer, 398 data: data, 399 start: start, 400 end: end, 401 fn: fn, 402 wg: &wg, 403 params: params, 404 }) 405 } 406 buffer := ds.NewLink[OutT]() 407 for idx := 0; idx < avg; idx++ { 408 fn(data[idx], params, buffer) 409 } 410 pcr(buffer) 411 wg.Wait() 412 for _, b := range buffers { 413 pcr(b) 414 } 415 } 416 417 func newParallelWorker() *parallelWorker { 418 return ¶llelWorker{ 419 jobCh: make(chan IJob, 32), 420 } 421 } 422 423 type parallelWorker struct { 424 jobCh chan IJob 425 } 426 427 func (w *parallelWorker) PushJob(job IJob) { 428 w.jobCh <- job 429 } 430 431 func (w *parallelWorker) start() { 432 for j := range w.jobCh { 433 j.Do() 434 } 435 } 436 437 type IJob interface { 438 Do() 439 } 440 441 type fnJob struct { 442 fns []util.FnAnySlc 443 start, end int 444 wg *sync.WaitGroup 445 params []any 446 } 447 448 func (j *fnJob) Do() { 449 for i := j.start; i < j.end; i++ { 450 j.fns[i](j.params) 451 } 452 j.wg.Done() 453 } 454 455 type slcJobBase[DT any] struct { 456 data []DT 457 start, end int 458 fn func(int, DT) 459 wg *sync.WaitGroup 460 } 461 462 type slcJob[DT any] struct { 463 slcJobBase[DT] 464 fn func(DT) 465 } 466 467 func (j *slcJob[DT]) Do() { 468 for i := j.start; i < j.end; i++ { 469 j.fn(j.data[i]) 470 } 471 j.wg.Done() 472 } 473 474 type slcJobParams[DT any] struct { 475 slcJobBase[DT] 476 fn func(DT, []any) 477 params []any 478 } 479 480 func (j *slcJobParams[DT]) Do() { 481 for i := j.start; i < j.end; i++ { 482 j.fn(j.data[i], j.params) 483 } 484 j.wg.Done() 485 } 486 487 type slcToLnkJob[DT, BT any] struct { 488 buffer *ds.Link[BT] 489 data []DT 490 start, end int 491 fn func(DT, *ds.Link[BT]) 492 wg *sync.WaitGroup 493 } 494 495 func (j *slcToLnkJob[DT, BT]) Do() { 496 for i := j.start; i < j.end; i++ { 497 j.fn(j.data[i], j.buffer) 498 } 499 j.wg.Done() 500 } 501 502 type slcToLnkJobParams[DT, BT any] struct { 503 buffer *ds.Link[BT] 504 data []DT 505 start, end int 506 fn func(DT, []any, *ds.Link[BT]) 507 wg *sync.WaitGroup 508 params []any 509 } 510 511 func (j *slcToLnkJobParams[DT, BT]) Do() { 512 for i := j.start; i < j.end; i++ { 513 j.fn(j.data[i], j.params, j.buffer) 514 } 515 j.wg.Done() 516 } 517 518 type slcToFnJobBase[DT any] struct { 519 buffer *ds.FnLink 520 data []DT 521 start, end int 522 wg *sync.WaitGroup 523 } 524 525 type slcToFnJob[DT any] struct { 526 slcToFnJobBase[DT] 527 fn func(DT, *ds.FnLink) 528 } 529 530 func (j *slcToFnJob[DT]) Do() { 531 for i := j.start; i < j.end; i++ { 532 j.fn(j.data[i], j.buffer) 533 } 534 j.wg.Done() 535 } 536 537 type slcToFnJobParams[DT any] struct { 538 slcToFnJobBase[DT] 539 fn func(DT, []any, *ds.FnLink) 540 params []any 541 } 542 543 func (j *slcToFnJobParams[DT]) Do() { 544 for i := j.start; i < j.end; i++ { 545 j.fn(j.data[i], j.params, j.buffer) 546 } 547 j.wg.Done() 548 }