github.com/matrixorigin/matrixone@v1.2.0/pkg/sql/compile/operator.go (about) 1 // Copyright 2021 Matrix Origin 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package compile 16 17 import ( 18 "context" 19 "fmt" 20 21 "github.com/matrixorigin/matrixone/pkg/sql/colexec/aggexec" 22 23 "github.com/matrixorigin/matrixone/pkg/sql/colexec/indexbuild" 24 "github.com/matrixorigin/matrixone/pkg/sql/colexec/indexjoin" 25 26 "github.com/google/uuid" 27 "github.com/matrixorigin/matrixone/pkg/catalog" 28 "github.com/matrixorigin/matrixone/pkg/common/moerr" 29 "github.com/matrixorigin/matrixone/pkg/container/batch" 30 "github.com/matrixorigin/matrixone/pkg/container/types" 31 "github.com/matrixorigin/matrixone/pkg/container/vector" 32 "github.com/matrixorigin/matrixone/pkg/defines" 33 "github.com/matrixorigin/matrixone/pkg/pb/pipeline" 34 "github.com/matrixorigin/matrixone/pkg/pb/plan" 35 "github.com/matrixorigin/matrixone/pkg/pb/timestamp" 36 "github.com/matrixorigin/matrixone/pkg/sql/colexec" 37 "github.com/matrixorigin/matrixone/pkg/sql/colexec/anti" 38 "github.com/matrixorigin/matrixone/pkg/sql/colexec/connector" 39 "github.com/matrixorigin/matrixone/pkg/sql/colexec/deletion" 40 "github.com/matrixorigin/matrixone/pkg/sql/colexec/dispatch" 41 "github.com/matrixorigin/matrixone/pkg/sql/colexec/external" 42 "github.com/matrixorigin/matrixone/pkg/sql/colexec/fill" 43 "github.com/matrixorigin/matrixone/pkg/sql/colexec/fuzzyfilter" 44 "github.com/matrixorigin/matrixone/pkg/sql/colexec/group" 45 "github.com/matrixorigin/matrixone/pkg/sql/colexec/hashbuild" 46 "github.com/matrixorigin/matrixone/pkg/sql/colexec/insert" 47 "github.com/matrixorigin/matrixone/pkg/sql/colexec/intersect" 48 "github.com/matrixorigin/matrixone/pkg/sql/colexec/intersectall" 49 "github.com/matrixorigin/matrixone/pkg/sql/colexec/join" 50 "github.com/matrixorigin/matrixone/pkg/sql/colexec/left" 51 "github.com/matrixorigin/matrixone/pkg/sql/colexec/limit" 52 "github.com/matrixorigin/matrixone/pkg/sql/colexec/lockop" 53 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopanti" 54 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopjoin" 55 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopleft" 56 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopmark" 57 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopsemi" 58 "github.com/matrixorigin/matrixone/pkg/sql/colexec/loopsingle" 59 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mark" 60 "github.com/matrixorigin/matrixone/pkg/sql/colexec/merge" 61 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergecte" 62 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergegroup" 63 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergelimit" 64 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergeoffset" 65 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergeorder" 66 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergerecursive" 67 "github.com/matrixorigin/matrixone/pkg/sql/colexec/mergetop" 68 "github.com/matrixorigin/matrixone/pkg/sql/colexec/minus" 69 "github.com/matrixorigin/matrixone/pkg/sql/colexec/offset" 70 "github.com/matrixorigin/matrixone/pkg/sql/colexec/onduplicatekey" 71 "github.com/matrixorigin/matrixone/pkg/sql/colexec/order" 72 "github.com/matrixorigin/matrixone/pkg/sql/colexec/partition" 73 "github.com/matrixorigin/matrixone/pkg/sql/colexec/preinsert" 74 "github.com/matrixorigin/matrixone/pkg/sql/colexec/preinsertsecondaryindex" 75 "github.com/matrixorigin/matrixone/pkg/sql/colexec/preinsertunique" 76 "github.com/matrixorigin/matrixone/pkg/sql/colexec/product" 77 "github.com/matrixorigin/matrixone/pkg/sql/colexec/projection" 78 "github.com/matrixorigin/matrixone/pkg/sql/colexec/restrict" 79 "github.com/matrixorigin/matrixone/pkg/sql/colexec/right" 80 "github.com/matrixorigin/matrixone/pkg/sql/colexec/rightanti" 81 "github.com/matrixorigin/matrixone/pkg/sql/colexec/rightsemi" 82 "github.com/matrixorigin/matrixone/pkg/sql/colexec/sample" 83 "github.com/matrixorigin/matrixone/pkg/sql/colexec/semi" 84 "github.com/matrixorigin/matrixone/pkg/sql/colexec/shuffle" 85 "github.com/matrixorigin/matrixone/pkg/sql/colexec/single" 86 "github.com/matrixorigin/matrixone/pkg/sql/colexec/source" 87 "github.com/matrixorigin/matrixone/pkg/sql/colexec/table_function" 88 "github.com/matrixorigin/matrixone/pkg/sql/colexec/timewin" 89 "github.com/matrixorigin/matrixone/pkg/sql/colexec/top" 90 "github.com/matrixorigin/matrixone/pkg/sql/colexec/window" 91 "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" 92 plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" 93 "github.com/matrixorigin/matrixone/pkg/sql/plan/function" 94 "github.com/matrixorigin/matrixone/pkg/vm" 95 "github.com/matrixorigin/matrixone/pkg/vm/engine" 96 "github.com/matrixorigin/matrixone/pkg/vm/process" 97 ) 98 99 var constBat *batch.Batch 100 101 func init() { 102 constBat = batch.NewWithSize(0) 103 constBat.SetRowCount(1) 104 } 105 106 func dupInstruction(sourceIns *vm.Instruction, regMap map[*process.WaitRegister]*process.WaitRegister, index int) vm.Instruction { 107 res := vm.Instruction{ 108 Op: sourceIns.Op, 109 Idx: sourceIns.Idx, 110 IsFirst: sourceIns.IsFirst, 111 IsLast: sourceIns.IsLast, 112 CnAddr: sourceIns.CnAddr, 113 OperatorID: sourceIns.OperatorID, 114 MaxParallel: sourceIns.MaxParallel, 115 ParallelID: sourceIns.ParallelID, 116 } 117 switch sourceIns.Op { 118 case vm.Anti: 119 t := sourceIns.Arg.(*anti.Argument) 120 arg := anti.NewArgument() 121 arg.Cond = t.Cond 122 arg.Typs = t.Typs 123 arg.Conditions = t.Conditions 124 arg.Result = t.Result 125 arg.HashOnPK = t.HashOnPK 126 arg.IsShuffle = t.IsShuffle 127 res.Arg = arg 128 case vm.Group: 129 t := sourceIns.Arg.(*group.Argument) 130 arg := group.NewArgument() 131 arg.IsShuffle = t.IsShuffle 132 arg.PreAllocSize = t.PreAllocSize 133 arg.NeedEval = t.NeedEval 134 arg.Ibucket = t.Ibucket 135 arg.Nbucket = t.Nbucket 136 arg.Exprs = t.Exprs 137 arg.Types = t.Types 138 arg.Aggs = t.Aggs 139 res.Arg = arg 140 case vm.Sample: 141 t := sourceIns.Arg.(*sample.Argument) 142 res.Arg = t.SimpleDup() 143 case vm.Join: 144 t := sourceIns.Arg.(*join.Argument) 145 arg := join.NewArgument() 146 arg.Result = t.Result 147 arg.Cond = t.Cond 148 arg.Typs = t.Typs 149 arg.Conditions = t.Conditions 150 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 151 arg.HashOnPK = t.HashOnPK 152 arg.IsShuffle = t.IsShuffle 153 res.Arg = arg 154 case vm.Left: 155 t := sourceIns.Arg.(*left.Argument) 156 arg := left.NewArgument() 157 arg.Cond = t.Cond 158 arg.Result = t.Result 159 arg.Typs = t.Typs 160 arg.Conditions = t.Conditions 161 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 162 arg.HashOnPK = t.HashOnPK 163 arg.IsShuffle = t.IsShuffle 164 res.Arg = arg 165 case vm.Right: 166 t := sourceIns.Arg.(*right.Argument) 167 arg := right.NewArgument() 168 arg.Cond = t.Cond 169 arg.Result = t.Result 170 arg.RightTypes = t.RightTypes 171 arg.LeftTypes = t.LeftTypes 172 arg.Conditions = t.Conditions 173 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 174 arg.HashOnPK = t.HashOnPK 175 arg.IsShuffle = t.IsShuffle 176 res.Arg = arg 177 case vm.RightSemi: 178 t := sourceIns.Arg.(*rightsemi.Argument) 179 arg := rightsemi.NewArgument() 180 arg.Cond = t.Cond 181 arg.Result = t.Result 182 arg.RightTypes = t.RightTypes 183 arg.Conditions = t.Conditions 184 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 185 arg.HashOnPK = t.HashOnPK 186 arg.IsShuffle = t.IsShuffle 187 res.Arg = arg 188 case vm.RightAnti: 189 t := sourceIns.Arg.(*rightanti.Argument) 190 arg := rightanti.NewArgument() 191 arg.Cond = t.Cond 192 arg.Result = t.Result 193 arg.RightTypes = t.RightTypes 194 arg.Conditions = t.Conditions 195 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 196 arg.HashOnPK = t.HashOnPK 197 arg.IsShuffle = t.IsShuffle 198 res.Arg = arg 199 case vm.Limit: 200 t := sourceIns.Arg.(*limit.Argument) 201 arg := limit.NewArgument() 202 arg.Limit = t.Limit 203 res.Arg = arg 204 case vm.LoopAnti: 205 t := sourceIns.Arg.(*loopanti.Argument) 206 arg := loopanti.NewArgument() 207 arg.Result = t.Result 208 arg.Cond = t.Cond 209 arg.Typs = t.Typs 210 res.Arg = arg 211 case vm.LoopJoin: 212 t := sourceIns.Arg.(*loopjoin.Argument) 213 arg := loopjoin.NewArgument() 214 arg.Result = t.Result 215 arg.Cond = t.Cond 216 arg.Typs = t.Typs 217 res.Arg = arg 218 case vm.IndexJoin: 219 t := sourceIns.Arg.(*indexjoin.Argument) 220 arg := indexjoin.NewArgument() 221 arg.Result = t.Result 222 arg.Typs = t.Typs 223 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 224 res.Arg = arg 225 case vm.LoopLeft: 226 t := sourceIns.Arg.(*loopleft.Argument) 227 arg := loopleft.NewArgument() 228 arg.Cond = t.Cond 229 arg.Typs = t.Typs 230 arg.Result = t.Result 231 res.Arg = arg 232 case vm.LoopSemi: 233 t := sourceIns.Arg.(*loopsemi.Argument) 234 arg := loopsemi.NewArgument() 235 arg.Result = t.Result 236 arg.Cond = t.Cond 237 arg.Typs = t.Typs 238 res.Arg = arg 239 case vm.LoopSingle: 240 t := sourceIns.Arg.(*loopsingle.Argument) 241 arg := loopsingle.NewArgument() 242 arg.Result = t.Result 243 arg.Cond = t.Cond 244 arg.Typs = t.Typs 245 res.Arg = arg 246 case vm.LoopMark: 247 t := sourceIns.Arg.(*loopmark.Argument) 248 arg := loopmark.NewArgument() 249 arg.Result = t.Result 250 arg.Cond = t.Cond 251 arg.Typs = t.Typs 252 res.Arg = arg 253 case vm.Offset: 254 t := sourceIns.Arg.(*offset.Argument) 255 arg := offset.NewArgument() 256 arg.Offset = t.Offset 257 res.Arg = arg 258 case vm.Order: 259 t := sourceIns.Arg.(*order.Argument) 260 arg := order.NewArgument() 261 arg.OrderBySpec = t.OrderBySpec 262 res.Arg = arg 263 case vm.Product: 264 t := sourceIns.Arg.(*product.Argument) 265 arg := product.NewArgument() 266 arg.Result = t.Result 267 arg.Typs = t.Typs 268 arg.IsShuffle = t.IsShuffle 269 res.Arg = arg 270 case vm.Projection: 271 t := sourceIns.Arg.(*projection.Argument) 272 arg := projection.NewArgument() 273 arg.Es = t.Es 274 res.Arg = arg 275 case vm.Restrict: 276 t := sourceIns.Arg.(*restrict.Argument) 277 arg := restrict.NewArgument() 278 arg.E = t.E 279 res.Arg = arg 280 case vm.Semi: 281 t := sourceIns.Arg.(*semi.Argument) 282 arg := semi.NewArgument() 283 arg.Result = t.Result 284 arg.Cond = t.Cond 285 arg.Typs = t.Typs 286 arg.Conditions = t.Conditions 287 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 288 arg.HashOnPK = t.HashOnPK 289 arg.IsShuffle = t.IsShuffle 290 res.Arg = arg 291 case vm.Single: 292 t := sourceIns.Arg.(*single.Argument) 293 arg := single.NewArgument() 294 arg.Result = t.Result 295 arg.Cond = t.Cond 296 arg.Typs = t.Typs 297 arg.Conditions = t.Conditions 298 arg.RuntimeFilterSpecs = t.RuntimeFilterSpecs 299 arg.HashOnPK = t.HashOnPK 300 res.Arg = arg 301 case vm.Top: 302 t := sourceIns.Arg.(*top.Argument) 303 arg := top.NewArgument() 304 arg.Limit = t.Limit 305 arg.TopValueTag = t.TopValueTag 306 arg.Fs = t.Fs 307 res.Arg = arg 308 case vm.Intersect: 309 t := sourceIns.Arg.(*intersect.Argument) 310 arg := intersect.NewArgument() 311 arg.IBucket = t.IBucket 312 arg.NBucket = t.NBucket 313 res.Arg = arg 314 case vm.Minus: // 2 315 t := sourceIns.Arg.(*minus.Argument) 316 arg := minus.NewArgument() 317 arg.IBucket = t.IBucket 318 arg.NBucket = t.NBucket 319 res.Arg = arg 320 case vm.IntersectAll: 321 t := sourceIns.Arg.(*intersectall.Argument) 322 arg := intersectall.NewArgument() 323 arg.IBucket = t.IBucket 324 arg.NBucket = t.NBucket 325 res.Arg = arg 326 case vm.Merge: 327 t := sourceIns.Arg.(*merge.Argument) 328 arg := merge.NewArgument() 329 arg.SinkScan = t.SinkScan 330 res.Arg = arg 331 case vm.MergeRecursive: 332 res.Arg = mergerecursive.NewArgument() 333 case vm.MergeCTE: 334 res.Arg = mergecte.NewArgument() 335 case vm.MergeGroup: 336 t := sourceIns.Arg.(*mergegroup.Argument) 337 arg := mergegroup.NewArgument() 338 arg.NeedEval = t.NeedEval 339 arg.PartialResults = t.PartialResults 340 arg.PartialResultTypes = t.PartialResultTypes 341 res.Arg = arg 342 case vm.MergeLimit: 343 t := sourceIns.Arg.(*mergelimit.Argument) 344 arg := mergelimit.NewArgument() 345 arg.Limit = t.Limit 346 res.Arg = arg 347 case vm.MergeOffset: 348 t := sourceIns.Arg.(*mergeoffset.Argument) 349 arg := mergeoffset.NewArgument() 350 arg.Offset = t.Offset 351 res.Arg = arg 352 case vm.MergeTop: 353 t := sourceIns.Arg.(*mergetop.Argument) 354 arg := mergetop.NewArgument() 355 arg.Limit = t.Limit 356 arg.Fs = t.Fs 357 res.Arg = arg 358 case vm.MergeOrder: 359 t := sourceIns.Arg.(*mergeorder.Argument) 360 arg := mergeorder.NewArgument() 361 arg.OrderBySpecs = t.OrderBySpecs 362 res.Arg = arg 363 case vm.Mark: 364 t := sourceIns.Arg.(*mark.Argument) 365 arg := mark.NewArgument() 366 arg.Result = t.Result 367 arg.Conditions = t.Conditions 368 arg.Typs = t.Typs 369 arg.Cond = t.Cond 370 arg.OnList = t.OnList 371 arg.HashOnPK = t.HashOnPK 372 res.Arg = arg 373 case vm.TableFunction: 374 t := sourceIns.Arg.(*table_function.Argument) 375 arg := table_function.NewArgument() 376 arg.FuncName = t.FuncName 377 arg.Args = t.Args 378 arg.Rets = t.Rets 379 arg.Attrs = t.Attrs 380 arg.Params = t.Params 381 res.Arg = arg 382 case vm.HashBuild: 383 t := sourceIns.Arg.(*hashbuild.Argument) 384 arg := hashbuild.NewArgument() 385 arg.NeedHashMap = t.NeedHashMap 386 arg.NeedExpr = t.NeedExpr 387 arg.Ibucket = t.Ibucket 388 arg.Nbucket = t.Nbucket 389 arg.Typs = t.Typs 390 arg.Conditions = t.Conditions 391 arg.HashOnPK = t.HashOnPK 392 arg.NeedMergedBatch = t.NeedMergedBatch 393 arg.NeedAllocateSels = t.NeedAllocateSels 394 res.Arg = arg 395 case vm.External: 396 t := sourceIns.Arg.(*external.Argument) 397 res.Arg = external.NewArgument().WithEs( 398 &external.ExternalParam{ 399 ExParamConst: external.ExParamConst{ 400 Attrs: t.Es.Attrs, 401 Cols: t.Es.Cols, 402 Idx: index, 403 Name2ColIndex: t.Es.Name2ColIndex, 404 CreateSql: t.Es.CreateSql, 405 FileList: t.Es.FileList, 406 FileSize: t.Es.FileSize, 407 FileOffsetTotal: t.Es.FileOffsetTotal, 408 Extern: t.Es.Extern, 409 }, 410 ExParam: external.ExParam{ 411 Filter: &external.FilterParam{ 412 FilterExpr: t.Es.Filter.FilterExpr, 413 }, 414 Fileparam: &external.ExFileparam{ 415 End: t.Es.Fileparam.End, 416 FileCnt: t.Es.Fileparam.FileCnt, 417 FileFin: t.Es.Fileparam.FileFin, 418 FileIndex: t.Es.Fileparam.FileIndex, 419 }, 420 }, 421 }, 422 ) 423 case vm.Source: 424 t := sourceIns.Arg.(*source.Argument) 425 arg := source.NewArgument() 426 arg.TblDef = t.TblDef 427 arg.Limit = t.Limit 428 arg.Offset = t.Offset 429 arg.Configs = t.Configs 430 res.Arg = arg 431 case vm.Connector: 432 ok := false 433 if regMap != nil { 434 arg := connector.NewArgument() 435 sourceReg := sourceIns.Arg.(*connector.Argument).Reg 436 if arg.Reg, ok = regMap[sourceReg]; !ok { 437 panic("nonexistent wait register") 438 } 439 res.Arg = arg 440 } 441 case vm.Shuffle: 442 sourceArg := sourceIns.Arg.(*shuffle.Argument) 443 arg := shuffle.NewArgument() 444 arg.ShuffleType = sourceArg.ShuffleType 445 arg.ShuffleColIdx = sourceArg.ShuffleColIdx 446 arg.ShuffleColMax = sourceArg.ShuffleColMax 447 arg.ShuffleColMin = sourceArg.ShuffleColMin 448 arg.AliveRegCnt = sourceArg.AliveRegCnt 449 arg.ShuffleRangeInt64 = sourceArg.ShuffleRangeInt64 450 arg.ShuffleRangeUint64 = sourceArg.ShuffleRangeUint64 451 res.Arg = arg 452 case vm.Dispatch: 453 ok := false 454 if regMap != nil { 455 sourceArg := sourceIns.Arg.(*dispatch.Argument) 456 arg := dispatch.NewArgument() 457 arg.IsSink = sourceArg.IsSink 458 arg.RecSink = sourceArg.RecSink 459 arg.FuncId = sourceArg.FuncId 460 arg.LocalRegs = make([]*process.WaitRegister, len(sourceArg.LocalRegs)) 461 arg.RemoteRegs = make([]colexec.ReceiveInfo, len(sourceArg.RemoteRegs)) 462 for j := range arg.LocalRegs { 463 sourceReg := sourceArg.LocalRegs[j] 464 if arg.LocalRegs[j], ok = regMap[sourceReg]; !ok { 465 panic("nonexistent wait register") 466 } 467 } 468 for j := range arg.RemoteRegs { 469 arg.RemoteRegs[j] = sourceArg.RemoteRegs[j] 470 } 471 res.Arg = arg 472 } 473 case vm.Insert: 474 t := sourceIns.Arg.(*insert.Argument) 475 arg := insert.NewArgument() 476 arg.InsertCtx = t.InsertCtx 477 arg.ToWriteS3 = t.ToWriteS3 478 res.Arg = arg 479 case vm.PreInsert: 480 t := sourceIns.Arg.(*preinsert.Argument) 481 arg := preinsert.NewArgument() 482 arg.SchemaName = t.SchemaName 483 arg.TableDef = t.TableDef 484 arg.Attrs = t.Attrs 485 arg.IsUpdate = t.IsUpdate 486 arg.HasAutoCol = t.HasAutoCol 487 arg.EstimatedRowCount = t.EstimatedRowCount 488 res.Arg = arg 489 case vm.Deletion: 490 t := sourceIns.Arg.(*deletion.Argument) 491 arg := deletion.NewArgument() 492 arg.IBucket = t.IBucket 493 arg.Nbucket = t.Nbucket 494 arg.DeleteCtx = t.DeleteCtx 495 arg.RemoteDelete = t.RemoteDelete 496 arg.SegmentMap = t.SegmentMap 497 res.Arg = arg 498 case vm.LockOp: 499 t := sourceIns.Arg.(*lockop.Argument) 500 arg := lockop.NewArgument() 501 *arg = *t 502 res.Arg = arg 503 case vm.FuzzyFilter: 504 t := sourceIns.Arg.(*fuzzyfilter.Argument) 505 arg := fuzzyfilter.NewArgument() 506 arg.N = t.N 507 arg.PkName = t.PkName 508 arg.PkTyp = t.PkTyp 509 res.Arg = arg 510 default: 511 panic(fmt.Sprintf("unexpected instruction type '%d' to dup", sourceIns.Op)) 512 } 513 return res 514 } 515 516 func constructRestrict(n *plan.Node, filterExpr *plan2.Expr) *restrict.Argument { 517 arg := restrict.NewArgument() 518 arg.E = filterExpr 519 arg.IsEnd = n.IsEnd 520 return arg 521 } 522 523 func constructDeletion(n *plan.Node, eg engine.Engine, proc *process.Process) (*deletion.Argument, error) { 524 oldCtx := n.DeleteCtx 525 delCtx := &deletion.DeleteCtx{ 526 Ref: oldCtx.Ref, 527 RowIdIdx: int(oldCtx.RowIdIdx), 528 CanTruncate: oldCtx.CanTruncate, 529 AddAffectedRows: oldCtx.AddAffectedRows, 530 PartitionTableIDs: oldCtx.PartitionTableIds, 531 PartitionTableNames: oldCtx.PartitionTableNames, 532 PartitionIndexInBatch: int(oldCtx.PartitionIdx), 533 PrimaryKeyIdx: int(oldCtx.PrimaryKeyIdx), 534 } 535 // get the relation instance of the original table 536 rel, _, err := getRel(proc.Ctx, proc, eg, oldCtx.Ref, nil) 537 if err != nil { 538 return nil, err 539 } 540 delCtx.Source = rel 541 if len(oldCtx.PartitionTableNames) > 0 { 542 dbSource, err := eg.Database(proc.Ctx, oldCtx.Ref.SchemaName, proc.TxnOperator) 543 if err != nil { 544 return nil, err 545 } 546 547 delCtx.PartitionSources = make([]engine.Relation, len(oldCtx.PartitionTableNames)) 548 // get the relation instances for each partition sub table 549 for i, pTableName := range oldCtx.PartitionTableNames { 550 pRel, err := dbSource.Relation(proc.Ctx, pTableName, proc) 551 if err != nil { 552 return nil, err 553 } 554 delCtx.PartitionSources[i] = pRel 555 } 556 } 557 558 arg := deletion.NewArgument() 559 arg.DeleteCtx = delCtx 560 return arg, nil 561 } 562 563 func constructOnduplicateKey(n *plan.Node, eg engine.Engine) *onduplicatekey.Argument { 564 oldCtx := n.OnDuplicateKey 565 arg := onduplicatekey.NewArgument() 566 arg.Engine = eg 567 arg.OnDuplicateIdx = oldCtx.OnDuplicateIdx 568 arg.OnDuplicateExpr = oldCtx.OnDuplicateExpr 569 arg.Attrs = oldCtx.Attrs 570 arg.InsertColCount = oldCtx.InsertColCount 571 arg.UniqueCols = oldCtx.UniqueCols 572 arg.UniqueColCheckExpr = oldCtx.UniqueColCheckExpr 573 arg.IsIgnore = oldCtx.IsIgnore 574 return arg 575 } 576 577 func constructFuzzyFilter(c *Compile, n, right *plan.Node) *fuzzyfilter.Argument { 578 pkName := n.TableDef.Pkey.PkeyColName 579 var pkTyp plan.Type 580 if pkName == catalog.CPrimaryKeyColName { 581 pkTyp = n.TableDef.Pkey.CompPkeyCol.Typ 582 } else { 583 cols := n.TableDef.Cols 584 for _, c := range cols { 585 if c.Name == pkName { 586 pkTyp = c.Typ 587 } 588 } 589 } 590 591 arg := fuzzyfilter.NewArgument() 592 arg.PkName = pkName 593 arg.PkTyp = pkTyp 594 arg.N = right.Stats.Outcnt 595 if len(n.RuntimeFilterBuildList) > 0 { 596 arg.RuntimeFilterSpec = n.RuntimeFilterBuildList[0] 597 } 598 return arg 599 } 600 601 func constructPreInsert(ns []*plan.Node, n *plan.Node, eg engine.Engine, proc *process.Process) (*preinsert.Argument, error) { 602 preCtx := n.PreInsertCtx 603 schemaName := preCtx.Ref.SchemaName 604 605 //var attrs []string 606 attrs := make([]string, 0) 607 for _, col := range preCtx.TableDef.Cols { 608 if col.Hidden && col.Name != catalog.FakePrimaryKeyColName { 609 continue 610 } 611 attrs = append(attrs, col.Name) 612 } 613 614 ctx := proc.Ctx 615 txnOp := proc.TxnOperator 616 if n.ScanSnapshot != nil && n.ScanSnapshot.TS != nil { 617 if !n.ScanSnapshot.TS.Equal(timestamp.Timestamp{LogicalTime: 0, PhysicalTime: 0}) && 618 n.ScanSnapshot.TS.Less(proc.TxnOperator.Txn().SnapshotTS) { 619 txnOp = proc.TxnOperator.CloneSnapshotOp(*n.ScanSnapshot.TS) 620 621 if n.ScanSnapshot.Tenant != nil { 622 ctx = context.WithValue(ctx, defines.TenantIDKey{}, n.ScanSnapshot.Tenant.TenantID) 623 } 624 } 625 } 626 627 if preCtx.Ref.SchemaName != "" { 628 dbSource, err := eg.Database(ctx, preCtx.Ref.SchemaName, txnOp) 629 if err != nil { 630 return nil, err 631 } 632 if _, err = dbSource.Relation(ctx, preCtx.Ref.ObjName, proc); err != nil { 633 schemaName = defines.TEMPORARY_DBNAME 634 } 635 } 636 637 arg := preinsert.NewArgument() 638 arg.Ctx = proc.Ctx 639 arg.HasAutoCol = preCtx.HasAutoCol 640 arg.SchemaName = schemaName 641 arg.TableDef = preCtx.TableDef 642 arg.Attrs = attrs 643 arg.IsUpdate = preCtx.IsUpdate 644 arg.EstimatedRowCount = int64(ns[n.Children[0]].Stats.Outcnt) 645 646 return arg, nil 647 } 648 649 func constructPreInsertUk(n *plan.Node, proc *process.Process) (*preinsertunique.Argument, error) { 650 preCtx := n.PreInsertUkCtx 651 arg := preinsertunique.NewArgument() 652 arg.Ctx = proc.Ctx 653 arg.PreInsertCtx = preCtx 654 return arg, nil 655 } 656 657 func constructPreInsertSk(n *plan.Node, proc *process.Process) (*preinsertsecondaryindex.Argument, error) { 658 arg := preinsertsecondaryindex.NewArgument() 659 arg.Ctx = proc.Ctx 660 arg.PreInsertCtx = n.PreInsertSkCtx 661 return arg, nil 662 } 663 664 func constructLockOp(n *plan.Node, eng engine.Engine) (*lockop.Argument, error) { 665 arg := lockop.NewArgumentByEngine(eng) 666 for _, target := range n.LockTargets { 667 typ := plan2.MakeTypeByPlan2Type(target.PrimaryColTyp) 668 if target.IsPartitionTable { 669 arg.AddLockTargetWithPartition(target.GetPartitionTableIds(), target.GetPrimaryColIdxInBat(), typ, target.GetRefreshTsIdxInBat(), target.GetFilterColIdxInBat()) 670 } else { 671 arg.AddLockTarget(target.GetTableId(), target.GetPrimaryColIdxInBat(), typ, target.GetRefreshTsIdxInBat()) 672 } 673 674 } 675 for _, target := range n.LockTargets { 676 if target.LockTable { 677 if target.IsPartitionTable { 678 for _, pTblId := range target.PartitionTableIds { 679 arg.LockTable(pTblId, false) 680 } 681 } else { 682 arg.LockTable(target.TableId, false) 683 } 684 } 685 } 686 return arg, nil 687 } 688 689 func constructInsert(n *plan.Node, eg engine.Engine, proc *process.Process) (*insert.Argument, error) { 690 oldCtx := n.InsertCtx 691 ctx := proc.Ctx 692 693 var attrs []string 694 for _, col := range oldCtx.TableDef.Cols { 695 if col.Name != catalog.Row_ID { 696 attrs = append(attrs, col.Name) 697 } 698 } 699 originRel, _, err := getRel(ctx, proc, eg, oldCtx.Ref, nil) 700 if err != nil { 701 return nil, err 702 } 703 newCtx := &insert.InsertCtx{ 704 Ref: oldCtx.Ref, 705 AddAffectedRows: oldCtx.AddAffectedRows, 706 Rel: originRel, 707 Attrs: attrs, 708 PartitionTableIDs: oldCtx.PartitionTableIds, 709 PartitionTableNames: oldCtx.PartitionTableNames, 710 PartitionIndexInBatch: int(oldCtx.PartitionIdx), 711 TableDef: oldCtx.TableDef, 712 } 713 714 if len(oldCtx.PartitionTableNames) > 0 { 715 dbSource, err := eg.Database(proc.Ctx, oldCtx.Ref.SchemaName, proc.TxnOperator) 716 if err != nil { 717 return nil, err 718 } 719 720 newCtx.PartitionSources = make([]engine.Relation, len(oldCtx.PartitionTableNames)) 721 // get the relation instances for each partition sub table 722 for i, pTableName := range oldCtx.PartitionTableNames { 723 pRel, err := dbSource.Relation(proc.Ctx, pTableName, proc) 724 if err != nil { 725 return nil, err 726 } 727 newCtx.PartitionSources[i] = pRel 728 } 729 } 730 731 arg := insert.NewArgument() 732 arg.InsertCtx = newCtx 733 return arg, nil 734 } 735 736 func constructProjection(n *plan.Node) *projection.Argument { 737 arg := projection.NewArgument() 738 arg.Es = n.ProjectList 739 return arg 740 } 741 742 func constructExternal(n *plan.Node, param *tree.ExternParam, ctx context.Context, fileList []string, FileSize []int64, fileOffset []*pipeline.FileOffset) *external.Argument { 743 attrs := make([]string, len(n.TableDef.Cols)) 744 for j, col := range n.TableDef.Cols { 745 attrs[j] = col.Name 746 } 747 return external.NewArgument().WithEs( 748 &external.ExternalParam{ 749 ExParamConst: external.ExParamConst{ 750 Attrs: attrs, 751 Cols: n.TableDef.Cols, 752 Extern: param, 753 Name2ColIndex: n.TableDef.Name2ColIndex, 754 FileOffsetTotal: fileOffset, 755 CreateSql: n.TableDef.Createsql, 756 Ctx: ctx, 757 FileList: fileList, 758 FileSize: FileSize, 759 ClusterTable: n.GetClusterTable(), 760 }, 761 ExParam: external.ExParam{ 762 Fileparam: new(external.ExFileparam), 763 Filter: &external.FilterParam{ 764 FilterExpr: colexec.RewriteFilterExprList(n.FilterList), 765 }, 766 }, 767 }, 768 ) 769 } 770 771 func constructStream(n *plan.Node, p [2]int64) *source.Argument { 772 arg := source.NewArgument() 773 arg.TblDef = n.TableDef 774 arg.Offset = p[0] 775 arg.Limit = p[1] 776 return arg 777 } 778 779 func constructTableFunction(n *plan.Node) *table_function.Argument { 780 attrs := make([]string, len(n.TableDef.Cols)) 781 for j, col := range n.TableDef.Cols { 782 attrs[j] = col.Name 783 } 784 arg := table_function.NewArgument() 785 arg.Attrs = attrs 786 arg.Rets = n.TableDef.Cols 787 arg.Args = n.TblFuncExprList 788 arg.FuncName = n.TableDef.TblFunc.Name 789 arg.Params = n.TableDef.TblFunc.Param 790 return arg 791 } 792 793 func constructTop(n *plan.Node, topN int64) *top.Argument { 794 arg := top.NewArgument() 795 arg.Fs = n.OrderBy 796 arg.Limit = topN 797 if len(n.SendMsgList) > 0 { 798 arg.TopValueTag = n.SendMsgList[0].MsgTag 799 } 800 return arg 801 } 802 803 func constructJoin(n *plan.Node, typs []types.Type, proc *process.Process) *join.Argument { 804 result := make([]colexec.ResultPos, len(n.ProjectList)) 805 for i, expr := range n.ProjectList { 806 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 807 } 808 cond, conds := extraJoinConditions(n.OnList) 809 810 arg := join.NewArgument() 811 arg.Typs = typs 812 arg.Result = result 813 arg.Cond = cond 814 arg.Conditions = constructJoinConditions(conds, proc) 815 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 816 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 817 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 818 return arg 819 } 820 821 func constructSemi(n *plan.Node, typs []types.Type, proc *process.Process) *semi.Argument { 822 result := make([]int32, len(n.ProjectList)) 823 for i, expr := range n.ProjectList { 824 rel, pos := constructJoinResult(expr, proc) 825 if rel != 0 { 826 panic(moerr.NewNYI(proc.Ctx, "semi result '%s'", expr)) 827 } 828 result[i] = pos 829 } 830 cond, conds := extraJoinConditions(n.OnList) 831 arg := semi.NewArgument() 832 arg.Typs = typs 833 arg.Result = result 834 arg.Cond = cond 835 arg.Conditions = constructJoinConditions(conds, proc) 836 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 837 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 838 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 839 return arg 840 } 841 842 func constructLeft(n *plan.Node, typs []types.Type, proc *process.Process) *left.Argument { 843 result := make([]colexec.ResultPos, len(n.ProjectList)) 844 for i, expr := range n.ProjectList { 845 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 846 } 847 cond, conds := extraJoinConditions(n.OnList) 848 arg := left.NewArgument() 849 arg.Typs = typs 850 arg.Result = result 851 arg.Cond = cond 852 arg.Conditions = constructJoinConditions(conds, proc) 853 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 854 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 855 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 856 return arg 857 } 858 859 func constructRight(n *plan.Node, left_typs, right_typs []types.Type, Ibucket, Nbucket uint64, proc *process.Process) *right.Argument { 860 result := make([]colexec.ResultPos, len(n.ProjectList)) 861 for i, expr := range n.ProjectList { 862 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 863 } 864 cond, conds := extraJoinConditions(n.OnList) 865 arg := right.NewArgument() 866 arg.LeftTypes = left_typs 867 arg.RightTypes = right_typs 868 arg.Result = result 869 arg.Cond = cond 870 arg.Conditions = constructJoinConditions(conds, proc) 871 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 872 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 873 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 874 return arg 875 } 876 877 func constructRightSemi(n *plan.Node, right_typs []types.Type, Ibucket, Nbucket uint64, proc *process.Process) *rightsemi.Argument { 878 result := make([]int32, len(n.ProjectList)) 879 for i, expr := range n.ProjectList { 880 _, result[i] = constructJoinResult(expr, proc) 881 } 882 cond, conds := extraJoinConditions(n.OnList) 883 // 使用NewArgument来初始化 884 arg := rightsemi.NewArgument() 885 arg.RightTypes = right_typs 886 arg.Result = result 887 arg.Cond = cond 888 arg.Conditions = constructJoinConditions(conds, proc) 889 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 890 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 891 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 892 return arg 893 } 894 895 func constructRightAnti(n *plan.Node, right_typs []types.Type, Ibucket, Nbucket uint64, proc *process.Process) *rightanti.Argument { 896 result := make([]int32, len(n.ProjectList)) 897 for i, expr := range n.ProjectList { 898 _, result[i] = constructJoinResult(expr, proc) 899 } 900 cond, conds := extraJoinConditions(n.OnList) 901 arg := rightanti.NewArgument() 902 arg.RightTypes = right_typs 903 arg.Result = result 904 arg.Cond = cond 905 arg.Conditions = constructJoinConditions(conds, proc) 906 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 907 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 908 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 909 return arg 910 } 911 912 func constructSingle(n *plan.Node, typs []types.Type, proc *process.Process) *single.Argument { 913 result := make([]colexec.ResultPos, len(n.ProjectList)) 914 for i, expr := range n.ProjectList { 915 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 916 } 917 cond, conds := extraJoinConditions(n.OnList) 918 arg := single.NewArgument() 919 arg.Typs = typs 920 arg.Result = result 921 arg.Cond = cond 922 arg.Conditions = constructJoinConditions(conds, proc) 923 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 924 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 925 return arg 926 } 927 928 func constructProduct(n *plan.Node, typs []types.Type, proc *process.Process) *product.Argument { 929 result := make([]colexec.ResultPos, len(n.ProjectList)) 930 for i, expr := range n.ProjectList { 931 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 932 } 933 arg := product.NewArgument() 934 arg.Typs = typs 935 arg.Result = result 936 return arg 937 } 938 939 func constructAnti(n *plan.Node, typs []types.Type, proc *process.Process) *anti.Argument { 940 result := make([]int32, len(n.ProjectList)) 941 for i, expr := range n.ProjectList { 942 rel, pos := constructJoinResult(expr, proc) 943 if rel != 0 { 944 panic(moerr.NewNYI(proc.Ctx, "anti result '%s'", expr)) 945 } 946 result[i] = pos 947 } 948 cond, conds := extraJoinConditions(n.OnList) 949 arg := anti.NewArgument() 950 arg.Typs = typs 951 arg.Result = result 952 arg.Cond = cond 953 arg.Conditions = constructJoinConditions(conds, proc) 954 arg.HashOnPK = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.HashOnPK 955 arg.IsShuffle = n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle 956 return arg 957 } 958 959 /* 960 func constructMark(n *plan.Node, typs []types.Type, proc *process.Process) *mark.Argument { 961 result := make([]int32, len(n.ProjectList)) 962 for i, expr := range n.ProjectList { 963 rel, pos := constructJoinResult(expr, proc) 964 if rel == 0 { 965 result[i] = pos 966 } else if rel == -1 { 967 result[i] = -1 968 } else { 969 panic(moerr.NewNYI(proc.Ctx, "loop mark result '%s'", expr)) 970 } 971 } 972 cond, conds := extraJoinConditions(n.OnList) 973 return &mark.Argument{ 974 Typs: typs, 975 Result: result, 976 Cond: cond, 977 Conditions: constructJoinConditions(conds, proc), 978 OnList: n.OnList, 979 } 980 } 981 */ 982 983 func constructOrder(n *plan.Node) *order.Argument { 984 arg := order.NewArgument() 985 arg.OrderBySpec = n.OrderBy 986 return arg 987 } 988 989 func constructFill(n *plan.Node) *fill.Argument { 990 aggIdx := make([]int32, len(n.AggList)) 991 for i, expr := range n.AggList { 992 f := expr.Expr.(*plan.Expr_F) 993 obj := int64(uint64(f.F.Func.Obj) & function.DistinctMask) 994 aggIdx[i], _ = function.DecodeOverloadID(obj) 995 } 996 arg := fill.NewArgument() 997 arg.ColLen = len(n.AggList) 998 arg.FillType = n.FillType 999 arg.FillVal = n.FillVal 1000 arg.AggIds = aggIdx 1001 return arg 1002 } 1003 1004 func constructTimeWindow(_ context.Context, n *plan.Node) *timewin.Argument { 1005 var aggregationExpressions []aggexec.AggFuncExecExpression = nil 1006 var typs []types.Type 1007 var wStart, wEnd bool 1008 i := 0 1009 for _, expr := range n.AggList { 1010 if e, ok := expr.Expr.(*plan.Expr_Col); ok { 1011 if e.Col.Name == plan2.TimeWindowStart { 1012 wStart = true 1013 } 1014 if e.Col.Name == plan2.TimeWindowEnd { 1015 wEnd = true 1016 } 1017 continue 1018 } 1019 f := expr.Expr.(*plan.Expr_F) 1020 isDistinct := (uint64(f.F.Func.Obj) & function.Distinct) != 0 1021 functionID := int64(uint64(f.F.Func.Obj) & function.DistinctMask) 1022 e := f.F.Args[0] 1023 if e != nil { 1024 aggregationExpressions = append( 1025 aggregationExpressions, 1026 aggexec.MakeAggFunctionExpression(functionID, isDistinct, f.F.Args, nil)) 1027 1028 typs = append(typs, types.New(types.T(e.Typ.Id), e.Typ.Width, e.Typ.Scale)) 1029 } 1030 i++ 1031 } 1032 1033 var err error 1034 str := n.Interval.Expr.(*plan.Expr_List).List.List[1].Expr.(*plan.Expr_Lit).Lit.Value.(*plan.Literal_Sval).Sval 1035 itr := &timewin.Interval{} 1036 itr.Typ, err = types.IntervalTypeOf(str) 1037 if err != nil { 1038 panic(err) 1039 } 1040 itr.Val = n.Interval.Expr.(*plan.Expr_List).List.List[0].Expr.(*plan.Expr_Lit).Lit.Value.(*plan.Literal_I64Val).I64Val 1041 1042 var sld *timewin.Interval 1043 if n.Sliding != nil { 1044 sld = &timewin.Interval{} 1045 str = n.Sliding.Expr.(*plan.Expr_List).List.List[1].Expr.(*plan.Expr_Lit).Lit.Value.(*plan.Literal_Sval).Sval 1046 sld.Typ, err = types.IntervalTypeOf(str) 1047 if err != nil { 1048 panic(err) 1049 } 1050 sld.Val = n.Sliding.Expr.(*plan.Expr_List).List.List[0].Expr.(*plan.Expr_Lit).Lit.Value.(*plan.Literal_I64Val).I64Val 1051 } 1052 1053 arg := timewin.NewArgument() 1054 arg.Types = typs 1055 arg.Aggs = aggregationExpressions 1056 arg.Ts = n.OrderBy[0].Expr 1057 arg.WStart = wStart 1058 arg.WEnd = wEnd 1059 arg.Interval = itr 1060 arg.Sliding = sld 1061 return arg 1062 } 1063 1064 func constructWindow(_ context.Context, n *plan.Node, proc *process.Process) *window.Argument { 1065 aggregationExpressions := make([]aggexec.AggFuncExecExpression, len(n.WinSpecList)) 1066 typs := make([]types.Type, len(n.WinSpecList)) 1067 1068 for i, expr := range n.WinSpecList { 1069 f := expr.Expr.(*plan.Expr_W).W.WindowFunc.Expr.(*plan.Expr_F) 1070 isDistinct := (uint64(f.F.Func.Obj) & function.Distinct) != 0 1071 functionID := int64(uint64(f.F.Func.Obj) & function.DistinctMask) 1072 1073 var e *plan.Expr = nil 1074 var cfg []byte = nil 1075 var args = f.F.Args 1076 if len(f.F.Args) > 0 { 1077 1078 //for group_concat, the last arg is separator string 1079 //for cluster_centers, the last arg is kmeans_args string 1080 if (f.F.Func.ObjName == plan2.NameGroupConcat || 1081 f.F.Func.ObjName == plan2.NameClusterCenters) && len(f.F.Args) > 1 { 1082 argExpr := f.F.Args[len(f.F.Args)-1] 1083 vec, err := colexec.EvalExpressionOnce(proc, argExpr, []*batch.Batch{constBat}) 1084 if err != nil { 1085 panic(err) 1086 } 1087 cfg = []byte(vec.GetStringAt(0)) 1088 vec.Free(proc.Mp()) 1089 1090 args = f.F.Args[:len(f.F.Args)-1] 1091 } 1092 1093 e = f.F.Args[0] 1094 } 1095 aggregationExpressions[i] = aggexec.MakeAggFunctionExpression( 1096 functionID, isDistinct, args, cfg) 1097 1098 if e != nil { 1099 typs[i] = types.New(types.T(e.Typ.Id), e.Typ.Width, e.Typ.Scale) 1100 } 1101 } 1102 arg := window.NewArgument() 1103 arg.Types = typs 1104 arg.Aggs = aggregationExpressions 1105 arg.WinSpecList = n.WinSpecList 1106 return arg 1107 } 1108 1109 /* 1110 func constructOffset(n *plan.Node, proc *process.Process) *offset.Argument { 1111 vec, err := colexec.EvalExpr(constBat, proc, n.Offset) 1112 if err != nil { 1113 panic(err) 1114 } 1115 return &offset.Argument{ 1116 Offset: uint64(vec.Col.([]int64)[0]), 1117 } 1118 } 1119 */ 1120 1121 func constructLimit(n *plan.Node, proc *process.Process) *limit.Argument { 1122 executor, err := colexec.NewExpressionExecutor(proc, n.Limit) 1123 if err != nil { 1124 panic(err) 1125 } 1126 defer executor.Free() 1127 vec, err := executor.Eval(proc, []*batch.Batch{constBat}) 1128 if err != nil { 1129 panic(err) 1130 } 1131 1132 arg := limit.NewArgument() 1133 arg.Limit = uint64(vector.MustFixedCol[int64](vec)[0]) 1134 return arg 1135 } 1136 1137 func constructSample(n *plan.Node, outputRowCount bool) *sample.Argument { 1138 if n.SampleFunc.Rows != plan2.NotSampleByRows { 1139 return sample.NewSampleByRows(int(n.SampleFunc.Rows), n.AggList, n.GroupBy, n.SampleFunc.UsingRow, outputRowCount) 1140 } 1141 if n.SampleFunc.Percent != plan2.NotSampleByPercents { 1142 return sample.NewSampleByPercent(n.SampleFunc.Percent, n.AggList, n.GroupBy) 1143 } 1144 panic("only support sample by rows / percent now.") 1145 } 1146 1147 func constructGroup(_ context.Context, n, cn *plan.Node, ibucket, nbucket int, needEval bool, shuffleDop int, proc *process.Process) *group.Argument { 1148 aggregationExpressions := make([]aggexec.AggFuncExecExpression, len(n.AggList)) 1149 for i, expr := range n.AggList { 1150 if f, ok := expr.Expr.(*plan.Expr_F); ok { 1151 isDistinct := (uint64(f.F.Func.Obj) & function.Distinct) != 0 1152 functionID := int64(uint64(f.F.Func.Obj) & function.DistinctMask) 1153 1154 var cfg []byte = nil 1155 var args = f.F.Args 1156 if len(f.F.Args) > 0 { 1157 //for group_concat, the last arg is separator string 1158 //for cluster_centers, the last arg is kmeans_args string 1159 if (f.F.Func.ObjName == plan2.NameGroupConcat || 1160 f.F.Func.ObjName == plan2.NameClusterCenters) && len(f.F.Args) > 1 { 1161 argExpr := f.F.Args[len(f.F.Args)-1] 1162 vec, err := colexec.EvalExpressionOnce(proc, argExpr, []*batch.Batch{constBat}) 1163 if err != nil { 1164 panic(err) 1165 } 1166 cfg = []byte(vec.GetStringAt(0)) 1167 vec.Free(proc.Mp()) 1168 1169 args = f.F.Args[:len(f.F.Args)-1] 1170 } 1171 } 1172 1173 aggregationExpressions[i] = aggexec.MakeAggFunctionExpression( 1174 functionID, isDistinct, args, cfg) 1175 } 1176 } 1177 1178 typs := make([]types.Type, len(cn.ProjectList)) 1179 for i, e := range cn.ProjectList { 1180 typs[i] = types.New(types.T(e.Typ.Id), e.Typ.Width, e.Typ.Scale) 1181 } 1182 1183 shuffle := false 1184 var preAllocSize uint64 = 0 1185 if n.Stats != nil && n.Stats.HashmapStats != nil && n.Stats.HashmapStats.Shuffle { 1186 shuffle = true 1187 if cn.NodeType == plan.Node_TABLE_SCAN && len(cn.FilterList) == 0 { 1188 // if group on scan without filter, stats for hashmap is accurate to do preAlloc 1189 // tune it up a little bit in case it is not so average after shuffle 1190 preAllocSize = uint64(n.Stats.HashmapStats.HashmapSize / float64(shuffleDop) * 1.05) 1191 } 1192 } 1193 1194 arg := group.NewArgument() 1195 arg.Aggs = aggregationExpressions 1196 arg.Types = typs 1197 arg.NeedEval = needEval 1198 arg.Exprs = n.GroupBy 1199 arg.Ibucket = uint64(ibucket) 1200 arg.Nbucket = uint64(nbucket) 1201 arg.IsShuffle = shuffle 1202 arg.PreAllocSize = preAllocSize 1203 return arg 1204 } 1205 1206 // ibucket: bucket number 1207 // nbucket: 1208 // construct operator argument 1209 func constructIntersectAll(ibucket, nbucket int) *intersectall.Argument { 1210 arg := intersectall.NewArgument() 1211 arg.IBucket = uint64(ibucket) 1212 arg.NBucket = uint64(nbucket) 1213 return arg 1214 } 1215 1216 func constructMinus(ibucket, nbucket int) *minus.Argument { 1217 arg := minus.NewArgument() 1218 arg.IBucket = uint64(ibucket) 1219 arg.NBucket = uint64(nbucket) 1220 return arg 1221 } 1222 1223 func constructIntersect(ibucket, nbucket int) *intersect.Argument { 1224 arg := intersect.NewArgument() 1225 arg.IBucket = uint64(ibucket) 1226 arg.NBucket = uint64(nbucket) 1227 return arg 1228 } 1229 1230 func constructDispatchLocal(all bool, isSink, RecSink bool, regs []*process.WaitRegister) *dispatch.Argument { 1231 arg := dispatch.NewArgument() 1232 arg.LocalRegs = regs 1233 arg.IsSink = isSink 1234 arg.RecSink = RecSink 1235 if all { 1236 arg.FuncId = dispatch.SendToAllLocalFunc 1237 } else { 1238 arg.FuncId = dispatch.SendToAnyLocalFunc 1239 } 1240 return arg 1241 } 1242 1243 // ss[currentIdx] means it's local scope the dispatch rule should be like below: 1244 // dispatch batch to all other cn and also put one into proc.MergeReciever[0] for 1245 // local deletion 1246 func constructDeleteDispatchAndLocal( 1247 currentIdx int, 1248 rs []*Scope, 1249 ss []*Scope, 1250 uuids []uuid.UUID, 1251 c *Compile) { 1252 arg := dispatch.NewArgument() 1253 arg.RemoteRegs = make([]colexec.ReceiveInfo, 0, len(ss)-1) 1254 // rs is used to get batch from dispatch operator (include 1255 // local batch) 1256 rs[currentIdx].NodeInfo = ss[currentIdx].NodeInfo 1257 rs[currentIdx].Magic = Remote 1258 rs[currentIdx].PreScopes = append(rs[currentIdx].PreScopes, ss[currentIdx]) 1259 rs[currentIdx].Proc = process.NewWithAnalyze(c.proc, c.ctx, len(ss), c.anal.analInfos) 1260 rs[currentIdx].RemoteReceivRegInfos = make([]RemoteReceivRegInfo, 0, len(ss)-1) 1261 1262 // use arg.RemoteRegs to know the uuid, 1263 // use this uuid to register Server.uuidCsChanMap (uuid,proc.DispatchNotifyCh), 1264 // So how to use this? 1265 // the answer is below: 1266 // when the remote Cn run the scope, if scope's RemoteReceivRegInfos 1267 // is not empty, it will start to give a PrepareDoneNotifyMessage to 1268 // tell the dispatcher it's prepared, and also to know,this messgae 1269 // will carry the uuid and a clientSession. In dispatch instruction, 1270 // first it will use the uuid to get the proc.DispatchNotifyCh from the Server. 1271 // (remember the DispatchNotifyCh is in a process,not a global one,because we 1272 // need to send the WrapCs (a struct,contains clientSession,uuid and So on) in the 1273 // sepcified process). 1274 // And then Dispatcher will use this clientSession to dispatch batches to remoteCN. 1275 // When remoteCn get the batches, it should know send it to where by itself. 1276 for i := 0; i < len(ss); i++ { 1277 if i != currentIdx { 1278 // just use this uuid in dispatch, we need to 1279 // use it in the prepare func (store the map [uuid -> proc.DispatchNotifyCh]) 1280 arg.RemoteRegs = append( 1281 arg.RemoteRegs, 1282 colexec.ReceiveInfo{ 1283 Uuid: uuids[i], 1284 NodeAddr: ss[i].NodeInfo.Addr, 1285 }) 1286 // let remote scope knows it need to recieve bacth from 1287 // remote CN, it will use this to send PrepareDoneNotifyMessage 1288 // and then to recieve batches from remote CNs 1289 rs[currentIdx].RemoteReceivRegInfos = append( 1290 rs[currentIdx].RemoteReceivRegInfos, 1291 RemoteReceivRegInfo{ 1292 Uuid: uuids[currentIdx], 1293 // I use i to tag, scope should send the batches (recieved from remote CNs) 1294 // to process.MergeRecievers[i] 1295 Idx: i, 1296 FromAddr: ss[i].NodeInfo.Addr, 1297 }) 1298 } 1299 } 1300 if len(arg.RemoteRegs) == 0 { 1301 arg.FuncId = dispatch.SendToAllLocalFunc 1302 } else { 1303 arg.FuncId = dispatch.SendToAllFunc 1304 } 1305 1306 arg.LocalRegs = append( 1307 arg.LocalRegs, 1308 rs[currentIdx].Proc.Reg.MergeReceivers[currentIdx]) 1309 1310 ss[currentIdx].appendInstruction(vm.Instruction{ 1311 Op: vm.Dispatch, 1312 Arg: arg, 1313 }) 1314 // add merge to recieve all batches 1315 rs[currentIdx].appendInstruction(vm.Instruction{ 1316 Op: vm.Merge, 1317 Arg: merge.NewArgument(), 1318 }) 1319 } 1320 1321 // This function do not setting funcId. 1322 // PLEASE SETTING FuncId AFTER YOU CALL IT. 1323 func constructDispatchLocalAndRemote(idx int, ss []*Scope, currentCNAddr string) (bool, *dispatch.Argument) { 1324 arg := dispatch.NewArgument() 1325 scopeLen := len(ss) 1326 arg.LocalRegs = make([]*process.WaitRegister, 0, scopeLen) 1327 arg.RemoteRegs = make([]colexec.ReceiveInfo, 0, scopeLen) 1328 arg.ShuffleRegIdxLocal = make([]int, 0, len(ss)) 1329 arg.ShuffleRegIdxRemote = make([]int, 0, len(ss)) 1330 hasRemote := false 1331 for i, s := range ss { 1332 if s.IsEnd { 1333 continue 1334 } 1335 if len(s.NodeInfo.Addr) == 0 || len(currentCNAddr) == 0 || 1336 isSameCN(s.NodeInfo.Addr, currentCNAddr) { 1337 // Local reg. 1338 // Put them into arg.LocalRegs 1339 arg.LocalRegs = append(arg.LocalRegs, s.Proc.Reg.MergeReceivers[idx]) 1340 arg.ShuffleRegIdxLocal = append(arg.ShuffleRegIdxLocal, i) 1341 } else { 1342 // Remote reg. 1343 // Generate uuid for them and put into arg.RemoteRegs & scope. receive info 1344 hasRemote = true 1345 newUuid, _ := uuid.NewV7() 1346 1347 arg.RemoteRegs = append(arg.RemoteRegs, colexec.ReceiveInfo{ 1348 Uuid: newUuid, 1349 NodeAddr: s.NodeInfo.Addr, 1350 }) 1351 arg.ShuffleRegIdxRemote = append(arg.ShuffleRegIdxRemote, i) 1352 s.RemoteReceivRegInfos = append(s.RemoteReceivRegInfos, RemoteReceivRegInfo{ 1353 Idx: idx, 1354 Uuid: newUuid, 1355 FromAddr: currentCNAddr, 1356 }) 1357 } 1358 } 1359 return hasRemote, arg 1360 } 1361 1362 func constructShuffleJoinArg(ss []*Scope, node *plan.Node, left bool) *shuffle.Argument { 1363 arg := shuffle.NewArgument() 1364 var expr *plan.Expr 1365 cond := node.OnList[node.Stats.HashmapStats.ShuffleColIdx] 1366 switch condImpl := cond.Expr.(type) { 1367 case *plan.Expr_F: 1368 if left { 1369 expr = condImpl.F.Args[0] 1370 } else { 1371 expr = condImpl.F.Args[1] 1372 } 1373 } 1374 1375 hashCol, typ := plan2.GetHashColumn(expr) 1376 arg.ShuffleColIdx = hashCol.ColPos 1377 arg.ShuffleType = int32(node.Stats.HashmapStats.ShuffleType) 1378 arg.ShuffleColMin = node.Stats.HashmapStats.ShuffleColMin 1379 arg.ShuffleColMax = node.Stats.HashmapStats.ShuffleColMax 1380 arg.AliveRegCnt = int32(len(ss)) 1381 switch types.T(typ) { 1382 case types.T_int64, types.T_int32, types.T_int16: 1383 arg.ShuffleRangeInt64 = plan2.ShuffleRangeReEvalSigned(node.Stats.HashmapStats.Ranges, int(arg.AliveRegCnt), node.Stats.HashmapStats.Nullcnt, int64(node.Stats.TableCnt)) 1384 case types.T_uint64, types.T_uint32, types.T_uint16, types.T_varchar, types.T_char, types.T_text, types.T_bit: 1385 arg.ShuffleRangeUint64 = plan2.ShuffleRangeReEvalUnsigned(node.Stats.HashmapStats.Ranges, int(arg.AliveRegCnt), node.Stats.HashmapStats.Nullcnt, int64(node.Stats.TableCnt)) 1386 } 1387 return arg 1388 } 1389 1390 func constructShuffleGroupArg(ss []*Scope, node *plan.Node) *shuffle.Argument { 1391 arg := shuffle.NewArgument() 1392 hashCol, typ := plan2.GetHashColumn(node.GroupBy[node.Stats.HashmapStats.ShuffleColIdx]) 1393 arg.ShuffleColIdx = hashCol.ColPos 1394 arg.ShuffleType = int32(node.Stats.HashmapStats.ShuffleType) 1395 arg.ShuffleColMin = node.Stats.HashmapStats.ShuffleColMin 1396 arg.ShuffleColMax = node.Stats.HashmapStats.ShuffleColMax 1397 arg.AliveRegCnt = int32(len(ss)) 1398 switch types.T(typ) { 1399 case types.T_int64, types.T_int32, types.T_int16: 1400 arg.ShuffleRangeInt64 = plan2.ShuffleRangeReEvalSigned(node.Stats.HashmapStats.Ranges, int(arg.AliveRegCnt), node.Stats.HashmapStats.Nullcnt, int64(node.Stats.TableCnt)) 1401 case types.T_uint64, types.T_uint32, types.T_uint16, types.T_varchar, types.T_char, types.T_text, types.T_bit: 1402 arg.ShuffleRangeUint64 = plan2.ShuffleRangeReEvalUnsigned(node.Stats.HashmapStats.Ranges, int(arg.AliveRegCnt), node.Stats.HashmapStats.Nullcnt, int64(node.Stats.TableCnt)) 1403 } 1404 return arg 1405 } 1406 1407 // cross-cn dispath will send same batch to all register 1408 func constructDispatch(idx int, ss []*Scope, currentCNAddr string, node *plan.Node, left bool) *dispatch.Argument { 1409 hasRemote, arg := constructDispatchLocalAndRemote(idx, ss, currentCNAddr) 1410 if node.Stats.HashmapStats.Shuffle { 1411 arg.FuncId = dispatch.ShuffleToAllFunc 1412 if node.Stats.HashmapStats.ShuffleTypeForMultiCN == plan.ShuffleTypeForMultiCN_Hybrid { 1413 if left { 1414 arg.ShuffleType = plan2.ShuffleToLocalMatchedReg 1415 } else { 1416 arg.ShuffleType = plan2.ShuffleToMultiMatchedReg 1417 } 1418 } else { 1419 arg.ShuffleType = plan2.ShuffleToRegIndex 1420 } 1421 return arg 1422 } 1423 if hasRemote { 1424 arg.FuncId = dispatch.SendToAllFunc 1425 } else { 1426 arg.FuncId = dispatch.SendToAllLocalFunc 1427 } 1428 return arg 1429 } 1430 1431 func constructMergeGroup(needEval bool) *mergegroup.Argument { 1432 arg := mergegroup.NewArgument() 1433 arg.NeedEval = needEval 1434 return arg 1435 } 1436 1437 func constructMergeTop(n *plan.Node, topN int64) *mergetop.Argument { 1438 arg := mergetop.NewArgument() 1439 arg.Fs = n.OrderBy 1440 arg.Limit = topN 1441 return arg 1442 } 1443 1444 func constructMergeOffset(n *plan.Node, proc *process.Process) *mergeoffset.Argument { 1445 executor, err := colexec.NewExpressionExecutor(proc, n.Offset) 1446 if err != nil { 1447 panic(err) 1448 } 1449 defer executor.Free() 1450 vec, err := executor.Eval(proc, []*batch.Batch{constBat}) 1451 if err != nil { 1452 panic(err) 1453 } 1454 1455 arg := mergeoffset.NewArgument() 1456 arg.Offset = uint64(vector.MustFixedCol[int64](vec)[0]) 1457 return arg 1458 } 1459 1460 func constructMergeLimit(n *plan.Node, proc *process.Process) *mergelimit.Argument { 1461 executor, err := colexec.NewExpressionExecutor(proc, n.Limit) 1462 if err != nil { 1463 panic(err) 1464 } 1465 defer executor.Free() 1466 vec, err := executor.Eval(proc, []*batch.Batch{constBat}) 1467 if err != nil { 1468 panic(err) 1469 } 1470 1471 arg := mergelimit.NewArgument() 1472 arg.Limit = uint64(vector.MustFixedCol[int64](vec)[0]) 1473 return arg 1474 } 1475 1476 func constructMergeOrder(n *plan.Node) *mergeorder.Argument { 1477 arg := mergeorder.NewArgument() 1478 arg.OrderBySpecs = n.OrderBy 1479 return arg 1480 } 1481 1482 func constructPartition(n *plan.Node) *partition.Argument { 1483 arg := partition.NewArgument() 1484 arg.OrderBySpecs = n.OrderBy 1485 return arg 1486 } 1487 1488 func constructIndexJoin(n *plan.Node, typs []types.Type, proc *process.Process) *indexjoin.Argument { 1489 result := make([]int32, len(n.ProjectList)) 1490 for i, expr := range n.ProjectList { 1491 rel, pos := constructJoinResult(expr, proc) 1492 if rel != 0 { 1493 panic(moerr.NewNYI(proc.Ctx, "loop semi result '%s'", expr)) 1494 } 1495 result[i] = pos 1496 } 1497 arg := indexjoin.NewArgument() 1498 arg.Typs = typs 1499 arg.Result = result 1500 arg.RuntimeFilterSpecs = n.RuntimeFilterBuildList 1501 return arg 1502 } 1503 1504 func constructLoopJoin(n *plan.Node, typs []types.Type, proc *process.Process) *loopjoin.Argument { 1505 result := make([]colexec.ResultPos, len(n.ProjectList)) 1506 for i, expr := range n.ProjectList { 1507 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 1508 } 1509 arg := loopjoin.NewArgument() 1510 arg.Typs = typs 1511 arg.Result = result 1512 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1513 return arg 1514 } 1515 1516 func constructLoopSemi(n *plan.Node, typs []types.Type, proc *process.Process) *loopsemi.Argument { 1517 result := make([]int32, len(n.ProjectList)) 1518 for i, expr := range n.ProjectList { 1519 rel, pos := constructJoinResult(expr, proc) 1520 if rel != 0 { 1521 panic(moerr.NewNYI(proc.Ctx, "loop semi result '%s'", expr)) 1522 } 1523 result[i] = pos 1524 } 1525 arg := loopsemi.NewArgument() 1526 arg.Typs = typs 1527 arg.Result = result 1528 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1529 return arg 1530 } 1531 1532 func constructLoopLeft(n *plan.Node, typs []types.Type, proc *process.Process) *loopleft.Argument { 1533 result := make([]colexec.ResultPos, len(n.ProjectList)) 1534 for i, expr := range n.ProjectList { 1535 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 1536 } 1537 arg := loopleft.NewArgument() 1538 arg.Typs = typs 1539 arg.Result = result 1540 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1541 return arg 1542 } 1543 1544 func constructLoopSingle(n *plan.Node, typs []types.Type, proc *process.Process) *loopsingle.Argument { 1545 result := make([]colexec.ResultPos, len(n.ProjectList)) 1546 for i, expr := range n.ProjectList { 1547 result[i].Rel, result[i].Pos = constructJoinResult(expr, proc) 1548 } 1549 arg := loopsingle.NewArgument() 1550 arg.Typs = typs 1551 arg.Result = result 1552 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1553 return arg 1554 } 1555 1556 func constructLoopAnti(n *plan.Node, typs []types.Type, proc *process.Process) *loopanti.Argument { 1557 result := make([]int32, len(n.ProjectList)) 1558 for i, expr := range n.ProjectList { 1559 rel, pos := constructJoinResult(expr, proc) 1560 if rel != 0 { 1561 panic(moerr.NewNYI(proc.Ctx, "loop anti result '%s'", expr)) 1562 } 1563 result[i] = pos 1564 } 1565 arg := loopanti.NewArgument() 1566 arg.Typs = typs 1567 arg.Result = result 1568 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1569 return arg 1570 } 1571 1572 func constructLoopMark(n *plan.Node, typs []types.Type, proc *process.Process) *loopmark.Argument { 1573 result := make([]int32, len(n.ProjectList)) 1574 for i, expr := range n.ProjectList { 1575 rel, pos := constructJoinResult(expr, proc) 1576 if rel == 0 { 1577 result[i] = pos 1578 } else if rel == -1 { 1579 result[i] = -1 1580 } else { 1581 panic(moerr.NewNYI(proc.Ctx, "loop mark result '%s'", expr)) 1582 } 1583 } 1584 arg := loopmark.NewArgument() 1585 arg.Typs = typs 1586 arg.Result = result 1587 arg.Cond = colexec.RewriteFilterExprList(n.OnList) 1588 return arg 1589 } 1590 1591 func constructJoinBuildInstruction(c *Compile, in vm.Instruction, shuffleCnt int, isDup bool) vm.Instruction { 1592 switch in.Op { 1593 case vm.IndexJoin: 1594 arg := in.Arg.(*indexjoin.Argument) 1595 ret := indexbuild.NewArgument() 1596 if len(arg.RuntimeFilterSpecs) > 0 { 1597 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1598 } 1599 return vm.Instruction{ 1600 Op: vm.IndexBuild, 1601 Idx: in.Idx, 1602 IsFirst: true, 1603 Arg: ret, 1604 } 1605 default: 1606 return vm.Instruction{ 1607 Op: vm.HashBuild, 1608 Idx: in.Idx, 1609 IsFirst: true, 1610 Arg: constructHashBuild(c, in, c.proc, shuffleCnt, isDup), 1611 } 1612 } 1613 } 1614 1615 func constructHashBuild(c *Compile, in vm.Instruction, proc *process.Process, shuffleCnt int, isDup bool) *hashbuild.Argument { 1616 // XXX BUG 1617 // relation index of arg.Conditions should be rewritten to 0 here. 1618 ret := hashbuild.NewArgument() 1619 1620 switch in.Op { 1621 case vm.Anti: 1622 arg := in.Arg.(*anti.Argument) 1623 ret.NeedHashMap = true 1624 ret.Typs = arg.Typs 1625 ret.Conditions = arg.Conditions[1] 1626 ret.IsDup = isDup 1627 ret.HashOnPK = arg.HashOnPK 1628 if arg.Cond == nil { 1629 ret.NeedMergedBatch = false 1630 ret.NeedAllocateSels = false 1631 } else { 1632 ret.NeedMergedBatch = true 1633 ret.NeedAllocateSels = true 1634 } 1635 1636 case vm.Mark: 1637 arg := in.Arg.(*mark.Argument) 1638 ret.NeedHashMap = true 1639 ret.Typs = arg.Typs 1640 ret.Conditions = arg.Conditions[1] 1641 ret.IsDup = isDup 1642 ret.NeedMergedBatch = true 1643 ret.HashOnPK = arg.HashOnPK 1644 ret.NeedAllocateSels = true 1645 1646 case vm.Join: 1647 arg := in.Arg.(*join.Argument) 1648 ret.NeedHashMap = true 1649 ret.Typs = arg.Typs 1650 ret.Conditions = arg.Conditions[1] 1651 ret.IsDup = isDup 1652 ret.HashOnPK = arg.HashOnPK 1653 1654 // to find if hashmap need to keep build batches for probe 1655 var needMergedBatch bool 1656 if arg.Cond != nil { 1657 needMergedBatch = true 1658 } 1659 for _, rp := range arg.Result { 1660 if rp.Rel == 1 { 1661 needMergedBatch = true 1662 break 1663 } 1664 } 1665 ret.NeedMergedBatch = needMergedBatch 1666 ret.NeedAllocateSels = true 1667 if len(arg.RuntimeFilterSpecs) > 0 { 1668 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1669 } 1670 1671 case vm.Left: 1672 arg := in.Arg.(*left.Argument) 1673 ret.NeedHashMap = true 1674 ret.Typs = arg.Typs 1675 ret.Conditions = arg.Conditions[1] 1676 ret.IsDup = isDup 1677 ret.NeedMergedBatch = true 1678 ret.HashOnPK = arg.HashOnPK 1679 ret.NeedAllocateSels = true 1680 if len(arg.RuntimeFilterSpecs) > 0 { 1681 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1682 } 1683 1684 case vm.Right: 1685 arg := in.Arg.(*right.Argument) 1686 ret.NeedHashMap = true 1687 ret.Typs = arg.RightTypes 1688 ret.Conditions = arg.Conditions[1] 1689 ret.IsDup = isDup 1690 ret.NeedMergedBatch = true 1691 ret.HashOnPK = arg.HashOnPK 1692 ret.NeedAllocateSels = true 1693 if len(arg.RuntimeFilterSpecs) > 0 { 1694 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1695 } 1696 1697 case vm.RightSemi: 1698 arg := in.Arg.(*rightsemi.Argument) 1699 ret.NeedHashMap = true 1700 ret.Typs = arg.RightTypes 1701 ret.Conditions = arg.Conditions[1] 1702 ret.IsDup = isDup 1703 ret.NeedMergedBatch = true 1704 ret.HashOnPK = arg.HashOnPK 1705 ret.NeedAllocateSels = true 1706 if len(arg.RuntimeFilterSpecs) > 0 { 1707 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1708 } 1709 1710 case vm.RightAnti: 1711 arg := in.Arg.(*rightanti.Argument) 1712 ret.NeedHashMap = true 1713 ret.Typs = arg.RightTypes 1714 ret.Conditions = arg.Conditions[1] 1715 ret.IsDup = isDup 1716 ret.NeedMergedBatch = true 1717 ret.HashOnPK = arg.HashOnPK 1718 ret.NeedAllocateSels = true 1719 if len(arg.RuntimeFilterSpecs) > 0 { 1720 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1721 } 1722 1723 case vm.Semi: 1724 arg := in.Arg.(*semi.Argument) 1725 ret.NeedHashMap = true 1726 ret.Typs = arg.Typs 1727 ret.Conditions = arg.Conditions[1] 1728 ret.IsDup = isDup 1729 ret.HashOnPK = arg.HashOnPK 1730 if arg.Cond == nil { 1731 ret.NeedMergedBatch = false 1732 ret.NeedAllocateSels = false 1733 } else { 1734 ret.NeedMergedBatch = true 1735 ret.NeedAllocateSels = true 1736 } 1737 if len(arg.RuntimeFilterSpecs) > 0 { 1738 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1739 } 1740 1741 case vm.Single: 1742 arg := in.Arg.(*single.Argument) 1743 ret.NeedHashMap = true 1744 ret.Typs = arg.Typs 1745 ret.Conditions = arg.Conditions[1] 1746 ret.IsDup = isDup 1747 ret.NeedMergedBatch = true 1748 ret.HashOnPK = arg.HashOnPK 1749 ret.NeedAllocateSels = true 1750 if len(arg.RuntimeFilterSpecs) > 0 { 1751 ret.RuntimeFilterSpec = arg.RuntimeFilterSpecs[0] 1752 } 1753 1754 case vm.Product: 1755 arg := in.Arg.(*product.Argument) 1756 ret.NeedHashMap = false 1757 ret.Typs = arg.Typs 1758 ret.IsDup = isDup 1759 ret.NeedMergedBatch = true 1760 ret.NeedAllocateSels = true 1761 1762 case vm.LoopAnti: 1763 arg := in.Arg.(*loopanti.Argument) 1764 ret.NeedHashMap = false 1765 ret.Typs = arg.Typs 1766 ret.IsDup = isDup 1767 ret.NeedMergedBatch = true 1768 ret.NeedAllocateSels = true 1769 1770 case vm.LoopJoin: 1771 arg := in.Arg.(*loopjoin.Argument) 1772 ret.NeedHashMap = false 1773 ret.Typs = arg.Typs 1774 ret.IsDup = isDup 1775 ret.NeedMergedBatch = true 1776 ret.NeedAllocateSels = true 1777 1778 case vm.LoopLeft: 1779 arg := in.Arg.(*loopleft.Argument) 1780 ret.NeedHashMap = false 1781 ret.Typs = arg.Typs 1782 ret.IsDup = isDup 1783 ret.NeedMergedBatch = true 1784 ret.NeedAllocateSels = true 1785 1786 case vm.LoopSemi: 1787 arg := in.Arg.(*loopsemi.Argument) 1788 ret.NeedHashMap = false 1789 ret.Typs = arg.Typs 1790 ret.IsDup = isDup 1791 ret.NeedMergedBatch = true 1792 ret.NeedAllocateSels = true 1793 1794 case vm.LoopSingle: 1795 arg := in.Arg.(*loopsingle.Argument) 1796 ret.NeedHashMap = false 1797 ret.Typs = arg.Typs 1798 ret.IsDup = isDup 1799 ret.NeedMergedBatch = true 1800 ret.NeedAllocateSels = true 1801 1802 case vm.LoopMark: 1803 arg := in.Arg.(*loopmark.Argument) 1804 ret.NeedHashMap = false 1805 ret.Typs = arg.Typs 1806 ret.IsDup = isDup 1807 ret.NeedMergedBatch = true 1808 ret.NeedAllocateSels = true 1809 1810 default: 1811 ret.Release() 1812 panic(moerr.NewInternalError(proc.Ctx, "unsupport join type '%v'", in.Op)) 1813 } 1814 return ret 1815 } 1816 1817 func constructJoinResult(expr *plan.Expr, proc *process.Process) (int32, int32) { 1818 e, ok := expr.Expr.(*plan.Expr_Col) 1819 if !ok { 1820 panic(moerr.NewNYI(proc.Ctx, "join result '%s'", expr)) 1821 } 1822 return e.Col.RelPos, e.Col.ColPos 1823 } 1824 1825 func constructJoinConditions(exprs []*plan.Expr, proc *process.Process) [][]*plan.Expr { 1826 conds := make([][]*plan.Expr, 2) 1827 conds[0] = make([]*plan.Expr, len(exprs)) 1828 conds[1] = make([]*plan.Expr, len(exprs)) 1829 for i, expr := range exprs { 1830 conds[0][i], conds[1][i] = constructJoinCondition(expr, proc) 1831 } 1832 return conds 1833 } 1834 1835 func constructJoinCondition(expr *plan.Expr, proc *process.Process) (*plan.Expr, *plan.Expr) { 1836 if e, ok := expr.Expr.(*plan.Expr_Lit); ok { // constant bool 1837 b, ok := e.Lit.Value.(*plan.Literal_Bval) 1838 if !ok { 1839 panic(moerr.NewNYI(proc.Ctx, "join condition '%s'", expr)) 1840 } 1841 if b.Bval { 1842 return expr, expr 1843 } 1844 return expr, &plan.Expr{ 1845 Typ: expr.Typ, 1846 Expr: &plan.Expr_Lit{ 1847 Lit: &plan.Literal{ 1848 Value: &plan.Literal_Bval{Bval: true}, 1849 }, 1850 }, 1851 } 1852 } 1853 e, ok := expr.Expr.(*plan.Expr_F) 1854 if !ok || !plan2.IsEqualFunc(e.F.Func.GetObj()) { 1855 panic(moerr.NewNYI(proc.Ctx, "join condition '%s'", expr)) 1856 } 1857 if exprRelPos(e.F.Args[0]) == 1 { 1858 return e.F.Args[1], e.F.Args[0] 1859 } 1860 return e.F.Args[0], e.F.Args[1] 1861 } 1862 1863 func extraJoinConditions(exprs []*plan.Expr) (*plan.Expr, []*plan.Expr) { 1864 exprs = colexec.SplitAndExprs(exprs) 1865 eqConds := make([]*plan.Expr, 0, len(exprs)) 1866 notEqConds := make([]*plan.Expr, 0, len(exprs)) 1867 for i, expr := range exprs { 1868 if e, ok := expr.Expr.(*plan.Expr_F); ok { 1869 if !plan2.IsEqualFunc(e.F.Func.GetObj()) { 1870 notEqConds = append(notEqConds, exprs[i]) 1871 continue 1872 } 1873 lpos, rpos := plan2.HasColExpr(e.F.Args[0], -1), plan2.HasColExpr(e.F.Args[1], -1) 1874 if lpos == -1 || rpos == -1 || (lpos == rpos) { 1875 notEqConds = append(notEqConds, exprs[i]) 1876 continue 1877 } 1878 eqConds = append(eqConds, exprs[i]) 1879 } else { 1880 notEqConds = append(notEqConds, exprs[i]) 1881 } 1882 } 1883 if len(notEqConds) == 0 { 1884 return nil, eqConds 1885 } 1886 return colexec.RewriteFilterExprList(notEqConds), eqConds 1887 } 1888 1889 func exprRelPos(expr *plan.Expr) int32 { 1890 switch e := expr.Expr.(type) { 1891 case *plan.Expr_Col: 1892 return e.Col.RelPos 1893 case *plan.Expr_F: 1894 for i := range e.F.Args { 1895 if relPos := exprRelPos(e.F.Args[i]); relPos >= 0 { 1896 return relPos 1897 } 1898 } 1899 } 1900 return -1 1901 } 1902 1903 // Get the 'engine.Relation' of the table by using 'ObjectRef' and 'TableDef', if 'TableDef' is nil, the relations of its index table will not be obtained 1904 // the first return value is Relation of the original table 1905 // the second return value is Relations of index tables 1906 func getRel(ctx context.Context, proc *process.Process, eg engine.Engine, ref *plan.ObjectRef, tableDef *plan.TableDef) (engine.Relation, []engine.Relation, error) { 1907 var dbSource engine.Database 1908 var relation engine.Relation 1909 var err error 1910 var isTemp bool 1911 oldDbName := ref.SchemaName 1912 if ref.SchemaName != "" { 1913 dbSource, err = eg.Database(ctx, ref.SchemaName, proc.TxnOperator) 1914 if err != nil { 1915 return nil, nil, err 1916 } 1917 relation, err = dbSource.Relation(ctx, ref.ObjName, proc) 1918 if err == nil { 1919 isTemp = defines.TEMPORARY_DBNAME == ref.SchemaName 1920 } else { 1921 dbSource, err = eg.Database(ctx, defines.TEMPORARY_DBNAME, proc.TxnOperator) 1922 if err != nil { 1923 return nil, nil, moerr.NewNoSuchTable(ctx, ref.SchemaName, ref.ObjName) 1924 } 1925 newObjeName := engine.GetTempTableName(ref.SchemaName, ref.ObjName) 1926 newSchemaName := defines.TEMPORARY_DBNAME 1927 ref.SchemaName = newSchemaName 1928 ref.ObjName = newObjeName 1929 relation, err = dbSource.Relation(ctx, newObjeName, proc) 1930 if err != nil { 1931 return nil, nil, err 1932 } 1933 isTemp = true 1934 } 1935 } else { 1936 _, _, relation, err = eg.GetRelationById(ctx, proc.TxnOperator, uint64(ref.Obj)) 1937 if err != nil { 1938 return nil, nil, err 1939 } 1940 } 1941 1942 var uniqueIndexTables []engine.Relation 1943 if tableDef != nil { 1944 uniqueIndexTables = make([]engine.Relation, 0) 1945 if tableDef.Indexes != nil { 1946 for _, indexdef := range tableDef.Indexes { 1947 var indexTable engine.Relation 1948 if indexdef.TableExist { 1949 if isTemp { 1950 indexTable, err = dbSource.Relation(ctx, engine.GetTempTableName(oldDbName, indexdef.IndexTableName), proc) 1951 } else { 1952 indexTable, err = dbSource.Relation(ctx, indexdef.IndexTableName, proc) 1953 } 1954 if err != nil { 1955 return nil, nil, err 1956 } 1957 // NOTE: uniqueIndexTables is not yet used by the callee 1958 uniqueIndexTables = append(uniqueIndexTables, indexTable) 1959 } 1960 } 1961 } 1962 } 1963 return relation, uniqueIndexTables, err 1964 }