github.com/matrixorigin/matrixone@v1.2.0/pkg/sql/plan/query_builder.go (about) 1 // Copyright 2022 Matrix Origin 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package plan 16 17 import ( 18 "context" 19 "encoding/json" 20 "fmt" 21 "go/constant" 22 "strconv" 23 "strings" 24 "time" 25 26 "github.com/google/uuid" 27 28 "github.com/matrixorigin/matrixone/pkg/catalog" 29 "github.com/matrixorigin/matrixone/pkg/common/moerr" 30 "github.com/matrixorigin/matrixone/pkg/container/batch" 31 "github.com/matrixorigin/matrixone/pkg/container/types" 32 "github.com/matrixorigin/matrixone/pkg/container/vector" 33 "github.com/matrixorigin/matrixone/pkg/pb/plan" 34 "github.com/matrixorigin/matrixone/pkg/pb/timestamp" 35 "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect" 36 "github.com/matrixorigin/matrixone/pkg/sql/parsers/dialect/mysql" 37 "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" 38 "github.com/matrixorigin/matrixone/pkg/sql/plan/function" 39 "github.com/matrixorigin/matrixone/pkg/sql/util" 40 "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/options" 41 ) 42 43 func NewQueryBuilder(queryType plan.Query_StatementType, ctx CompilerContext, isPrepareStatement bool) *QueryBuilder { 44 var mysqlCompatible bool 45 46 mode, err := ctx.ResolveVariable("sql_mode", true, false) 47 if err == nil { 48 if modeStr, ok := mode.(string); ok { 49 if !strings.Contains(modeStr, "ONLY_FULL_GROUP_BY") { 50 mysqlCompatible = true 51 } 52 } 53 } 54 55 return &QueryBuilder{ 56 qry: &Query{ 57 StmtType: queryType, 58 }, 59 compCtx: ctx, 60 ctxByNode: []*BindContext{}, 61 nameByColRef: make(map[[2]int32]string), 62 nextTag: 0, 63 mysqlCompatible: mysqlCompatible, 64 tag2Table: make(map[int32]*TableDef), 65 isPrepareStatement: isPrepareStatement, 66 deleteNode: make(map[uint64]int32), 67 } 68 } 69 70 func (builder *QueryBuilder) remapColRefForExpr(expr *Expr, colMap map[[2]int32][2]int32) error { 71 switch ne := expr.Expr.(type) { 72 case *plan.Expr_Col: 73 mapID := [2]int32{ne.Col.RelPos, ne.Col.ColPos} 74 if ids, ok := colMap[mapID]; ok { 75 ne.Col.RelPos = ids[0] 76 ne.Col.ColPos = ids[1] 77 ne.Col.Name = builder.nameByColRef[mapID] 78 } else { 79 var keys []string 80 for k := range colMap { 81 keys = append(keys, fmt.Sprintf("%v", k)) 82 } 83 mapKeys := fmt.Sprintf("{ %s }", strings.Join(keys, ", ")) 84 return moerr.NewParseError(builder.GetContext(), "can't find column %v in context's map %s", mapID, mapKeys) 85 } 86 87 case *plan.Expr_F: 88 for _, arg := range ne.F.GetArgs() { 89 err := builder.remapColRefForExpr(arg, colMap) 90 if err != nil { 91 return err 92 } 93 } 94 case *plan.Expr_W: 95 err := builder.remapColRefForExpr(ne.W.WindowFunc, colMap) 96 if err != nil { 97 return err 98 } 99 //for _, arg := range ne.W.PartitionBy { 100 // err = builder.remapColRefForExpr(arg, colMap) 101 // if err != nil { 102 // return err 103 // } 104 //} 105 for _, order := range ne.W.OrderBy { 106 err = builder.remapColRefForExpr(order.Expr, colMap) 107 if err != nil { 108 return err 109 } 110 } 111 } 112 return nil 113 } 114 115 type ColRefRemapping struct { 116 globalToLocal map[[2]int32][2]int32 117 localToGlobal [][2]int32 118 } 119 120 func (m *ColRefRemapping) addColRef(colRef [2]int32) { 121 m.globalToLocal[colRef] = [2]int32{0, int32(len(m.localToGlobal))} 122 m.localToGlobal = append(m.localToGlobal, colRef) 123 } 124 125 func (builder *QueryBuilder) copyNode(ctx *BindContext, nodeId int32) int32 { 126 node := builder.qry.Nodes[nodeId] 127 newNode := DeepCopyNode(node) 128 newNode.Children = make([]int32, 0, len(node.Children)) 129 for _, child := range node.Children { 130 newNode.Children = append(newNode.Children, builder.copyNode(ctx, child)) 131 } 132 newNodeId := builder.appendNode(newNode, ctx) 133 return newNodeId 134 } 135 136 func (builder *QueryBuilder) remapAllColRefs(nodeID int32, step int32, colRefCnt map[[2]int32]int, colRefBool map[[2]int32]bool, sinkColRef map[[2]int32]int) (*ColRefRemapping, error) { 137 node := builder.qry.Nodes[nodeID] 138 139 remapping := &ColRefRemapping{ 140 globalToLocal: make(map[[2]int32][2]int32), 141 } 142 143 switch node.NodeType { 144 case plan.Node_FUNCTION_SCAN: 145 for _, expr := range node.FilterList { 146 increaseRefCnt(expr, 1, colRefCnt) 147 } 148 149 internalRemapping := &ColRefRemapping{ 150 globalToLocal: make(map[[2]int32][2]int32), 151 } 152 153 tag := node.BindingTags[0] 154 newTableDef := DeepCopyTableDef(node.TableDef, false) 155 156 for i, col := range node.TableDef.Cols { 157 globalRef := [2]int32{tag, int32(i)} 158 if colRefCnt[globalRef] == 0 { 159 continue 160 } 161 162 internalRemapping.addColRef(globalRef) 163 164 newTableDef.Cols = append(newTableDef.Cols, DeepCopyColDef(col)) 165 } 166 167 if len(newTableDef.Cols) == 0 { 168 internalRemapping.addColRef([2]int32{tag, 0}) 169 newTableDef.Cols = append(newTableDef.Cols, DeepCopyColDef(node.TableDef.Cols[0])) 170 } 171 172 node.TableDef = newTableDef 173 174 for _, expr := range node.FilterList { 175 increaseRefCnt(expr, -1, colRefCnt) 176 err := builder.remapColRefForExpr(expr, internalRemapping.globalToLocal) 177 if err != nil { 178 return nil, err 179 } 180 } 181 182 for i, col := range node.TableDef.Cols { 183 if colRefCnt[internalRemapping.localToGlobal[i]] == 0 { 184 continue 185 } 186 187 remapping.addColRef(internalRemapping.localToGlobal[i]) 188 189 node.ProjectList = append(node.ProjectList, &plan.Expr{ 190 Typ: col.Typ, 191 Expr: &plan.Expr_Col{ 192 Col: &plan.ColRef{ 193 RelPos: 0, 194 ColPos: int32(i), 195 Name: col.Name, 196 }, 197 }, 198 }) 199 } 200 201 if len(node.ProjectList) == 0 { 202 if len(node.TableDef.Cols) == 0 { 203 globalRef := [2]int32{tag, 0} 204 remapping.addColRef(globalRef) 205 206 node.ProjectList = append(node.ProjectList, &plan.Expr{ 207 Typ: node.TableDef.Cols[0].Typ, 208 Expr: &plan.Expr_Col{ 209 Col: &plan.ColRef{ 210 RelPos: 0, 211 ColPos: 0, 212 Name: node.TableDef.Cols[0].Name, 213 }, 214 }, 215 }) 216 } else { 217 remapping.addColRef(internalRemapping.localToGlobal[0]) 218 node.ProjectList = append(node.ProjectList, &plan.Expr{ 219 Typ: node.TableDef.Cols[0].Typ, 220 Expr: &plan.Expr_Col{ 221 Col: &plan.ColRef{ 222 RelPos: 0, 223 ColPos: 0, 224 Name: node.TableDef.Cols[0].Name, 225 }, 226 }, 227 }) 228 } 229 } 230 childId := node.Children[0] 231 childNode := builder.qry.Nodes[childId] 232 233 if childNode.NodeType == plan.Node_VALUE_SCAN { 234 break 235 } 236 for _, expr := range node.TblFuncExprList { 237 increaseRefCnt(expr, 1, colRefCnt) 238 } 239 childMap, err := builder.remapAllColRefs(childId, step, colRefCnt, colRefBool, sinkColRef) 240 241 if err != nil { 242 return nil, err 243 } 244 245 for _, expr := range node.TblFuncExprList { 246 increaseRefCnt(expr, -1, colRefCnt) 247 err = builder.remapColRefForExpr(expr, childMap.globalToLocal) 248 if err != nil { 249 return nil, err 250 } 251 } 252 253 case plan.Node_TABLE_SCAN, plan.Node_MATERIAL_SCAN, plan.Node_EXTERNAL_SCAN, plan.Node_SOURCE_SCAN: 254 for _, expr := range node.FilterList { 255 increaseRefCnt(expr, 1, colRefCnt) 256 } 257 258 for _, expr := range node.BlockFilterList { 259 increaseRefCnt(expr, 1, colRefCnt) 260 } 261 262 for _, rfSpec := range node.RuntimeFilterProbeList { 263 if rfSpec.Expr != nil { 264 increaseRefCnt(rfSpec.Expr, 1, colRefCnt) 265 } 266 } 267 268 internalRemapping := &ColRefRemapping{ 269 globalToLocal: make(map[[2]int32][2]int32), 270 } 271 272 tag := node.BindingTags[0] 273 newTableDef := DeepCopyTableDef(node.TableDef, false) 274 275 for i, col := range node.TableDef.Cols { 276 globalRef := [2]int32{tag, int32(i)} 277 if colRefCnt[globalRef] == 0 { 278 continue 279 } 280 281 internalRemapping.addColRef(globalRef) 282 283 newTableDef.Cols = append(newTableDef.Cols, DeepCopyColDef(col)) 284 } 285 286 if len(newTableDef.Cols) == 0 { 287 internalRemapping.addColRef([2]int32{tag, 0}) 288 newTableDef.Cols = append(newTableDef.Cols, DeepCopyColDef(node.TableDef.Cols[0])) 289 } 290 291 node.TableDef = newTableDef 292 293 for _, expr := range node.FilterList { 294 increaseRefCnt(expr, -1, colRefCnt) 295 err := builder.remapColRefForExpr(expr, internalRemapping.globalToLocal) 296 if err != nil { 297 return nil, err 298 } 299 } 300 301 for _, expr := range node.BlockFilterList { 302 increaseRefCnt(expr, -1, colRefCnt) 303 err := builder.remapColRefForExpr(expr, internalRemapping.globalToLocal) 304 if err != nil { 305 return nil, err 306 } 307 } 308 309 for _, rfSpec := range node.RuntimeFilterProbeList { 310 if rfSpec.Expr != nil { 311 increaseRefCnt(rfSpec.Expr, -1, colRefCnt) 312 err := builder.remapColRefForExpr(rfSpec.Expr, internalRemapping.globalToLocal) 313 if err != nil { 314 return nil, err 315 } 316 } 317 } 318 319 for i, col := range node.TableDef.Cols { 320 if colRefCnt[internalRemapping.localToGlobal[i]] == 0 { 321 continue 322 } 323 324 remapping.addColRef(internalRemapping.localToGlobal[i]) 325 326 node.ProjectList = append(node.ProjectList, &plan.Expr{ 327 Typ: col.Typ, 328 Expr: &plan.Expr_Col{ 329 Col: &plan.ColRef{ 330 RelPos: 0, 331 ColPos: int32(i), 332 Name: builder.nameByColRef[internalRemapping.localToGlobal[i]], 333 }, 334 }, 335 }) 336 } 337 338 if len(node.ProjectList) == 0 { 339 if len(node.TableDef.Cols) == 0 { 340 globalRef := [2]int32{tag, 0} 341 remapping.addColRef(globalRef) 342 343 node.ProjectList = append(node.ProjectList, &plan.Expr{ 344 Typ: node.TableDef.Cols[0].Typ, 345 Expr: &plan.Expr_Col{ 346 Col: &plan.ColRef{ 347 RelPos: 0, 348 ColPos: 0, 349 Name: builder.nameByColRef[globalRef], 350 }, 351 }, 352 }) 353 } else { 354 remapping.addColRef(internalRemapping.localToGlobal[0]) 355 node.ProjectList = append(node.ProjectList, &plan.Expr{ 356 Typ: node.TableDef.Cols[0].Typ, 357 Expr: &plan.Expr_Col{ 358 Col: &plan.ColRef{ 359 RelPos: 0, 360 ColPos: 0, 361 Name: builder.nameByColRef[internalRemapping.localToGlobal[0]], 362 }, 363 }, 364 }) 365 } 366 } 367 368 case plan.Node_INTERSECT, plan.Node_INTERSECT_ALL, 369 plan.Node_UNION, plan.Node_UNION_ALL, 370 plan.Node_MINUS, plan.Node_MINUS_ALL: 371 372 thisTag := node.BindingTags[0] 373 leftID := node.Children[0] 374 rightID := node.Children[1] 375 for i, expr := range node.ProjectList { 376 increaseRefCnt(expr, 1, colRefCnt) 377 globalRef := [2]int32{thisTag, int32(i)} 378 remapping.addColRef(globalRef) 379 } 380 381 rightNode := builder.qry.Nodes[rightID] 382 if rightNode.NodeType == plan.Node_PROJECT { 383 projectTag := rightNode.BindingTags[0] 384 for i := range rightNode.ProjectList { 385 increaseRefCnt(&plan.Expr{ 386 Expr: &plan.Expr_Col{ 387 Col: &plan.ColRef{ 388 RelPos: projectTag, 389 ColPos: int32(i), 390 }, 391 }}, 1, colRefCnt) 392 } 393 } 394 395 internalMap := make(map[[2]int32][2]int32) 396 397 leftRemapping, err := builder.remapAllColRefs(leftID, step, colRefCnt, colRefBool, sinkColRef) 398 if err != nil { 399 return nil, err 400 } 401 for k, v := range leftRemapping.globalToLocal { 402 internalMap[k] = v 403 } 404 405 _, err = builder.remapAllColRefs(rightID, step, colRefCnt, colRefBool, sinkColRef) 406 if err != nil { 407 return nil, err 408 } 409 410 for _, expr := range node.ProjectList { 411 increaseRefCnt(expr, -1, colRefCnt) 412 err := builder.remapColRefForExpr(expr, internalMap) 413 if err != nil { 414 return nil, err 415 } 416 } 417 418 case plan.Node_JOIN: 419 for _, expr := range node.OnList { 420 increaseRefCnt(expr, 1, colRefCnt) 421 } 422 423 internalMap := make(map[[2]int32][2]int32) 424 425 leftID := node.Children[0] 426 leftRemapping, err := builder.remapAllColRefs(leftID, step, colRefCnt, colRefBool, sinkColRef) 427 if err != nil { 428 return nil, err 429 } 430 431 for k, v := range leftRemapping.globalToLocal { 432 internalMap[k] = v 433 } 434 435 rightID := node.Children[1] 436 rightRemapping, err := builder.remapAllColRefs(rightID, step, colRefCnt, colRefBool, sinkColRef) 437 if err != nil { 438 return nil, err 439 } 440 441 for k, v := range rightRemapping.globalToLocal { 442 internalMap[k] = [2]int32{1, v[1]} 443 } 444 445 for _, expr := range node.OnList { 446 increaseRefCnt(expr, -1, colRefCnt) 447 err := builder.remapColRefForExpr(expr, internalMap) 448 if err != nil { 449 return nil, err 450 } 451 } 452 453 childProjList := builder.qry.Nodes[leftID].ProjectList 454 for i, globalRef := range leftRemapping.localToGlobal { 455 if colRefCnt[globalRef] == 0 { 456 continue 457 } 458 459 remapping.addColRef(globalRef) 460 if node.JoinType == plan.Node_RIGHT { 461 childProjList[i].Typ.NotNullable = false 462 } 463 node.ProjectList = append(node.ProjectList, &plan.Expr{ 464 Typ: childProjList[i].Typ, 465 Expr: &plan.Expr_Col{ 466 Col: &plan.ColRef{ 467 RelPos: 0, 468 ColPos: int32(i), 469 Name: builder.nameByColRef[globalRef], 470 }, 471 }, 472 }) 473 } 474 475 if node.JoinType == plan.Node_MARK { 476 globalRef := [2]int32{node.BindingTags[0], 0} 477 remapping.addColRef(globalRef) 478 479 node.ProjectList = append(node.ProjectList, &plan.Expr{ 480 Typ: plan.Type{ 481 Id: int32(types.T_bool), 482 NotNullable: false, 483 }, 484 Expr: &plan.Expr_Col{ 485 Col: &plan.ColRef{ 486 RelPos: -1, 487 ColPos: 0, 488 Name: builder.nameByColRef[globalRef], 489 }, 490 }, 491 }) 492 } else { 493 childProjList = builder.qry.Nodes[rightID].ProjectList 494 for i, globalRef := range rightRemapping.localToGlobal { 495 if colRefCnt[globalRef] == 0 { 496 continue 497 } 498 499 remapping.addColRef(globalRef) 500 501 if node.JoinType == plan.Node_LEFT { 502 childProjList[i].Typ.NotNullable = false 503 } 504 505 node.ProjectList = append(node.ProjectList, &plan.Expr{ 506 Typ: childProjList[i].Typ, 507 Expr: &plan.Expr_Col{ 508 Col: &plan.ColRef{ 509 RelPos: 1, 510 ColPos: int32(i), 511 Name: builder.nameByColRef[globalRef], 512 }, 513 }, 514 }) 515 } 516 } 517 518 if len(node.ProjectList) == 0 && len(leftRemapping.localToGlobal) > 0 { 519 globalRef := leftRemapping.localToGlobal[0] 520 remapping.addColRef(globalRef) 521 522 node.ProjectList = append(node.ProjectList, &plan.Expr{ 523 Typ: builder.qry.Nodes[leftID].ProjectList[0].Typ, 524 Expr: &plan.Expr_Col{ 525 Col: &plan.ColRef{ 526 RelPos: 0, 527 ColPos: 0, 528 Name: builder.nameByColRef[globalRef], 529 }, 530 }, 531 }) 532 } 533 534 case plan.Node_AGG: 535 for _, expr := range node.GroupBy { 536 increaseRefCnt(expr, 1, colRefCnt) 537 } 538 539 for _, expr := range node.AggList { 540 increaseRefCnt(expr, 1, colRefCnt) 541 } 542 543 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 544 if err != nil { 545 return nil, err 546 } 547 548 groupTag := node.BindingTags[0] 549 aggregateTag := node.BindingTags[1] 550 groupSize := int32(len(node.GroupBy)) 551 552 for _, expr := range node.FilterList { 553 builder.remapHavingClause(expr, groupTag, aggregateTag, groupSize) 554 } 555 556 for idx, expr := range node.GroupBy { 557 increaseRefCnt(expr, -1, colRefCnt) 558 err := builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 559 if err != nil { 560 return nil, err 561 } 562 563 globalRef := [2]int32{groupTag, int32(idx)} 564 if colRefCnt[globalRef] == 0 { 565 continue 566 } 567 568 remapping.addColRef(globalRef) 569 570 node.ProjectList = append(node.ProjectList, &plan.Expr{ 571 Typ: expr.Typ, 572 Expr: &plan.Expr_Col{ 573 Col: &ColRef{ 574 RelPos: -1, 575 ColPos: int32(idx), 576 Name: builder.nameByColRef[globalRef], 577 }, 578 }, 579 }) 580 } 581 582 for idx, expr := range node.AggList { 583 increaseRefCnt(expr, -1, colRefCnt) 584 err := builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 585 if err != nil { 586 return nil, err 587 } 588 589 globalRef := [2]int32{aggregateTag, int32(idx)} 590 if colRefCnt[globalRef] == 0 { 591 continue 592 } 593 594 remapping.addColRef(globalRef) 595 596 node.ProjectList = append(node.ProjectList, &Expr{ 597 Typ: expr.Typ, 598 Expr: &plan.Expr_Col{ 599 Col: &ColRef{ 600 RelPos: -2, 601 ColPos: int32(idx) + groupSize, 602 Name: builder.nameByColRef[globalRef], 603 }, 604 }, 605 }) 606 } 607 608 if len(node.ProjectList) == 0 { 609 if groupSize > 0 { 610 globalRef := [2]int32{groupTag, 0} 611 remapping.addColRef(globalRef) 612 613 node.ProjectList = append(node.ProjectList, &plan.Expr{ 614 Typ: node.GroupBy[0].Typ, 615 Expr: &plan.Expr_Col{ 616 Col: &plan.ColRef{ 617 RelPos: -1, 618 ColPos: 0, 619 Name: builder.nameByColRef[globalRef], 620 }, 621 }, 622 }) 623 } else { 624 globalRef := [2]int32{aggregateTag, 0} 625 remapping.addColRef(globalRef) 626 627 node.ProjectList = append(node.ProjectList, &plan.Expr{ 628 Typ: node.AggList[0].Typ, 629 Expr: &plan.Expr_Col{ 630 Col: &plan.ColRef{ 631 RelPos: -2, 632 ColPos: 0, 633 Name: builder.nameByColRef[globalRef], 634 }, 635 }, 636 }) 637 } 638 } 639 640 child := builder.qry.Nodes[node.Children[0]] 641 if child.NodeType == plan.Node_TABLE_SCAN && len(child.FilterList) == 0 && len(node.GroupBy) == 0 && child.Limit == nil && child.Offset == nil { 642 child.AggList = make([]*Expr, 0, len(node.AggList)) 643 for _, agg := range node.AggList { 644 switch agg.GetF().Func.ObjName { 645 case "starcount", "count", "min", "max": 646 child.AggList = append(child.AggList, DeepCopyExpr(agg)) 647 default: 648 child.AggList = nil 649 } 650 if child.AggList == nil { 651 break 652 } 653 } 654 } 655 656 case plan.Node_SAMPLE: 657 groupTag := node.BindingTags[0] 658 sampleTag := node.BindingTags[1] 659 increaseRefCntForExprList(node.GroupBy, 1, colRefCnt) 660 increaseRefCntForExprList(node.AggList, 1, colRefCnt) 661 662 // the result order of sample will follow [group by columns, sample columns, other columns]. 663 // and the projection list needs to be based on the result order. 664 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 665 if err != nil { 666 return nil, err 667 } 668 669 for _, expr := range node.FilterList { 670 builder.remapHavingClause(expr, groupTag, sampleTag, int32(len(node.GroupBy))) 671 } 672 673 // deal with group col and sample col. 674 for i, expr := range node.GroupBy { 675 increaseRefCnt(expr, -1, colRefCnt) 676 err = builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 677 if err != nil { 678 return nil, err 679 } 680 681 globalRef := [2]int32{groupTag, int32(i)} 682 if colRefCnt[globalRef] == 0 { 683 continue 684 } 685 686 remapping.addColRef(globalRef) 687 688 node.ProjectList = append(node.ProjectList, &plan.Expr{ 689 Typ: expr.Typ, 690 Expr: &plan.Expr_Col{ 691 Col: &ColRef{ 692 RelPos: -1, 693 ColPos: int32(len(node.ProjectList)), 694 Name: builder.nameByColRef[globalRef], 695 }, 696 }, 697 }) 698 } 699 700 offsetSize := int32(len(node.GroupBy)) 701 for i, expr := range node.AggList { 702 increaseRefCnt(expr, -1, colRefCnt) 703 err = builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 704 if err != nil { 705 return nil, err 706 } 707 708 globalRef := [2]int32{sampleTag, int32(i)} 709 if colRefCnt[globalRef] == 0 { 710 continue 711 } 712 713 remapping.addColRef(globalRef) 714 715 node.ProjectList = append(node.ProjectList, &Expr{ 716 Typ: expr.Typ, 717 Expr: &plan.Expr_Col{ 718 Col: &ColRef{ 719 RelPos: -2, 720 ColPos: int32(i) + offsetSize, 721 Name: builder.nameByColRef[globalRef], 722 }, 723 }, 724 }) 725 } 726 727 offsetSize += int32(len(node.AggList)) 728 childProjectionList := builder.qry.Nodes[node.Children[0]].ProjectList 729 for i, globalRef := range childRemapping.localToGlobal { 730 if colRefCnt[globalRef] == 0 { 731 continue 732 } 733 remapping.addColRef(globalRef) 734 735 node.ProjectList = append(node.ProjectList, &plan.Expr{ 736 Typ: childProjectionList[i].Typ, 737 Expr: &plan.Expr_Col{ 738 Col: &plan.ColRef{ 739 RelPos: 0, 740 ColPos: int32(i) + offsetSize, 741 Name: builder.nameByColRef[globalRef], 742 }, 743 }, 744 }) 745 } 746 747 case plan.Node_TIME_WINDOW: 748 for _, expr := range node.AggList { 749 increaseRefCnt(expr, 1, colRefCnt) 750 } 751 increaseRefCnt(node.OrderBy[0].Expr, 1, colRefCnt) 752 753 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 754 if err != nil { 755 return nil, err 756 } 757 758 timeTag := node.BindingTags[0] 759 760 for i, expr := range node.FilterList { 761 builder.remapWindowClause(expr, timeTag, int32(i)) 762 } 763 764 increaseRefCnt(node.OrderBy[0].Expr, -1, colRefCnt) 765 err = builder.remapColRefForExpr(node.OrderBy[0].Expr, childRemapping.globalToLocal) 766 if err != nil { 767 return nil, err 768 } 769 770 idx := 0 771 var wstart, wend *plan.Expr 772 var i, j int 773 for k, expr := range node.AggList { 774 if e, ok := expr.Expr.(*plan.Expr_Col); ok { 775 if e.Col.Name == TimeWindowStart { 776 wstart = expr 777 i = k 778 } 779 if e.Col.Name == TimeWindowEnd { 780 wend = expr 781 j = k 782 } 783 continue 784 } 785 increaseRefCnt(expr, -1, colRefCnt) 786 err = builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 787 if err != nil { 788 return nil, err 789 } 790 791 globalRef := [2]int32{timeTag, int32(k)} 792 if colRefCnt[globalRef] == 0 { 793 continue 794 } 795 796 remapping.addColRef(globalRef) 797 798 node.ProjectList = append(node.ProjectList, &plan.Expr{ 799 Typ: expr.Typ, 800 Expr: &plan.Expr_Col{ 801 Col: &ColRef{ 802 RelPos: -1, 803 ColPos: int32(idx), 804 Name: builder.nameByColRef[globalRef], 805 }, 806 }, 807 }) 808 idx++ 809 } 810 811 if wstart != nil { 812 increaseRefCnt(wstart, -1, colRefCnt) 813 814 globalRef := [2]int32{timeTag, int32(i)} 815 if colRefCnt[globalRef] == 0 { 816 break 817 } 818 819 remapping.addColRef(globalRef) 820 821 node.ProjectList = append(node.ProjectList, &plan.Expr{ 822 Typ: wstart.Typ, 823 Expr: &plan.Expr_Col{ 824 Col: &ColRef{ 825 RelPos: -1, 826 ColPos: int32(idx), 827 Name: builder.nameByColRef[globalRef], 828 }, 829 }, 830 }) 831 idx++ 832 } 833 834 if wend != nil { 835 increaseRefCnt(wend, -1, colRefCnt) 836 837 globalRef := [2]int32{timeTag, int32(j)} 838 if colRefCnt[globalRef] == 0 { 839 break 840 } 841 842 remapping.addColRef(globalRef) 843 844 node.ProjectList = append(node.ProjectList, &plan.Expr{ 845 Typ: wstart.Typ, 846 Expr: &plan.Expr_Col{ 847 Col: &ColRef{ 848 RelPos: -1, 849 ColPos: int32(idx), 850 Name: builder.nameByColRef[globalRef], 851 }, 852 }, 853 }) 854 } 855 856 case plan.Node_WINDOW: 857 for _, expr := range node.WinSpecList { 858 increaseRefCnt(expr, 1, colRefCnt) 859 } 860 861 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 862 if err != nil { 863 return nil, err 864 } 865 866 childProjList := builder.qry.Nodes[node.Children[0]].ProjectList 867 for i, globalRef := range childRemapping.localToGlobal { 868 if colRefCnt[globalRef] == 0 { 869 continue 870 } 871 872 remapping.addColRef(globalRef) 873 874 node.ProjectList = append(node.ProjectList, &plan.Expr{ 875 Typ: childProjList[i].Typ, 876 Expr: &plan.Expr_Col{ 877 Col: &plan.ColRef{ 878 RelPos: 0, 879 ColPos: int32(i), 880 Name: builder.nameByColRef[globalRef], 881 }, 882 }, 883 }) 884 } 885 886 windowTag := node.BindingTags[0] 887 l := len(childProjList) 888 889 for _, expr := range node.FilterList { 890 builder.remapWindowClause(expr, windowTag, int32(l)) 891 } 892 893 for _, expr := range node.WinSpecList { 894 increaseRefCnt(expr, -1, colRefCnt) 895 err = builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 896 if err != nil { 897 return nil, err 898 } 899 900 globalRef := [2]int32{windowTag, int32(node.GetWindowIdx())} 901 if colRefCnt[globalRef] == 0 { 902 continue 903 } 904 905 remapping.addColRef(globalRef) 906 907 node.ProjectList = append(node.ProjectList, &plan.Expr{ 908 Typ: expr.Typ, 909 Expr: &plan.Expr_Col{ 910 Col: &ColRef{ 911 RelPos: -1, 912 ColPos: int32(l), 913 Name: builder.nameByColRef[globalRef], 914 }, 915 }, 916 }) 917 } 918 919 case plan.Node_FILL: 920 921 //for _, expr := range node.AggList { 922 // increaseRefCnt(expr, 1, colRefCnt) 923 //} 924 925 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 926 if err != nil { 927 return nil, err 928 } 929 930 childProjList := builder.qry.Nodes[node.Children[0]].ProjectList 931 for i, globalRef := range childRemapping.localToGlobal { 932 if colRefCnt[globalRef] == 0 { 933 continue 934 } 935 936 remapping.addColRef(globalRef) 937 938 node.ProjectList = append(node.ProjectList, &plan.Expr{ 939 Typ: childProjList[i].Typ, 940 Expr: &plan.Expr_Col{ 941 Col: &plan.ColRef{ 942 RelPos: 0, 943 ColPos: int32(i), 944 Name: builder.nameByColRef[globalRef], 945 }, 946 }, 947 }) 948 } 949 950 //for _, expr := range node.AggList { 951 // increaseRefCnt(expr, -1, colRefCnt) 952 // err = builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 953 // if err != nil { 954 // return nil, err 955 // } 956 //} 957 958 case plan.Node_SORT, plan.Node_PARTITION: 959 for _, orderBy := range node.OrderBy { 960 increaseRefCnt(orderBy.Expr, 1, colRefCnt) 961 } 962 963 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 964 if err != nil { 965 return nil, err 966 } 967 968 for _, orderBy := range node.OrderBy { 969 increaseRefCnt(orderBy.Expr, -1, colRefCnt) 970 err := builder.remapColRefForExpr(orderBy.Expr, childRemapping.globalToLocal) 971 if err != nil { 972 return nil, err 973 } 974 } 975 976 childProjList := builder.qry.Nodes[node.Children[0]].ProjectList 977 for i, globalRef := range childRemapping.localToGlobal { 978 if colRefCnt[globalRef] == 0 { 979 continue 980 } 981 982 remapping.addColRef(globalRef) 983 984 node.ProjectList = append(node.ProjectList, &plan.Expr{ 985 Typ: childProjList[i].Typ, 986 Expr: &plan.Expr_Col{ 987 Col: &plan.ColRef{ 988 RelPos: 0, 989 ColPos: int32(i), 990 Name: builder.nameByColRef[globalRef], 991 }, 992 }, 993 }) 994 } 995 996 if len(node.ProjectList) == 0 && len(childRemapping.localToGlobal) > 0 { 997 globalRef := childRemapping.localToGlobal[0] 998 remapping.addColRef(globalRef) 999 1000 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1001 Typ: childProjList[0].Typ, 1002 Expr: &plan.Expr_Col{ 1003 Col: &plan.ColRef{ 1004 RelPos: 0, 1005 ColPos: 0, 1006 Name: builder.nameByColRef[globalRef], 1007 }, 1008 }, 1009 }) 1010 } 1011 1012 case plan.Node_FILTER: 1013 for _, expr := range node.FilterList { 1014 increaseRefCnt(expr, 1, colRefCnt) 1015 } 1016 1017 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 1018 if err != nil { 1019 return nil, err 1020 } 1021 1022 for _, expr := range node.FilterList { 1023 increaseRefCnt(expr, -1, colRefCnt) 1024 err := builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 1025 if err != nil { 1026 return nil, err 1027 } 1028 } 1029 1030 childProjList := builder.qry.Nodes[node.Children[0]].ProjectList 1031 for i, globalRef := range childRemapping.localToGlobal { 1032 if colRefCnt[globalRef] == 0 { 1033 continue 1034 } 1035 1036 remapping.addColRef(globalRef) 1037 1038 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1039 Typ: childProjList[i].Typ, 1040 Expr: &plan.Expr_Col{ 1041 Col: &plan.ColRef{ 1042 RelPos: 0, 1043 ColPos: int32(i), 1044 Name: builder.nameByColRef[globalRef], 1045 }, 1046 }, 1047 }) 1048 } 1049 1050 if len(node.ProjectList) == 0 { 1051 if len(childRemapping.localToGlobal) > 0 { 1052 remapping.addColRef(childRemapping.localToGlobal[0]) 1053 } 1054 1055 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1056 Typ: childProjList[0].Typ, 1057 Expr: &plan.Expr_Col{ 1058 Col: &plan.ColRef{ 1059 RelPos: 0, 1060 ColPos: 0, 1061 }, 1062 }, 1063 }) 1064 } 1065 1066 case plan.Node_SINK_SCAN, plan.Node_RECURSIVE_SCAN, plan.Node_RECURSIVE_CTE: 1067 tag := node.BindingTags[0] 1068 var newProjList []*plan.Expr 1069 1070 for i, expr := range node.ProjectList { 1071 globalRef := [2]int32{tag, int32(i)} 1072 if colRefCnt[globalRef] == 0 { 1073 continue 1074 } 1075 newProjList = append(newProjList, &plan.Expr{ 1076 Typ: expr.Typ, 1077 Expr: &plan.Expr_Col{ 1078 Col: &ColRef{ 1079 RelPos: 0, 1080 ColPos: int32(i), 1081 }, 1082 }, 1083 }) 1084 remapping.addColRef(globalRef) 1085 for _, sourceStep := range node.SourceStep { 1086 if sourceStep >= step { 1087 continue 1088 } 1089 colRefBool[[2]int32{sourceStep, int32(i)}] = true 1090 } 1091 1092 } 1093 node.ProjectList = newProjList 1094 1095 case plan.Node_SINK: 1096 childNode := builder.qry.Nodes[node.Children[0]] 1097 resultTag := childNode.BindingTags[0] 1098 for i := range childNode.ProjectList { 1099 if colRefBool[[2]int32{step, int32(i)}] { 1100 colRefCnt[[2]int32{resultTag, int32(i)}] = 1 1101 } 1102 } 1103 1104 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 1105 if err != nil { 1106 return nil, err 1107 } 1108 var newProjList []*plan.Expr 1109 for i, expr := range node.ProjectList { 1110 if !colRefBool[[2]int32{step, int32(i)}] { 1111 continue 1112 } 1113 sinkColRef[[2]int32{step, int32(i)}] = len(newProjList) 1114 newProjList = append(newProjList, &plan.Expr{ 1115 Typ: expr.Typ, 1116 Expr: &plan.Expr_Col{ 1117 Col: &ColRef{ 1118 RelPos: 0, 1119 ColPos: childRemapping.globalToLocal[[2]int32{resultTag, int32(i)}][1], 1120 // Name: builder.nameByColRef[globalRef], 1121 }, 1122 }, 1123 }) 1124 } 1125 1126 node.ProjectList = newProjList 1127 1128 case plan.Node_PROJECT, plan.Node_MATERIAL: 1129 projectTag := node.BindingTags[0] 1130 1131 var neededProj []int32 1132 1133 for i, expr := range node.ProjectList { 1134 globalRef := [2]int32{projectTag, int32(i)} 1135 if colRefCnt[globalRef] == 0 { 1136 continue 1137 } 1138 1139 neededProj = append(neededProj, int32(i)) 1140 increaseRefCnt(expr, 1, colRefCnt) 1141 } 1142 1143 if len(neededProj) == 0 { 1144 increaseRefCnt(node.ProjectList[0], 1, colRefCnt) 1145 neededProj = append(neededProj, 0) 1146 } 1147 1148 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 1149 if err != nil { 1150 return nil, err 1151 } 1152 1153 childProjList := builder.qry.Nodes[node.Children[0]].ProjectList 1154 var newProjList []*plan.Expr 1155 for _, needed := range neededProj { 1156 expr := node.ProjectList[needed] 1157 increaseRefCnt(expr, -1, colRefCnt) 1158 err := builder.remapColRefForExpr(expr, childRemapping.globalToLocal) 1159 if err != nil { 1160 return nil, err 1161 } 1162 1163 switch ne := expr.Expr.(type) { 1164 case *plan.Expr_Col: 1165 expr.Typ.NotNullable = childProjList[ne.Col.ColPos].Typ.NotNullable 1166 } 1167 1168 globalRef := [2]int32{projectTag, needed} 1169 remapping.addColRef(globalRef) 1170 1171 newProjList = append(newProjList, expr) 1172 } 1173 1174 node.ProjectList = newProjList 1175 1176 case plan.Node_DISTINCT: 1177 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 1178 if err != nil { 1179 return nil, err 1180 } 1181 1182 // Rewrite DISTINCT to AGG 1183 node.NodeType = plan.Node_AGG 1184 preNode := builder.qry.Nodes[node.Children[0]] 1185 node.GroupBy = make([]*Expr, len(preNode.ProjectList)) 1186 node.ProjectList = make([]*Expr, len(preNode.ProjectList)) 1187 1188 for i, prjExpr := range preNode.ProjectList { 1189 node.GroupBy[i] = &plan.Expr{ 1190 Typ: prjExpr.Typ, 1191 Expr: &plan.Expr_Col{ 1192 Col: &plan.ColRef{ 1193 RelPos: 0, 1194 ColPos: int32(i), 1195 }, 1196 }, 1197 } 1198 1199 node.ProjectList[i] = &plan.Expr{ 1200 Typ: prjExpr.Typ, 1201 Expr: &plan.Expr_Col{ 1202 Col: &plan.ColRef{ 1203 RelPos: -1, 1204 ColPos: int32(i), 1205 }, 1206 }, 1207 } 1208 } 1209 1210 remapping = childRemapping 1211 1212 case plan.Node_VALUE_SCAN: 1213 node.NotCacheable = true 1214 // VALUE_SCAN always have one column now 1215 if node.TableDef == nil { // like select 1,2 1216 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1217 Typ: plan.Type{Id: int32(types.T_int64)}, 1218 Expr: &plan.Expr_Lit{Lit: &plan.Literal{Value: &plan.Literal_I64Val{I64Val: 0}}}, 1219 }) 1220 } else { 1221 internalRemapping := &ColRefRemapping{ 1222 globalToLocal: make(map[[2]int32][2]int32), 1223 } 1224 1225 tag := node.BindingTags[0] 1226 for i := range node.TableDef.Cols { 1227 globalRef := [2]int32{tag, int32(i)} 1228 if colRefCnt[globalRef] == 0 { 1229 continue 1230 } 1231 internalRemapping.addColRef(globalRef) 1232 } 1233 1234 for i, col := range node.TableDef.Cols { 1235 if colRefCnt[internalRemapping.localToGlobal[i]] == 0 { 1236 continue 1237 } 1238 1239 remapping.addColRef(internalRemapping.localToGlobal[i]) 1240 1241 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1242 Typ: col.Typ, 1243 Expr: &plan.Expr_Col{ 1244 Col: &plan.ColRef{ 1245 RelPos: 0, 1246 ColPos: int32(i), 1247 Name: col.Name, 1248 }, 1249 }, 1250 }) 1251 } 1252 } 1253 1254 case plan.Node_LOCK_OP: 1255 preNode := builder.qry.Nodes[node.Children[0]] 1256 pkexpr := &plan.Expr{ 1257 Typ: node.LockTargets[0].GetPrimaryColTyp(), 1258 Expr: &plan.Expr_Col{ 1259 Col: &plan.ColRef{ 1260 RelPos: node.BindingTags[1], 1261 ColPos: node.LockTargets[0].PrimaryColIdxInBat, 1262 }, 1263 }, 1264 } 1265 oldPos := [2]int32{node.BindingTags[1], node.LockTargets[0].PrimaryColIdxInBat} 1266 increaseRefCnt(pkexpr, 1, colRefCnt) 1267 childRemapping, err := builder.remapAllColRefs(node.Children[0], step, colRefCnt, colRefBool, sinkColRef) 1268 if err != nil { 1269 return nil, err 1270 } 1271 1272 tableDef := node.GetTableDef() 1273 if tableDef.Partition != nil { 1274 partitionIdx := len(preNode.ProjectList) 1275 partitionExpr := DeepCopyExpr(tableDef.Partition.PartitionExpression) 1276 preNode.ProjectList = append(preNode.ProjectList, partitionExpr) 1277 1278 partTableIDs, _ := getPartTableIdsAndNames(builder.compCtx, preNode.GetObjRef(), tableDef) 1279 node.LockTargets[0].IsPartitionTable = true 1280 node.LockTargets[0].PartitionTableIds = partTableIDs 1281 node.LockTargets[0].FilterColIdxInBat = int32(partitionIdx) 1282 } 1283 1284 if newPos, ok := childRemapping.globalToLocal[oldPos]; ok { 1285 node.LockTargets[0].PrimaryColIdxInBat = newPos[1] 1286 } 1287 increaseRefCnt(pkexpr, -1, colRefCnt) 1288 1289 for i, globalRef := range childRemapping.localToGlobal { 1290 if colRefCnt[globalRef] == 0 { 1291 continue 1292 } 1293 remapping.addColRef(globalRef) 1294 1295 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1296 Typ: preNode.ProjectList[i].Typ, 1297 Expr: &plan.Expr_Col{ 1298 Col: &plan.ColRef{ 1299 RelPos: 0, 1300 ColPos: int32(i), 1301 Name: builder.nameByColRef[globalRef], 1302 }, 1303 }, 1304 }) 1305 } 1306 1307 if len(node.ProjectList) == 0 { 1308 if len(childRemapping.localToGlobal) > 0 { 1309 remapping.addColRef(childRemapping.localToGlobal[0]) 1310 } 1311 1312 node.ProjectList = append(node.ProjectList, &plan.Expr{ 1313 Typ: preNode.ProjectList[0].Typ, 1314 Expr: &plan.Expr_Col{ 1315 Col: &plan.ColRef{ 1316 RelPos: 0, 1317 ColPos: 0, 1318 }, 1319 }, 1320 }) 1321 } 1322 1323 default: 1324 return nil, moerr.NewInternalError(builder.GetContext(), "unsupport node type") 1325 } 1326 1327 node.BindingTags = nil 1328 1329 return remapping, nil 1330 } 1331 1332 func (builder *QueryBuilder) markSinkProject(nodeID int32, step int32, colRefBool map[[2]int32]bool) { 1333 node := builder.qry.Nodes[nodeID] 1334 1335 switch node.NodeType { 1336 case plan.Node_SINK_SCAN, plan.Node_RECURSIVE_SCAN, plan.Node_RECURSIVE_CTE: 1337 for _, i := range node.SourceStep { 1338 if i >= step { 1339 for _, expr := range node.ProjectList { 1340 colRefBool[[2]int32{i, expr.GetCol().ColPos}] = true 1341 } 1342 } 1343 } 1344 default: 1345 for i := range node.Children { 1346 builder.markSinkProject(node.Children[i], step, colRefBool) 1347 } 1348 } 1349 } 1350 1351 func (builder *QueryBuilder) rewriteStarApproxCount(nodeID int32) { 1352 node := builder.qry.Nodes[nodeID] 1353 1354 switch node.NodeType { 1355 case plan.Node_AGG: 1356 if len(node.GroupBy) == 0 && len(node.AggList) == 1 { 1357 if agg := node.AggList[0].GetF(); agg != nil && agg.Func.ObjName == "approx_count" { 1358 if len(node.Children) == 1 { 1359 child := builder.qry.Nodes[node.Children[0]] 1360 if child.NodeType == plan.Node_TABLE_SCAN && len(child.FilterList) == 0 { 1361 agg.Func.ObjName = "sum" 1362 fr, _ := function.GetFunctionByName(context.TODO(), "sum", []types.Type{types.T_int64.ToType()}) 1363 agg.Func.Obj = fr.GetEncodedOverloadID() 1364 agg.Args[0] = &plan.Expr{ 1365 Expr: &plan.Expr_Col{ 1366 Col: &plan.ColRef{ 1367 RelPos: 0, 1368 ColPos: Metadata_Rows_Cnt_Pos, 1369 }, 1370 }, 1371 } 1372 1373 var exprs []*plan.Expr 1374 str := child.ObjRef.SchemaName + "." + child.TableDef.Name 1375 exprs = append(exprs, &plan.Expr{ 1376 Typ: Type{ 1377 Id: int32(types.T_varchar), 1378 NotNullable: true, 1379 Width: int32(len(str)), 1380 }, 1381 Expr: &plan.Expr_Lit{ 1382 Lit: &plan.Literal{ 1383 Value: &plan.Literal_Sval{ 1384 Sval: str, 1385 }, 1386 }, 1387 }, 1388 }) 1389 str = child.TableDef.Cols[0].Name 1390 exprs = append(exprs, &plan.Expr{ 1391 Typ: Type{ 1392 Id: int32(types.T_varchar), 1393 NotNullable: true, 1394 Width: int32(len(str)), 1395 }, 1396 Expr: &plan.Expr_Lit{ 1397 Lit: &plan.Literal{ 1398 Value: &plan.Literal_Sval{ 1399 Sval: str, 1400 }, 1401 }, 1402 }, 1403 }) 1404 scanNode := &plan.Node{ 1405 NodeType: plan.Node_VALUE_SCAN, 1406 } 1407 childId := builder.appendNode(scanNode, nil) 1408 node.Children[0] = builder.buildMetadataScan(nil, nil, exprs, childId) 1409 child = builder.qry.Nodes[node.Children[0]] 1410 switch expr := agg.Args[0].Expr.(type) { 1411 case *plan.Expr_Col: 1412 expr.Col.RelPos = child.BindingTags[0] 1413 agg.Args[0].Typ = child.TableDef.Cols[expr.Col.ColPos].Typ 1414 } 1415 } 1416 } 1417 } 1418 } 1419 default: 1420 for i := range node.Children { 1421 builder.rewriteStarApproxCount(node.Children[i]) 1422 } 1423 } 1424 } 1425 1426 func (builder *QueryBuilder) createQuery() (*Query, error) { 1427 colRefBool := make(map[[2]int32]bool) 1428 sinkColRef := make(map[[2]int32]int) 1429 1430 builder.parseOptimizeHints() 1431 for i, rootID := range builder.qry.Steps { 1432 builder.skipStats = builder.canSkipStats() 1433 builder.rewriteDistinctToAGG(rootID) 1434 builder.rewriteEffectlessAggToProject(rootID) 1435 rootID, _ = builder.pushdownFilters(rootID, nil, false) 1436 err := foldTableScanFilters(builder.compCtx.GetProcess(), builder.qry, rootID) 1437 if err != nil { 1438 return nil, err 1439 } 1440 builder.optimizeDateFormatExpr(rootID) 1441 1442 builder.pushdownLimitToTableScan(rootID) 1443 1444 colRefCnt := make(map[[2]int32]int) 1445 builder.countColRefs(rootID, colRefCnt) 1446 builder.removeSimpleProjections(rootID, plan.Node_UNKNOWN, false, colRefCnt) 1447 1448 rewriteFilterListByStats(builder.GetContext(), rootID, builder) 1449 ReCalcNodeStats(rootID, builder, true, true, true) 1450 builder.determineBuildAndProbeSide(rootID, true) 1451 determineHashOnPK(rootID, builder) 1452 tagCnt := make(map[int32]int) 1453 rootID = builder.removeEffectlessLeftJoins(rootID, tagCnt) 1454 ReCalcNodeStats(rootID, builder, true, false, true) 1455 builder.pushdownTopThroughLeftJoin(rootID) 1456 ReCalcNodeStats(rootID, builder, true, false, true) 1457 1458 rootID = builder.aggPushDown(rootID) 1459 ReCalcNodeStats(rootID, builder, true, false, true) 1460 rootID = builder.determineJoinOrder(rootID) 1461 colMap := make(map[[2]int32]int) 1462 colGroup := make([]int, 0) 1463 builder.removeRedundantJoinCond(rootID, colMap, colGroup) 1464 ReCalcNodeStats(rootID, builder, true, false, true) 1465 rootID = builder.applyAssociativeLaw(rootID) 1466 builder.determineBuildAndProbeSide(rootID, true) 1467 rootID = builder.aggPullup(rootID, rootID) 1468 ReCalcNodeStats(rootID, builder, true, false, true) 1469 rootID = builder.pushdownSemiAntiJoins(rootID) 1470 builder.optimizeDistinctAgg(rootID) 1471 ReCalcNodeStats(rootID, builder, true, false, true) 1472 builder.determineBuildAndProbeSide(rootID, true) 1473 1474 builder.qry.Steps[i] = rootID 1475 1476 // XXX: This will be removed soon, after merging implementation of all hash-join operators 1477 builder.swapJoinChildren(rootID) 1478 ReCalcNodeStats(rootID, builder, true, false, true) 1479 1480 builder.partitionPrune(rootID) 1481 1482 builder.optimizeLikeExpr(rootID) 1483 rootID = builder.applyIndices(rootID, colRefCnt, make(map[[2]int32]*plan.Expr)) 1484 ReCalcNodeStats(rootID, builder, true, false, true) 1485 1486 determineHashOnPK(rootID, builder) 1487 determineShuffleMethod(rootID, builder) 1488 determineShuffleMethod2(rootID, -1, builder) 1489 // after determine shuffle, be careful when calling ReCalcNodeStats again. 1490 // needResetHashMapStats should always be false from here 1491 1492 builder.generateRuntimeFilters(rootID) 1493 ReCalcNodeStats(rootID, builder, true, false, false) 1494 builder.forceJoinOnOneCN(rootID, false) 1495 // after this ,never call ReCalcNodeStats again !!! 1496 1497 if builder.isForUpdate { 1498 reCheckifNeedLockWholeTable(builder) 1499 } 1500 1501 builder.handleMessgaes(rootID) 1502 1503 builder.rewriteStarApproxCount(rootID) 1504 1505 rootNode := builder.qry.Nodes[rootID] 1506 1507 for j := range rootNode.ProjectList { 1508 colRefBool[[2]int32{int32(i), int32(j)}] = false 1509 if i == len(builder.qry.Steps)-1 { 1510 colRefBool[[2]int32{int32(i), int32(j)}] = true 1511 } 1512 } 1513 } 1514 1515 for i := range builder.qry.Steps { 1516 rootID := builder.qry.Steps[i] 1517 builder.markSinkProject(rootID, int32(i), colRefBool) 1518 } 1519 1520 for i := len(builder.qry.Steps) - 1; i >= 0; i-- { 1521 rootID := builder.qry.Steps[i] 1522 rootNode := builder.qry.Nodes[rootID] 1523 resultTag := rootNode.BindingTags[0] 1524 colRefCnt := make(map[[2]int32]int) 1525 for j := range rootNode.ProjectList { 1526 colRefCnt[[2]int32{resultTag, int32(j)}] = 1 1527 } 1528 _, err := builder.remapAllColRefs(rootID, int32(i), colRefCnt, colRefBool, sinkColRef) 1529 if err != nil { 1530 return nil, err 1531 } 1532 } 1533 1534 //for i := 1; i < len(builder.qry.Steps); i++ { 1535 // builder.remapSinkScanColRefs(builder.qry.Steps[i], int32(i), sinkColRef) 1536 //} 1537 1538 return builder.qry, nil 1539 } 1540 1541 func (builder *QueryBuilder) buildUnion(stmt *tree.UnionClause, astOrderBy tree.OrderBy, astLimit *tree.Limit, ctx *BindContext, isRoot bool) (int32, error) { 1542 var selectStmts []tree.Statement 1543 var unionTypes []plan.Node_NodeType 1544 1545 // get Union selectStmts 1546 err := getUnionSelects(builder.GetContext(), stmt, &selectStmts, &unionTypes) 1547 if err != nil { 1548 return 0, err 1549 } 1550 1551 if len(selectStmts) == 1 { 1552 switch sltStmt := selectStmts[0].(type) { 1553 case *tree.Select: 1554 if sltClause, ok := sltStmt.Select.(*tree.SelectClause); ok { 1555 sltClause.Distinct = true 1556 return builder.buildSelect(sltStmt, ctx, isRoot) 1557 } else { 1558 // rewrite sltStmt to select distinct * from (sltStmt) a 1559 tmpSltStmt := &tree.Select{ 1560 Select: &tree.SelectClause{ 1561 Distinct: true, 1562 1563 Exprs: []tree.SelectExpr{ 1564 {Expr: tree.StarExpr()}, 1565 }, 1566 From: &tree.From{ 1567 Tables: tree.TableExprs{ 1568 &tree.AliasedTableExpr{ 1569 Expr: &tree.ParenTableExpr{ 1570 Expr: sltStmt, 1571 }, 1572 As: tree.AliasClause{ 1573 Alias: "a", 1574 }, 1575 }, 1576 }, 1577 }, 1578 }, 1579 Limit: astLimit, 1580 OrderBy: astOrderBy, 1581 } 1582 return builder.buildSelect(tmpSltStmt, ctx, isRoot) 1583 } 1584 1585 case *tree.SelectClause: 1586 if !sltStmt.Distinct { 1587 sltStmt.Distinct = true 1588 } 1589 return builder.buildSelect(&tree.Select{Select: sltStmt, Limit: astLimit, OrderBy: astOrderBy}, ctx, isRoot) 1590 } 1591 } 1592 1593 // build selects 1594 var projectTypList [][]types.Type 1595 selectStmtLength := len(selectStmts) 1596 nodes := make([]int32, selectStmtLength) 1597 subCtxList := make([]*BindContext, selectStmtLength) 1598 var projectLength int 1599 var nodeID int32 1600 for idx, sltStmt := range selectStmts { 1601 subCtx := NewBindContext(builder, ctx) 1602 subCtx.unionSelect = subCtx.initSelect 1603 if slt, ok := sltStmt.(*tree.Select); ok { 1604 nodeID, err = builder.buildSelect(slt, subCtx, isRoot) 1605 } else { 1606 nodeID, err = builder.buildSelect(&tree.Select{Select: sltStmt}, subCtx, isRoot) 1607 } 1608 if err != nil { 1609 return 0, err 1610 } 1611 ctx.views = append(ctx.views, subCtx.views...) 1612 1613 if idx == 0 { 1614 projectLength = len(builder.qry.Nodes[nodeID].ProjectList) 1615 projectTypList = make([][]types.Type, projectLength) 1616 for i := 0; i < projectLength; i++ { 1617 projectTypList[i] = make([]types.Type, selectStmtLength) 1618 } 1619 } else { 1620 if projectLength != len(builder.qry.Nodes[nodeID].ProjectList) { 1621 return 0, moerr.NewParseError(builder.GetContext(), "SELECT statements have different number of columns") 1622 } 1623 } 1624 1625 for i, expr := range subCtx.results { 1626 projectTypList[i][idx] = makeTypeByPlan2Expr(expr) 1627 } 1628 subCtxList[idx] = subCtx 1629 nodes[idx] = nodeID 1630 } 1631 1632 // reset all select's return Projection(type cast up) 1633 // we use coalesce function's type check&type cast rule 1634 for columnIdx, argsType := range projectTypList { 1635 // we don't cast null as any type in function 1636 // but we will cast null as some target type in union/intersect/minus 1637 var tmpArgsType []types.Type 1638 for _, typ := range argsType { 1639 if typ.Oid != types.T_any { 1640 tmpArgsType = append(tmpArgsType, typ) 1641 } 1642 } 1643 1644 if len(tmpArgsType) > 0 { 1645 fGet, err := function.GetFunctionByName(builder.GetContext(), "coalesce", tmpArgsType) 1646 if err != nil { 1647 return 0, moerr.NewParseError(builder.GetContext(), "the %d column cann't cast to a same type", columnIdx) 1648 } 1649 argsCastType, _ := fGet.ShouldDoImplicitTypeCast() 1650 1651 if len(argsCastType) > 0 && int(argsCastType[0].Oid) == int(types.T_datetime) { 1652 for i := 0; i < len(argsCastType); i++ { 1653 argsCastType[i].Scale = 0 1654 } 1655 } 1656 var targetType plan.Type 1657 var targetArgType types.Type 1658 if len(argsCastType) == 0 { 1659 targetArgType = tmpArgsType[0] 1660 // if string union string, different length may cause error. 1661 if targetArgType.Oid == types.T_varchar || targetArgType.Oid == types.T_char { 1662 for _, typ := range argsType { 1663 if targetArgType.Width < typ.Width { 1664 targetArgType.Width = typ.Width 1665 } 1666 } 1667 } 1668 } else { 1669 targetArgType = argsCastType[0] 1670 } 1671 1672 if targetArgType.Oid == types.T_binary || targetArgType.Oid == types.T_varbinary { 1673 targetArgType = types.T_blob.ToType() 1674 } 1675 targetType = makePlan2Type(&targetArgType) 1676 1677 for idx, tmpID := range nodes { 1678 if !argsType[idx].Eq(targetArgType) { 1679 node := builder.qry.Nodes[tmpID] 1680 if argsType[idx].Oid == types.T_any { 1681 node.ProjectList[columnIdx].Typ = targetType 1682 } else { 1683 node.ProjectList[columnIdx], err = appendCastBeforeExpr(builder.GetContext(), node.ProjectList[columnIdx], targetType) 1684 if err != nil { 1685 return 0, err 1686 } 1687 } 1688 } 1689 } 1690 } 1691 } 1692 1693 firstSelectProjectNode := builder.qry.Nodes[nodes[0]] 1694 // set ctx's headings projects results 1695 ctx.headings = append(ctx.headings, subCtxList[0].headings...) 1696 1697 getProjectList := func(tag int32, thisTag int32) []*plan.Expr { 1698 projectList := make([]*plan.Expr, len(firstSelectProjectNode.ProjectList)) 1699 for i, expr := range firstSelectProjectNode.ProjectList { 1700 projectList[i] = &plan.Expr{ 1701 Typ: expr.Typ, 1702 Expr: &plan.Expr_Col{ 1703 Col: &plan.ColRef{ 1704 RelPos: tag, 1705 ColPos: int32(i), 1706 }, 1707 }, 1708 } 1709 builder.nameByColRef[[2]int32{thisTag, int32(i)}] = ctx.headings[i] 1710 } 1711 return projectList 1712 } 1713 1714 // build intersect node first. because intersect has higher precedence then UNION and MINUS 1715 var newNodes []int32 1716 var newUnionType []plan.Node_NodeType 1717 var lastTag int32 1718 newNodes = append(newNodes, nodes[0]) 1719 for i := 1; i < len(nodes); i++ { 1720 utIdx := i - 1 1721 lastNewNodeIdx := len(newNodes) - 1 1722 if unionTypes[utIdx] == plan.Node_INTERSECT || unionTypes[utIdx] == plan.Node_INTERSECT_ALL { 1723 lastTag = builder.genNewTag() 1724 leftNodeTag := builder.qry.Nodes[newNodes[lastNewNodeIdx]].BindingTags[0] 1725 newNodeID := builder.appendNode(&plan.Node{ 1726 NodeType: unionTypes[utIdx], 1727 Children: []int32{newNodes[lastNewNodeIdx], nodes[i]}, 1728 BindingTags: []int32{lastTag}, 1729 ProjectList: getProjectList(leftNodeTag, lastTag), 1730 }, ctx) 1731 newNodes[lastNewNodeIdx] = newNodeID 1732 } else { 1733 newNodes = append(newNodes, nodes[i]) 1734 newUnionType = append(newUnionType, unionTypes[utIdx]) 1735 } 1736 } 1737 1738 // build UNION/MINUS node one by one 1739 lastNodeID := newNodes[0] 1740 for i := 1; i < len(newNodes); i++ { 1741 utIdx := i - 1 1742 lastTag = builder.genNewTag() 1743 leftNodeTag := builder.qry.Nodes[lastNodeID].BindingTags[0] 1744 1745 lastNodeID = builder.appendNode(&plan.Node{ 1746 NodeType: newUnionType[utIdx], 1747 Children: []int32{lastNodeID, newNodes[i]}, 1748 BindingTags: []int32{lastTag}, 1749 ProjectList: getProjectList(leftNodeTag, lastTag), 1750 }, ctx) 1751 } 1752 1753 // set ctx base on selects[0] and it's ctx 1754 ctx.groupTag = builder.genNewTag() 1755 ctx.aggregateTag = builder.genNewTag() 1756 ctx.projectTag = builder.genNewTag() 1757 for i, v := range ctx.headings { 1758 ctx.aliasMap[v] = &aliasItem{ 1759 idx: int32(i), 1760 } 1761 builder.nameByColRef[[2]int32{ctx.projectTag, int32(i)}] = v 1762 } 1763 for i, expr := range firstSelectProjectNode.ProjectList { 1764 ctx.projects = append(ctx.projects, &plan.Expr{ 1765 Typ: expr.Typ, 1766 Expr: &plan.Expr_Col{ 1767 Col: &plan.ColRef{ 1768 RelPos: lastTag, 1769 ColPos: int32(i), 1770 }, 1771 }, 1772 }) 1773 } 1774 havingBinder := NewHavingBinder(builder, ctx) 1775 projectionBinder := NewProjectionBinder(builder, ctx, havingBinder) 1776 1777 // append a project node 1778 lastNodeID = builder.appendNode(&plan.Node{ 1779 NodeType: plan.Node_PROJECT, 1780 ProjectList: ctx.projects, 1781 Children: []int32{lastNodeID}, 1782 BindingTags: []int32{ctx.projectTag}, 1783 }, ctx) 1784 1785 // append orderBy 1786 if astOrderBy != nil { 1787 orderBinder := NewOrderBinder(projectionBinder, nil) 1788 orderBys := make([]*plan.OrderBySpec, 0, len(astOrderBy)) 1789 1790 for _, order := range astOrderBy { 1791 expr, err := orderBinder.BindExpr(order.Expr) 1792 if err != nil { 1793 return 0, err 1794 } 1795 1796 orderBy := &plan.OrderBySpec{ 1797 Expr: expr, 1798 Flag: plan.OrderBySpec_INTERNAL, 1799 } 1800 1801 switch order.Direction { 1802 case tree.Ascending: 1803 orderBy.Flag |= plan.OrderBySpec_ASC 1804 case tree.Descending: 1805 orderBy.Flag |= plan.OrderBySpec_DESC 1806 } 1807 1808 switch order.NullsPosition { 1809 case tree.NullsFirst: 1810 orderBy.Flag |= plan.OrderBySpec_NULLS_FIRST 1811 case tree.NullsLast: 1812 orderBy.Flag |= plan.OrderBySpec_NULLS_LAST 1813 } 1814 1815 orderBys = append(orderBys, orderBy) 1816 } 1817 1818 lastNodeID = builder.appendNode(&plan.Node{ 1819 NodeType: plan.Node_SORT, 1820 Children: []int32{lastNodeID}, 1821 OrderBy: orderBys, 1822 }, ctx) 1823 } 1824 1825 // append limit 1826 if astLimit != nil { 1827 node := builder.qry.Nodes[lastNodeID] 1828 1829 limitBinder := NewLimitBinder(builder, ctx) 1830 if astLimit.Offset != nil { 1831 node.Offset, err = limitBinder.BindExpr(astLimit.Offset, 0, true) 1832 if err != nil { 1833 return 0, err 1834 } 1835 } 1836 if astLimit.Count != nil { 1837 node.Limit, err = limitBinder.BindExpr(astLimit.Count, 0, true) 1838 if err != nil { 1839 return 0, err 1840 } 1841 1842 if cExpr, ok := node.Limit.Expr.(*plan.Expr_Lit); ok { 1843 if c, ok := cExpr.Lit.Value.(*plan.Literal_I64Val); ok { 1844 ctx.hasSingleRow = c.I64Val == 1 1845 } 1846 } 1847 } 1848 } 1849 1850 // append result PROJECT node 1851 if builder.qry.Nodes[lastNodeID].NodeType != plan.Node_PROJECT { 1852 for i := 0; i < len(ctx.projects); i++ { 1853 ctx.results = append(ctx.results, &plan.Expr{ 1854 Typ: ctx.projects[i].Typ, 1855 Expr: &plan.Expr_Col{ 1856 Col: &plan.ColRef{ 1857 RelPos: ctx.projectTag, 1858 ColPos: int32(i), 1859 }, 1860 }, 1861 }) 1862 } 1863 ctx.resultTag = builder.genNewTag() 1864 1865 lastNodeID = builder.appendNode(&plan.Node{ 1866 NodeType: plan.Node_PROJECT, 1867 ProjectList: ctx.results, 1868 Children: []int32{lastNodeID}, 1869 BindingTags: []int32{ctx.resultTag}, 1870 }, ctx) 1871 } else { 1872 ctx.results = ctx.projects 1873 } 1874 1875 if ctx.initSelect { 1876 lastNodeID = appendSinkNodeWithTag(builder, ctx, lastNodeID, ctx.sinkTag) 1877 } 1878 1879 // set heading 1880 if isRoot { 1881 builder.qry.Headings = append(builder.qry.Headings, ctx.headings...) 1882 } 1883 1884 return lastNodeID, nil 1885 } 1886 1887 const NameGroupConcat = "group_concat" 1888 const NameClusterCenters = "cluster_centers" 1889 1890 func (bc *BindContext) generateForceWinSpecList() ([]*plan.Expr, error) { 1891 windowsSpecList := make([]*plan.Expr, 0, len(bc.aggregates)) 1892 j := 0 1893 1894 if len(bc.windows) < 1 { 1895 panic("no winspeclist to be used to force") 1896 } 1897 1898 for i := range bc.aggregates { 1899 windowExpr := DeepCopyExpr(bc.windows[j]) 1900 windowSpec := windowExpr.GetW() 1901 if windowSpec == nil { 1902 panic("no winspeclist to be used to force") 1903 } 1904 windowSpec.WindowFunc = DeepCopyExpr(bc.aggregates[i]) 1905 windowExpr.Typ = bc.aggregates[i].Typ 1906 windowSpec.Name = bc.aggregates[i].GetF().Func.ObjName 1907 1908 if windowSpec.Name == NameGroupConcat { 1909 if j < len(bc.windows)-1 { 1910 j++ 1911 } 1912 // NOTE: no need to include NameClusterCenters 1913 } else { 1914 windowSpec.OrderBy = nil 1915 } 1916 windowsSpecList = append(windowsSpecList, windowExpr) 1917 } 1918 1919 return windowsSpecList, nil 1920 } 1921 1922 func (builder *QueryBuilder) buildSelect(stmt *tree.Select, ctx *BindContext, isRoot bool) (int32, error) { 1923 // preprocess CTEs 1924 if stmt.With != nil { 1925 ctx.cteByName = make(map[string]*CTERef) 1926 maskedNames := make([]string, len(stmt.With.CTEs)) 1927 1928 for i := range stmt.With.CTEs { 1929 idx := len(stmt.With.CTEs) - i - 1 1930 cte := stmt.With.CTEs[idx] 1931 1932 name := string(cte.Name.Alias) 1933 if _, ok := ctx.cteByName[name]; ok { 1934 return 0, moerr.NewSyntaxError(builder.GetContext(), "WITH query name %q specified more than once", name) 1935 } 1936 1937 var maskedCTEs map[string]bool 1938 if len(maskedNames) > 0 { 1939 maskedCTEs = make(map[string]bool) 1940 for _, mask := range maskedNames { 1941 maskedCTEs[mask] = true 1942 } 1943 } 1944 1945 maskedNames = append(maskedNames, name) 1946 1947 ctx.cteByName[name] = &CTERef{ 1948 ast: cte, 1949 isRecursive: stmt.With.IsRecursive, 1950 maskedCTEs: maskedCTEs, 1951 } 1952 } 1953 1954 // Try to do binding for CTE at declaration 1955 for _, cte := range stmt.With.CTEs { 1956 1957 table := string(cte.Name.Alias) 1958 cteRef := ctx.cteByName[table] 1959 1960 var err error 1961 var s *tree.Select 1962 switch stmt := cte.Stmt.(type) { 1963 case *tree.Select: 1964 s = stmt 1965 1966 case *tree.ParenSelect: 1967 s = stmt.Select 1968 1969 default: 1970 return 0, moerr.NewParseError(builder.GetContext(), "unexpected statement: '%v'", tree.String(stmt, dialect.MYSQL)) 1971 } 1972 1973 var left *tree.SelectStatement 1974 var stmts []tree.SelectStatement 1975 left, err = builder.splitRecursiveMember(&s.Select, table, &stmts) 1976 if err != nil { 1977 return 0, err 1978 } 1979 isR := len(stmts) > 0 1980 1981 if isR && !cteRef.isRecursive { 1982 return 0, moerr.NewParseError(builder.GetContext(), "not declare RECURSIVE: '%v'", tree.String(stmt, dialect.MYSQL)) 1983 } else if !isR { 1984 subCtx := NewBindContext(builder, ctx) 1985 subCtx.normalCTE = true 1986 subCtx.cteName = table 1987 subCtx.maskedCTEs = cteRef.maskedCTEs 1988 cteRef.isRecursive = false 1989 1990 oldSnapshot := builder.compCtx.GetSnapshot() 1991 builder.compCtx.SetSnapshot(subCtx.snapshot) 1992 nodeID, err := builder.buildSelect(s, subCtx, false) 1993 builder.compCtx.SetSnapshot(oldSnapshot) 1994 if err != nil { 1995 return 0, err 1996 } 1997 if len(cteRef.ast.Name.Cols) > 0 && len(cteRef.ast.Name.Cols) != len(builder.qry.Nodes[nodeID].ProjectList) { 1998 return 0, moerr.NewSyntaxError(builder.GetContext(), "table %q has %d columns available but %d columns specified", table, len(builder.qry.Nodes[nodeID].ProjectList), len(cteRef.ast.Name.Cols)) 1999 } 2000 ctx.views = append(ctx.views, subCtx.views...) 2001 } else { 2002 initCtx := NewBindContext(builder, ctx) 2003 initCtx.initSelect = true 2004 initCtx.sinkTag = builder.genNewTag() 2005 initCtx.isTryBindingCTE = true 2006 initLastNodeID, err := builder.buildSelect(&tree.Select{Select: *left}, initCtx, false) 2007 if err != nil { 2008 return 0, err 2009 } 2010 if len(cteRef.ast.Name.Cols) > 0 && len(cteRef.ast.Name.Cols) != len(builder.qry.Nodes[initLastNodeID].ProjectList) { 2011 return 0, moerr.NewSyntaxError(builder.GetContext(), "table %q has %d columns available but %d columns specified", table, len(builder.qry.Nodes[initLastNodeID].ProjectList), len(cteRef.ast.Name.Cols)) 2012 } 2013 //ctx.views = append(ctx.views, initCtx.views...) 2014 2015 recursiveNodeId := initLastNodeID 2016 for _, r := range stmts { 2017 subCtx := NewBindContext(builder, ctx) 2018 subCtx.maskedCTEs = cteRef.maskedCTEs 2019 subCtx.recSelect = true 2020 subCtx.sinkTag = builder.genNewTag() 2021 subCtx.isTryBindingCTE = true 2022 subCtx.cteByName = make(map[string]*CTERef) 2023 subCtx.cteByName[table] = cteRef 2024 err = builder.addBinding(initLastNodeID, *cteRef.ast.Name, subCtx) 2025 if err != nil { 2026 return 0, err 2027 } 2028 sourceStep := builder.appendStep(recursiveNodeId) 2029 nodeID := appendRecursiveScanNode(builder, subCtx, sourceStep, subCtx.sinkTag) 2030 subCtx.recRecursiveScanNodeId = nodeID 2031 recursiveNodeId, err = builder.buildSelect(&tree.Select{Select: r}, subCtx, false) 2032 if err != nil { 2033 return 0, err 2034 } 2035 //ctx.views = append(ctx.views, subCtx.views...) 2036 } 2037 builder.qry.Steps = builder.qry.Steps[:0] 2038 } 2039 } 2040 } 2041 2042 var clause *tree.SelectClause 2043 var valuesClause *tree.ValuesClause 2044 var nodeID int32 2045 var err error 2046 astOrderBy := stmt.OrderBy 2047 astLimit := stmt.Limit 2048 astTimeWindow := stmt.TimeWindow 2049 2050 if stmt.SelectLockInfo != nil && stmt.SelectLockInfo.LockType == tree.SelectLockForUpdate { 2051 builder.isForUpdate = true 2052 } 2053 2054 // strip parentheses 2055 // ((select a from t1)) order by b [ is equal ] select a from t1 order by b 2056 // (((select a from t1)) order by b) [ is equal ] select a from t1 order by b 2057 // 2058 // (select a from t1 union/union all select aa from t2) order by a 2059 // => we will strip parentheses, but order by only can use 'a' column from the union's output projectlist 2060 for { 2061 if selectClause, ok := stmt.Select.(*tree.ParenSelect); ok { 2062 if selectClause.Select.OrderBy != nil { 2063 if astOrderBy != nil { 2064 return 0, moerr.NewSyntaxError(builder.GetContext(), "multiple ORDER BY clauses not allowed") 2065 } 2066 astOrderBy = selectClause.Select.OrderBy 2067 } 2068 if selectClause.Select.Limit != nil { 2069 if astLimit != nil { 2070 return 0, moerr.NewSyntaxError(builder.GetContext(), "multiple LIMIT clauses not allowed") 2071 } 2072 astLimit = selectClause.Select.Limit 2073 } 2074 stmt = selectClause.Select 2075 } else { 2076 break 2077 } 2078 } 2079 2080 switch selectClause := stmt.Select.(type) { 2081 case *tree.SelectClause: 2082 clause = selectClause 2083 case *tree.UnionClause: 2084 if builder.isForUpdate { 2085 return 0, moerr.NewInternalError(builder.GetContext(), "not support select union for update") 2086 } 2087 return builder.buildUnion(selectClause, astOrderBy, astLimit, ctx, isRoot) 2088 case *tree.ValuesClause: 2089 valuesClause = selectClause 2090 default: 2091 return 0, moerr.NewNYI(builder.GetContext(), "statement '%s'", tree.String(stmt, dialect.MYSQL)) 2092 } 2093 2094 var projectionBinder *ProjectionBinder 2095 var havingList []*plan.Expr 2096 var selectList tree.SelectExprs 2097 var resultLen int 2098 var havingBinder *HavingBinder 2099 var lockNode *plan.Node 2100 var notCacheable bool 2101 2102 if clause == nil { 2103 proc := builder.compCtx.GetProcess() 2104 rowCount := len(valuesClause.Rows) 2105 if len(valuesClause.Rows) == 0 { 2106 return 0, moerr.NewInternalError(builder.GetContext(), "values statement have not rows") 2107 } 2108 bat := batch.NewWithSize(len(valuesClause.Rows[0])) 2109 strTyp := plan.Type{ 2110 Id: int32(types.T_text), 2111 NotNullable: false, 2112 } 2113 strColTyp := makeTypeByPlan2Type(strTyp) 2114 strColTargetTyp := &plan.Expr{ 2115 Typ: strTyp, 2116 Expr: &plan.Expr_T{ 2117 T: &plan.TargetType{}, 2118 }, 2119 } 2120 colCount := len(valuesClause.Rows[0]) 2121 for j := 1; j < rowCount; j++ { 2122 if len(valuesClause.Rows[j]) != colCount { 2123 return 0, moerr.NewInternalError(builder.GetContext(), fmt.Sprintf("have different column count in row '%v'", j)) 2124 } 2125 } 2126 2127 ctx.hasSingleRow = rowCount == 1 2128 rowSetData := &plan.RowsetData{ 2129 Cols: make([]*plan.ColData, colCount), 2130 } 2131 tableDef := &plan.TableDef{ 2132 TblId: 0, 2133 Name: "", 2134 Cols: make([]*plan.ColDef, colCount), 2135 } 2136 ctx.binder = NewWhereBinder(builder, ctx) 2137 for i := 0; i < colCount; i++ { 2138 vec := proc.GetVector(types.T_text.ToType()) 2139 bat.Vecs[i] = vec 2140 rowSetData.Cols[i] = &plan.ColData{} 2141 for j := 0; j < rowCount; j++ { 2142 if err := vector.AppendBytes(vec, nil, true, proc.Mp()); err != nil { 2143 bat.Clean(proc.Mp()) 2144 return 0, err 2145 } 2146 planExpr, err := ctx.binder.BindExpr(valuesClause.Rows[j][i], 0, true) 2147 if err != nil { 2148 return 0, err 2149 } 2150 planExpr, err = forceCastExpr2(builder.GetContext(), planExpr, strColTyp, strColTargetTyp) 2151 if err != nil { 2152 return 0, err 2153 } 2154 rowSetData.Cols[i].Data = append(rowSetData.Cols[i].Data, &plan.RowsetExpr{ 2155 RowPos: int32(j), 2156 Expr: planExpr, 2157 Pos: -1, 2158 }) 2159 } 2160 2161 colName := fmt.Sprintf("column_%d", i) // like MySQL 2162 as := tree.NewCStr(colName, 0) 2163 selectList = append(selectList, tree.SelectExpr{ 2164 Expr: &tree.UnresolvedName{ 2165 NumParts: 1, 2166 Star: false, 2167 Parts: [4]string{colName, "", "", ""}, 2168 }, 2169 As: as, 2170 }) 2171 ctx.headings = append(ctx.headings, colName) 2172 tableDef.Cols[i] = &plan.ColDef{ 2173 ColId: 0, 2174 Name: colName, 2175 Typ: strTyp, 2176 } 2177 } 2178 bat.SetRowCount(rowCount) 2179 nodeUUID, _ := uuid.NewV7() 2180 nodeID = builder.appendNode(&plan.Node{ 2181 NodeType: plan.Node_VALUE_SCAN, 2182 RowsetData: rowSetData, 2183 TableDef: tableDef, 2184 BindingTags: []int32{builder.genNewTag()}, 2185 Uuid: nodeUUID[:], 2186 NotCacheable: true, 2187 }, ctx) 2188 if builder.isPrepareStatement { 2189 proc.SetPrepareBatch(bat) 2190 } else { 2191 proc.SetValueScanBatch(nodeUUID, bat) 2192 } 2193 2194 err = builder.addBinding(nodeID, tree.AliasClause{ 2195 Alias: "_ValueScan", 2196 }, ctx) 2197 if err != nil { 2198 return 0, err 2199 } 2200 } else { 2201 if ctx.recSelect && clause.Distinct { 2202 return 0, moerr.NewParseError(builder.GetContext(), "not support DISTINCT in recursive cte") 2203 } 2204 // build FROM clause 2205 nodeID, err = builder.buildFrom(clause.From.Tables, ctx) 2206 if err != nil { 2207 return 0, err 2208 } 2209 2210 ctx.binder = NewWhereBinder(builder, ctx) 2211 if !ctx.isTryBindingCTE { 2212 if ctx.initSelect { 2213 clause.Exprs = append(clause.Exprs, makeZeroRecursiveLevel()) 2214 } else if ctx.recSelect { 2215 clause.Exprs = append(clause.Exprs, makePlusRecursiveLevel(ctx.cteName)) 2216 } 2217 } 2218 // unfold stars and generate headings 2219 selectList, err = appendSelectList(builder, ctx, selectList, clause.Exprs...) 2220 if err != nil { 2221 return 0, err 2222 } 2223 2224 if len(selectList) == 0 { 2225 return 0, moerr.NewParseError(builder.GetContext(), "No tables used") 2226 } 2227 2228 if builder.isForUpdate { 2229 tableDef := builder.qry.Nodes[nodeID].GetTableDef() 2230 pkPos, pkTyp := getPkPos(tableDef, false) 2231 lockTarget := &plan.LockTarget{ 2232 TableId: tableDef.TblId, 2233 PrimaryColIdxInBat: int32(pkPos), 2234 PrimaryColTyp: pkTyp, 2235 Block: true, 2236 RefreshTsIdxInBat: -1, //unsupport now 2237 FilterColIdxInBat: -1, //unsupport now 2238 } 2239 lockNode = &Node{ 2240 NodeType: plan.Node_LOCK_OP, 2241 Children: []int32{nodeID}, 2242 TableDef: tableDef, 2243 LockTargets: []*plan.LockTarget{lockTarget}, 2244 BindingTags: []int32{builder.genNewTag(), builder.qry.Nodes[nodeID].BindingTags[0]}, 2245 } 2246 2247 if astLimit == nil { 2248 nodeID = builder.appendNode(lockNode, ctx) 2249 } 2250 } 2251 2252 // rewrite right join to left join 2253 builder.rewriteRightJoinToLeftJoin(nodeID) 2254 2255 if !ctx.isTryBindingCTE && ctx.recSelect { 2256 f := &tree.FuncExpr{ 2257 Func: tree.FuncName2ResolvableFunctionReference(tree.SetUnresolvedName(moCheckRecursionLevelFun)), 2258 Exprs: tree.Exprs{tree.NewComparisonExpr(tree.LESS_THAN, tree.SetUnresolvedName(ctx.cteName, moRecursiveLevelCol), tree.NewNumValWithType(constant.MakeInt64(moDefaultRecursionMax), fmt.Sprintf("%d", moDefaultRecursionMax), false, tree.P_int64))}, 2259 } 2260 if clause.Where != nil { 2261 clause.Where = &tree.Where{Type: tree.AstWhere, Expr: tree.NewAndExpr(clause.Where.Expr, f)} 2262 } else { 2263 clause.Where = &tree.Where{Type: tree.AstWhere, Expr: f} 2264 } 2265 } 2266 if clause.Where != nil { 2267 whereList, err := splitAndBindCondition(clause.Where.Expr, NoAlias, ctx) 2268 if err != nil { 2269 return 0, err 2270 } 2271 2272 var newFilterList []*plan.Expr 2273 var expr *plan.Expr 2274 2275 for _, cond := range whereList { 2276 nodeID, expr, err = builder.flattenSubqueries(nodeID, cond, ctx) 2277 if err != nil { 2278 return 0, err 2279 } 2280 2281 newFilterList = append(newFilterList, expr) 2282 } 2283 2284 for _, filter := range newFilterList { 2285 if detectedExprWhetherTimeRelated(filter) { 2286 notCacheable = true 2287 } 2288 } 2289 2290 nodeID = builder.appendNode(&plan.Node{ 2291 NodeType: plan.Node_FILTER, 2292 Children: []int32{nodeID}, 2293 FilterList: newFilterList, 2294 NotCacheable: notCacheable, 2295 }, ctx) 2296 } 2297 2298 ctx.groupTag = builder.genNewTag() 2299 ctx.aggregateTag = builder.genNewTag() 2300 ctx.projectTag = builder.genNewTag() 2301 ctx.windowTag = builder.genNewTag() 2302 ctx.sampleTag = builder.genNewTag() 2303 if astTimeWindow != nil { 2304 ctx.timeTag = builder.genNewTag() // ctx.timeTag > 0 2305 } 2306 2307 // Preprocess aliases 2308 for i := range selectList { 2309 selectList[i].Expr, err = ctx.qualifyColumnNames(selectList[i].Expr, NoAlias) 2310 if err != nil { 2311 return 0, err 2312 } 2313 2314 builder.nameByColRef[[2]int32{ctx.projectTag, int32(i)}] = tree.String(selectList[i].Expr, dialect.MYSQL) 2315 2316 if selectList[i].As != nil && !selectList[i].As.Empty() { 2317 ctx.aliasMap[selectList[i].As.ToLower()] = &aliasItem{ 2318 idx: int32(i), 2319 astExpr: selectList[i].Expr, 2320 } 2321 } 2322 } 2323 2324 // bind GROUP BY clause 2325 if clause.GroupBy != nil { 2326 if ctx.recSelect { 2327 return 0, moerr.NewParseError(builder.GetContext(), "not support group by in recursive cte: '%v'", tree.String(&clause.GroupBy, dialect.MYSQL)) 2328 } 2329 groupBinder := NewGroupBinder(builder, ctx) 2330 for _, group := range clause.GroupBy { 2331 group, err = ctx.qualifyColumnNames(group, AliasAfterColumn) 2332 if err != nil { 2333 return 0, err 2334 } 2335 2336 _, err = groupBinder.BindExpr(group, 0, true) 2337 if err != nil { 2338 return 0, err 2339 } 2340 } 2341 } 2342 2343 // bind HAVING clause 2344 havingBinder = NewHavingBinder(builder, ctx) 2345 if clause.Having != nil { 2346 if ctx.recSelect { 2347 return 0, moerr.NewParseError(builder.GetContext(), "not support having in recursive cte: '%v'", tree.String(clause.Having, dialect.MYSQL)) 2348 } 2349 ctx.binder = havingBinder 2350 havingList, err = splitAndBindCondition(clause.Having.Expr, AliasAfterColumn, ctx) 2351 if err != nil { 2352 return 0, err 2353 } 2354 } 2355 2356 ctx.isDistinct = clause.Distinct 2357 } 2358 2359 // bind SELECT clause (Projection List) 2360 projectionBinder = NewProjectionBinder(builder, ctx, havingBinder) 2361 if err = bindProjectionList(ctx, projectionBinder, selectList); err != nil { 2362 return 0, err 2363 } 2364 2365 resultLen = len(ctx.projects) 2366 for i, proj := range ctx.projects { 2367 exprStr := proj.String() 2368 if _, ok := ctx.projectByExpr[exprStr]; !ok { 2369 ctx.projectByExpr[exprStr] = int32(i) 2370 } 2371 2372 if exprCol, ok := proj.Expr.(*plan.Expr_Col); ok { 2373 if col := exprCol.Col; col != nil { 2374 if binding, ok := ctx.bindingByTag[col.RelPos]; ok { 2375 col.DbName = binding.db 2376 col.TblName = binding.table 2377 } 2378 } 2379 } 2380 2381 if !notCacheable { 2382 if detectedExprWhetherTimeRelated(proj) { 2383 notCacheable = true 2384 } 2385 } 2386 } 2387 2388 // bind TIME WINDOW 2389 var fillType plan.Node_FillType 2390 var fillVals, fillCols []*Expr 2391 var inteval, sliding *Expr 2392 var orderBy *plan.OrderBySpec 2393 if astTimeWindow != nil { 2394 h := projectionBinder.havingBinder 2395 col, err := ctx.qualifyColumnNames(astTimeWindow.Interval.Col, NoAlias) 2396 if err != nil { 2397 return 0, err 2398 } 2399 r := col.(*tree.UnresolvedName) 2400 table := r.Parts[1] 2401 schema := ctx.defaultDatabase 2402 if len(r.Parts[2]) > 0 { 2403 schema = r.Parts[2] 2404 } 2405 2406 // snapshot to fix 2407 pk := builder.compCtx.GetPrimaryKeyDef(schema, table, Snapshot{TS: ×tamp.Timestamp{}}) 2408 if len(pk) > 1 || pk[0].Name != r.Parts[0] { 2409 return 0, moerr.NewNotSupported(builder.GetContext(), "%s is not primary key in time window", tree.String(col, dialect.MYSQL)) 2410 } 2411 h.insideAgg = true 2412 expr, err := h.BindExpr(col, 0, true) 2413 if err != nil { 2414 return 0, err 2415 } 2416 h.insideAgg = false 2417 if expr.Typ.Id != int32(types.T_timestamp) { 2418 return 0, moerr.NewNotSupported(builder.GetContext(), "the type of %s must be timestamp in time window", tree.String(col, dialect.MYSQL)) 2419 } 2420 orderBy = &plan.OrderBySpec{ 2421 Expr: expr, 2422 Flag: plan.OrderBySpec_INTERNAL | plan.OrderBySpec_ASC | plan.OrderBySpec_NULLS_FIRST, 2423 } 2424 2425 name := tree.SetUnresolvedName("interval") 2426 arg2 := tree.NewNumValWithType(constant.MakeString(astTimeWindow.Interval.Unit), astTimeWindow.Interval.Unit, false, tree.P_char) 2427 itr := &tree.FuncExpr{ 2428 Func: tree.FuncName2ResolvableFunctionReference(name), 2429 Exprs: tree.Exprs{astTimeWindow.Interval.Val, arg2}, 2430 } 2431 inteval, err = projectionBinder.BindExpr(itr, 0, true) 2432 if err != nil { 2433 return 0, err 2434 } 2435 2436 if astTimeWindow.Sliding != nil { 2437 arg2 = tree.NewNumValWithType(constant.MakeString(astTimeWindow.Sliding.Unit), astTimeWindow.Sliding.Unit, false, tree.P_char) 2438 sld := &tree.FuncExpr{ 2439 Func: tree.FuncName2ResolvableFunctionReference(name), 2440 Exprs: tree.Exprs{astTimeWindow.Sliding.Val, arg2}, 2441 } 2442 sliding, err = projectionBinder.BindExpr(sld, 0, true) 2443 if err != nil { 2444 return 0, err 2445 } 2446 } 2447 2448 if astTimeWindow.Fill != nil && astTimeWindow.Fill.Mode != tree.FillNone && astTimeWindow.Fill.Mode != tree.FillNull { 2449 switch astTimeWindow.Fill.Mode { 2450 case tree.FillPrev: 2451 fillType = plan.Node_PREV 2452 case tree.FillNext: 2453 fillType = plan.Node_NEXT 2454 case tree.FillValue: 2455 fillType = plan.Node_VALUE 2456 case tree.FillLinear: 2457 fillType = plan.Node_LINEAR 2458 } 2459 var v *Expr 2460 if astTimeWindow.Fill.Val != nil { 2461 v, err = projectionBinder.BindExpr(astTimeWindow.Fill.Val, 0, true) 2462 if err != nil { 2463 return 0, err 2464 } 2465 } 2466 2467 for _, t := range ctx.times { 2468 if e, ok := t.Expr.(*plan.Expr_Col); ok { 2469 if e.Col.Name == TimeWindowStart || e.Col.Name == TimeWindowEnd { 2470 continue 2471 } 2472 } 2473 if astTimeWindow.Fill.Val != nil { 2474 e, err := appendCastBeforeExpr(builder.GetContext(), v, t.Typ) 2475 if err != nil { 2476 return 0, err 2477 } 2478 fillVals = append(fillVals, e) 2479 } 2480 fillCols = append(fillCols, t) 2481 } 2482 if astTimeWindow.Fill.Mode == tree.FillLinear { 2483 for i, e := range ctx.timeAsts { 2484 b := &tree.BinaryExpr{ 2485 Op: tree.DIV, 2486 Left: &tree.ParenExpr{ 2487 Expr: &tree.BinaryExpr{ 2488 Op: tree.PLUS, 2489 Left: e, 2490 Right: e, 2491 }, 2492 }, 2493 Right: tree.NewNumValWithType(constant.MakeInt64(2), "2", false, tree.P_int64), 2494 } 2495 v, err = projectionBinder.BindExpr(b, 0, true) 2496 if err != nil { 2497 return 0, err 2498 } 2499 col, err := appendCastBeforeExpr(builder.GetContext(), v, fillCols[i].Typ) 2500 if err != nil { 2501 return 0, err 2502 } 2503 fillVals = append(fillVals, col) 2504 } 2505 } 2506 } 2507 } 2508 2509 // bind ORDER BY clause 2510 var orderBys []*plan.OrderBySpec 2511 if astOrderBy != nil { 2512 if ctx.recSelect { 2513 return 0, moerr.NewParseError(builder.GetContext(), "not support order by in recursive cte: '%v'", tree.String(&astOrderBy, dialect.MYSQL)) 2514 } 2515 orderBinder := NewOrderBinder(projectionBinder, selectList) 2516 orderBys = make([]*plan.OrderBySpec, 0, len(astOrderBy)) 2517 2518 for _, order := range astOrderBy { 2519 expr, err := orderBinder.BindExpr(order.Expr) 2520 if err != nil { 2521 return 0, err 2522 } 2523 2524 orderBy := &plan.OrderBySpec{ 2525 Expr: expr, 2526 Flag: plan.OrderBySpec_INTERNAL, 2527 } 2528 2529 switch order.Direction { 2530 case tree.Ascending: 2531 orderBy.Flag |= plan.OrderBySpec_ASC 2532 case tree.Descending: 2533 orderBy.Flag |= plan.OrderBySpec_DESC 2534 } 2535 2536 switch order.NullsPosition { 2537 case tree.NullsFirst: 2538 orderBy.Flag |= plan.OrderBySpec_NULLS_FIRST 2539 case tree.NullsLast: 2540 orderBy.Flag |= plan.OrderBySpec_NULLS_LAST 2541 } 2542 2543 orderBys = append(orderBys, orderBy) 2544 } 2545 } 2546 2547 // bind limit/offset clause 2548 var limitExpr *Expr 2549 var offsetExpr *Expr 2550 if astLimit != nil { 2551 limitBinder := NewLimitBinder(builder, ctx) 2552 if astLimit.Offset != nil { 2553 offsetExpr, err = limitBinder.BindExpr(astLimit.Offset, 0, true) 2554 if err != nil { 2555 return 0, err 2556 } 2557 2558 if cExpr, ok := offsetExpr.Expr.(*plan.Expr_Lit); ok { 2559 if c, ok := cExpr.Lit.Value.(*plan.Literal_I64Val); ok { 2560 if c.I64Val < 0 { 2561 return 0, moerr.NewSyntaxError(builder.GetContext(), "offset value must be nonnegative") 2562 } 2563 } 2564 } 2565 } 2566 if astLimit.Count != nil { 2567 limitExpr, err = limitBinder.BindExpr(astLimit.Count, 0, true) 2568 if err != nil { 2569 return 0, err 2570 } 2571 2572 if cExpr, ok := limitExpr.Expr.(*plan.Expr_Lit); ok { 2573 if c, ok := cExpr.Lit.Value.(*plan.Literal_I64Val); ok { 2574 if c.I64Val < 0 { 2575 return 0, moerr.NewSyntaxError(builder.GetContext(), "limit value must be nonnegative") 2576 } 2577 ctx.hasSingleRow = c.I64Val == 1 2578 } 2579 } 2580 } 2581 if builder.isForUpdate { 2582 nodeID = builder.appendNode(lockNode, ctx) 2583 } 2584 } 2585 2586 // return err if it's not a legal SAMPLE function. 2587 if err = validSample(ctx, builder); err != nil { 2588 return 0, err 2589 } 2590 2591 // sample can ignore these check because it supports the sql like 'select a, b, sample(c) from t group by a' whose b is not in group by clause. 2592 if !ctx.sampleFunc.hasSampleFunc { 2593 if (len(ctx.groups) > 0 || len(ctx.aggregates) > 0 || len(ctx.times) > 0) && len(projectionBinder.boundCols) > 0 { 2594 if !builder.mysqlCompatible { 2595 return 0, moerr.NewSyntaxError(builder.GetContext(), "column %q must appear in the GROUP BY clause or be used in an aggregate function", projectionBinder.boundCols[0]) 2596 } 2597 2598 // For MySQL compatibility, wrap bare ColRefs in any_value() 2599 for i, proj := range ctx.projects { 2600 ctx.projects[i] = builder.wrapBareColRefsInAnyValue(proj, ctx) 2601 } 2602 } 2603 } 2604 2605 if ctx.forceWindows { 2606 ctx.tmpGroups = ctx.groups 2607 ctx.windows, _ = ctx.generateForceWinSpecList() 2608 ctx.aggregates = nil 2609 ctx.groups = nil 2610 2611 for i := range ctx.projects { 2612 ctx.projects[i] = DeepProcessExprForGroupConcat(ctx.projects[i], ctx) 2613 } 2614 2615 for i := range havingList { 2616 havingList[i] = DeepProcessExprForGroupConcat(havingList[i], ctx) 2617 } 2618 2619 for i := range orderBys { 2620 orderBys[i].Expr = DeepProcessExprForGroupConcat(orderBys[i].Expr, ctx) 2621 } 2622 2623 } 2624 2625 // FIXME: delete this when SINGLE join is ready 2626 if len(ctx.groups) == 0 && len(ctx.aggregates) > 0 { 2627 ctx.hasSingleRow = true 2628 } 2629 2630 // append AGG node or SAMPLE node 2631 if ctx.sampleFunc.hasSampleFunc { 2632 nodeID = builder.appendNode(generateSamplePlanNode(ctx, nodeID), ctx) 2633 2634 if len(havingList) > 0 { 2635 var newFilterList []*plan.Expr 2636 var expr *plan.Expr 2637 2638 for _, cond := range havingList { 2639 nodeID, expr, err = builder.flattenSubqueries(nodeID, cond, ctx) 2640 if err != nil { 2641 return 0, err 2642 } 2643 2644 newFilterList = append(newFilterList, expr) 2645 } 2646 2647 nodeID = builder.appendNode(&plan.Node{ 2648 NodeType: plan.Node_FILTER, 2649 Children: []int32{nodeID}, 2650 FilterList: newFilterList, 2651 }, ctx) 2652 } 2653 2654 for name, id := range ctx.groupByAst { 2655 builder.nameByColRef[[2]int32{ctx.groupTag, id}] = name 2656 } 2657 2658 for name, id := range ctx.sampleByAst { 2659 builder.nameByColRef[[2]int32{ctx.sampleTag, id}] = name 2660 } 2661 } else { 2662 if len(ctx.groups) > 0 || len(ctx.aggregates) > 0 { 2663 if ctx.recSelect { 2664 return 0, moerr.NewInternalError(builder.GetContext(), "not support aggregate function recursive cte") 2665 } 2666 if builder.isForUpdate { 2667 return 0, moerr.NewInternalError(builder.GetContext(), "not support select aggregate function for update") 2668 } 2669 if ctx.forceWindows { 2670 } else { 2671 nodeID = builder.appendNode(&plan.Node{ 2672 NodeType: plan.Node_AGG, 2673 Children: []int32{nodeID}, 2674 GroupBy: ctx.groups, 2675 AggList: ctx.aggregates, 2676 BindingTags: []int32{ctx.groupTag, ctx.aggregateTag}, 2677 }, ctx) 2678 } 2679 if len(havingList) > 0 { 2680 var newFilterList []*plan.Expr 2681 var expr *plan.Expr 2682 2683 for _, cond := range havingList { 2684 nodeID, expr, err = builder.flattenSubqueries(nodeID, cond, ctx) 2685 if err != nil { 2686 return 0, err 2687 } 2688 2689 newFilterList = append(newFilterList, expr) 2690 } 2691 2692 nodeID = builder.appendNode(&plan.Node{ 2693 NodeType: plan.Node_FILTER, 2694 Children: []int32{nodeID}, 2695 FilterList: newFilterList, 2696 }, ctx) 2697 } 2698 2699 for name, id := range ctx.groupByAst { 2700 builder.nameByColRef[[2]int32{ctx.groupTag, id}] = name 2701 } 2702 2703 for name, id := range ctx.aggregateByAst { 2704 builder.nameByColRef[[2]int32{ctx.aggregateTag, id}] = name 2705 } 2706 } 2707 } 2708 2709 // append TIME WINDOW node 2710 if len(ctx.times) > 0 { 2711 if ctx.recSelect { 2712 return 0, moerr.NewInternalError(builder.GetContext(), "not support time window in recursive cte") 2713 } 2714 nodeID = builder.appendNode(&plan.Node{ 2715 NodeType: plan.Node_TIME_WINDOW, 2716 Children: []int32{nodeID}, 2717 AggList: ctx.times, 2718 BindingTags: []int32{ctx.timeTag}, 2719 OrderBy: []*plan.OrderBySpec{orderBy}, 2720 Interval: inteval, 2721 Sliding: sliding, 2722 }, ctx) 2723 2724 for name, id := range ctx.timeByAst { 2725 builder.nameByColRef[[2]int32{ctx.timeTag, id}] = name 2726 } 2727 2728 if astTimeWindow.Fill != nil { 2729 nodeID = builder.appendNode(&plan.Node{ 2730 NodeType: plan.Node_FILL, 2731 Children: []int32{nodeID}, 2732 AggList: fillCols, 2733 BindingTags: []int32{ctx.timeTag}, 2734 FillType: fillType, 2735 FillVal: fillVals, 2736 }, ctx) 2737 } 2738 } 2739 2740 // append WINDOW node 2741 if len(ctx.windows) > 0 { 2742 if ctx.recSelect { 2743 return 0, moerr.NewInternalError(builder.GetContext(), "not support window function in recursive cte") 2744 } 2745 2746 for i, w := range ctx.windows { 2747 e := w.GetW() 2748 if len(e.PartitionBy) > 0 { 2749 partitionBy := make([]*plan.OrderBySpec, 0, len(e.PartitionBy)) 2750 for _, p := range e.PartitionBy { 2751 partitionBy = append(partitionBy, &plan.OrderBySpec{ 2752 Expr: p, 2753 Flag: plan.OrderBySpec_INTERNAL, 2754 }) 2755 } 2756 nodeID = builder.appendNode(&plan.Node{ 2757 NodeType: plan.Node_PARTITION, 2758 Children: []int32{nodeID}, 2759 OrderBy: partitionBy, 2760 BindingTags: []int32{ctx.windowTag}, 2761 }, ctx) 2762 } 2763 nodeID = builder.appendNode(&plan.Node{ 2764 NodeType: plan.Node_WINDOW, 2765 Children: []int32{nodeID}, 2766 WinSpecList: []*Expr{w}, 2767 WindowIdx: int32(i), 2768 BindingTags: []int32{ctx.windowTag}, 2769 }, ctx) 2770 } 2771 2772 for name, id := range ctx.windowByAst { 2773 builder.nameByColRef[[2]int32{ctx.windowTag, id}] = name 2774 } 2775 2776 if ctx.forceWindows { 2777 if len(havingList) > 0 { 2778 var newFilterList []*plan.Expr 2779 var expr *plan.Expr 2780 2781 for _, cond := range havingList { 2782 nodeID, expr, err = builder.flattenSubqueries(nodeID, cond, ctx) 2783 if err != nil { 2784 return 0, err 2785 } 2786 2787 newFilterList = append(newFilterList, expr) 2788 } 2789 2790 nodeID = builder.appendNode(&plan.Node{ 2791 NodeType: plan.Node_FILTER, 2792 Children: []int32{nodeID}, 2793 FilterList: newFilterList, 2794 }, ctx) 2795 } 2796 } 2797 2798 } 2799 2800 // append PROJECT node 2801 for i, proj := range ctx.projects { 2802 nodeID, proj, err = builder.flattenSubqueries(nodeID, proj, ctx) 2803 if err != nil { 2804 return 0, err 2805 } 2806 2807 ctx.projects[i] = proj 2808 } 2809 2810 nodeID = builder.appendNode(&plan.Node{ 2811 NodeType: plan.Node_PROJECT, 2812 ProjectList: ctx.projects, 2813 Children: []int32{nodeID}, 2814 BindingTags: []int32{ctx.projectTag}, 2815 NotCacheable: notCacheable, 2816 }, ctx) 2817 2818 // append DISTINCT node 2819 if ctx.isDistinct { 2820 nodeID = builder.appendNode(&plan.Node{ 2821 NodeType: plan.Node_DISTINCT, 2822 Children: []int32{nodeID}, 2823 }, ctx) 2824 } 2825 2826 // append SORT node (include limit, offset) 2827 if len(orderBys) > 0 { 2828 nodeID = builder.appendNode(&plan.Node{ 2829 NodeType: plan.Node_SORT, 2830 Children: []int32{nodeID}, 2831 OrderBy: orderBys, 2832 }, ctx) 2833 } 2834 2835 if limitExpr != nil || offsetExpr != nil { 2836 node := builder.qry.Nodes[nodeID] 2837 2838 node.Limit = limitExpr 2839 node.Offset = offsetExpr 2840 } 2841 2842 // append result PROJECT node 2843 if builder.qry.Nodes[nodeID].NodeType != plan.Node_PROJECT { 2844 for i := 0; i < resultLen; i++ { 2845 ctx.results = append(ctx.results, &plan.Expr{ 2846 Typ: ctx.projects[i].Typ, 2847 Expr: &plan.Expr_Col{ 2848 Col: &plan.ColRef{ 2849 RelPos: ctx.projectTag, 2850 ColPos: int32(i), 2851 }, 2852 }, 2853 }) 2854 } 2855 2856 ctx.resultTag = builder.genNewTag() 2857 2858 nodeID = builder.appendNode(&plan.Node{ 2859 NodeType: plan.Node_PROJECT, 2860 ProjectList: ctx.results, 2861 Children: []int32{nodeID}, 2862 BindingTags: []int32{ctx.resultTag}, 2863 }, ctx) 2864 } else { 2865 ctx.results = ctx.projects 2866 } 2867 2868 if (ctx.initSelect || ctx.recSelect) && !ctx.unionSelect { 2869 nodeID = appendSinkNodeWithTag(builder, ctx, nodeID, ctx.sinkTag) 2870 } 2871 2872 if isRoot { 2873 builder.qry.Headings = append(builder.qry.Headings, ctx.headings...) 2874 } 2875 2876 return nodeID, nil 2877 } 2878 2879 func DeepProcessExprForGroupConcat(expr *Expr, ctx *BindContext) *Expr { 2880 if expr == nil { 2881 return nil 2882 } 2883 switch item := expr.Expr.(type) { 2884 case *plan.Expr_Col: 2885 if item.Col.RelPos == ctx.groupTag { 2886 expr = DeepCopyExpr(ctx.tmpGroups[item.Col.ColPos]) 2887 } 2888 if item.Col.RelPos == ctx.aggregateTag { 2889 item.Col.RelPos = ctx.windowTag 2890 } 2891 2892 case *plan.Expr_F: 2893 for i, arg := range item.F.Args { 2894 item.F.Args[i] = DeepProcessExprForGroupConcat(arg, ctx) 2895 } 2896 case *plan.Expr_W: 2897 for i, p := range item.W.PartitionBy { 2898 item.W.PartitionBy[i] = DeepProcessExprForGroupConcat(p, ctx) 2899 } 2900 for i, o := range item.W.OrderBy { 2901 item.W.OrderBy[i].Expr = DeepProcessExprForGroupConcat(o.Expr, ctx) 2902 } 2903 2904 case *plan.Expr_Sub: 2905 DeepProcessExprForGroupConcat(item.Sub.Child, ctx) 2906 2907 case *plan.Expr_Corr: 2908 if item.Corr.RelPos == ctx.groupTag { 2909 expr = DeepCopyExpr(ctx.tmpGroups[item.Corr.ColPos]) 2910 } 2911 if item.Corr.RelPos == ctx.aggregateTag { 2912 item.Corr.RelPos = ctx.windowTag 2913 } 2914 case *plan.Expr_List: 2915 for i, ie := range item.List.List { 2916 item.List.List[i] = DeepProcessExprForGroupConcat(ie, ctx) 2917 } 2918 } 2919 return expr 2920 2921 } 2922 2923 func appendSelectList( 2924 builder *QueryBuilder, 2925 ctx *BindContext, 2926 selectList tree.SelectExprs, exprs ...tree.SelectExpr) (tree.SelectExprs, error) { 2927 accountId, err := builder.compCtx.GetAccountId() 2928 if err != nil { 2929 return nil, err 2930 } 2931 for _, selectExpr := range exprs { 2932 switch expr := selectExpr.Expr.(type) { 2933 case tree.UnqualifiedStar: 2934 cols, names, err := ctx.unfoldStar(builder.GetContext(), "", accountId == catalog.System_Account) 2935 if err != nil { 2936 return nil, err 2937 } 2938 for i, name := range names { 2939 if ctx.finalSelect && name == moRecursiveLevelCol { 2940 continue 2941 } 2942 selectList = append(selectList, cols[i]) 2943 ctx.headings = append(ctx.headings, name) 2944 } 2945 2946 case *tree.SampleExpr: 2947 var err error 2948 if err = ctx.sampleFunc.GenerateSampleFunc(expr); err != nil { 2949 return nil, err 2950 } 2951 2952 oldLen := len(selectList) 2953 columns, isStar := expr.GetColumns() 2954 if isStar { 2955 if selectList, err = appendSelectList(builder, ctx, selectList, tree.SelectExpr{Expr: tree.UnqualifiedStar{}}); err != nil { 2956 return nil, err 2957 } 2958 } else { 2959 for _, column := range columns { 2960 if selectList, err = appendSelectList(builder, ctx, selectList, tree.SelectExpr{Expr: column}); err != nil { 2961 return nil, err 2962 } 2963 } 2964 } 2965 2966 // deal with alias. 2967 sampleCount := len(selectList) - oldLen 2968 if selectExpr.As != nil && !selectExpr.As.Empty() { 2969 if sampleCount != 1 { 2970 return nil, moerr.NewSyntaxError(builder.GetContext(), "sample multi columns cannot have alias") 2971 } 2972 ctx.headings[len(ctx.headings)-1] = selectExpr.As.Origin() 2973 selectList[len(selectList)-1].As = selectExpr.As 2974 } 2975 2976 ctx.sampleFunc.SetStartOffset(oldLen, sampleCount) 2977 2978 case *tree.UnresolvedName: 2979 if expr.Star { 2980 cols, names, err := ctx.unfoldStar(builder.GetContext(), expr.Parts[0], accountId == catalog.System_Account) 2981 if err != nil { 2982 return nil, err 2983 } 2984 selectList = append(selectList, cols...) 2985 ctx.headings = append(ctx.headings, names...) 2986 } else { 2987 if selectExpr.As != nil && !selectExpr.As.Empty() { 2988 ctx.headings = append(ctx.headings, selectExpr.As.Origin()) 2989 } else if expr.CStrParts[0] != nil { 2990 ctx.headings = append(ctx.headings, expr.CStrParts[0].Compare()) 2991 } else { 2992 ctx.headings = append(ctx.headings, expr.Parts[0]) 2993 } 2994 2995 newExpr, err := ctx.qualifyColumnNames(expr, NoAlias) 2996 if err != nil { 2997 return nil, err 2998 } 2999 3000 selectList = append(selectList, tree.SelectExpr{ 3001 Expr: newExpr, 3002 As: selectExpr.As, 3003 }) 3004 } 3005 case *tree.NumVal: 3006 if expr.ValType == tree.P_null { 3007 expr.ValType = tree.P_nulltext 3008 } 3009 3010 if selectExpr.As != nil && !selectExpr.As.Empty() { 3011 ctx.headings = append(ctx.headings, selectExpr.As.Origin()) 3012 } else { 3013 ctx.headings = append(ctx.headings, tree.String(expr, dialect.MYSQL)) 3014 } 3015 3016 selectList = append(selectList, tree.SelectExpr{ 3017 Expr: expr, 3018 As: selectExpr.As, 3019 }) 3020 default: 3021 if selectExpr.As != nil && !selectExpr.As.Empty() { 3022 ctx.headings = append(ctx.headings, selectExpr.As.Origin()) 3023 } else { 3024 for { 3025 if parenExpr, ok := expr.(*tree.ParenExpr); ok { 3026 expr = parenExpr.Expr 3027 } else { 3028 break 3029 } 3030 } 3031 ctx.headings = append(ctx.headings, tree.String(expr, dialect.MYSQL)) 3032 } 3033 3034 newExpr, err := ctx.qualifyColumnNames(expr, NoAlias) 3035 if err != nil { 3036 return nil, err 3037 } 3038 3039 selectList = append(selectList, tree.SelectExpr{ 3040 Expr: newExpr, 3041 As: selectExpr.As, 3042 }) 3043 } 3044 } 3045 return selectList, nil 3046 } 3047 3048 func bindProjectionList( 3049 ctx *BindContext, 3050 projectionBinder *ProjectionBinder, 3051 selectList tree.SelectExprs) error { 3052 ctx.binder = projectionBinder 3053 3054 sampleRangeLeft, sampleRangeRight := ctx.sampleFunc.start, ctx.sampleFunc.start+ctx.sampleFunc.offset 3055 if ctx.sampleFunc.hasSampleFunc { 3056 // IF sample function exists, we should bind the sample column first. 3057 sampleList, err := ctx.sampleFunc.BindSampleColumn(ctx, projectionBinder, selectList[sampleRangeLeft:sampleRangeRight]) 3058 if err != nil { 3059 return err 3060 } 3061 3062 for i := range selectList[:sampleRangeLeft] { 3063 expr, err := projectionBinder.BindExpr(selectList[i].Expr, 0, true) 3064 if err != nil { 3065 return err 3066 } 3067 ctx.projects = append(ctx.projects, expr) 3068 } 3069 ctx.projects = append(ctx.projects, sampleList...) 3070 } 3071 3072 for i := range selectList[sampleRangeRight:] { 3073 expr, err := projectionBinder.BindExpr(selectList[i].Expr, 0, true) 3074 if err != nil { 3075 return err 3076 } 3077 3078 ctx.projects = append(ctx.projects, expr) 3079 } 3080 return nil 3081 } 3082 3083 func (builder *QueryBuilder) appendStep(nodeID int32) int32 { 3084 stepPos := len(builder.qry.Steps) 3085 builder.qry.Steps = append(builder.qry.Steps, nodeID) 3086 return int32(stepPos) 3087 } 3088 3089 func (builder *QueryBuilder) appendNode(node *plan.Node, ctx *BindContext) int32 { 3090 nodeID := int32(len(builder.qry.Nodes)) 3091 node.NodeId = nodeID 3092 builder.qry.Nodes = append(builder.qry.Nodes, node) 3093 builder.ctxByNode = append(builder.ctxByNode, ctx) 3094 ReCalcNodeStats(nodeID, builder, false, true, true) 3095 return nodeID 3096 } 3097 3098 func (builder *QueryBuilder) rewriteRightJoinToLeftJoin(nodeID int32) { 3099 node := builder.qry.Nodes[nodeID] 3100 3101 for i := range node.Children { 3102 builder.rewriteRightJoinToLeftJoin(node.Children[i]) 3103 } 3104 3105 if node.NodeType == plan.Node_JOIN && node.JoinType == plan.Node_RIGHT { 3106 node.JoinType = plan.Node_LEFT 3107 node.Children[0], node.Children[1] = node.Children[1], node.Children[0] 3108 } 3109 } 3110 3111 func (builder *QueryBuilder) buildFrom(stmt tree.TableExprs, ctx *BindContext) (int32, error) { 3112 if len(stmt) == 1 { 3113 return builder.buildTable(stmt[0], ctx, -1, nil) 3114 } 3115 return 0, moerr.NewInternalError(ctx.binder.GetContext(), "stmt's length should be zero") 3116 // for now, stmt'length always be zero. if someday that change in parser, you should uncomment these codes 3117 // leftCtx := NewBindContext(builder, ctx) 3118 // leftChildID, err := builder.buildTable(stmt[0], leftCtx) 3119 // if err != nil { 3120 // return 0, err 3121 // } 3122 3123 // for i := 1; i < len(stmt); i++ { 3124 // rightCtx := NewBindContext(builder, ctx) 3125 // rightChildID, err := builder.buildTable(stmt[i], rightCtx) 3126 // if err != nil { 3127 // return 0, err 3128 // } 3129 3130 // leftChildID = builder.appendNode(&plan.Node{ 3131 // NodeType: plan.Node_JOIN, 3132 // Children: []int32{leftChildID, rightChildID}, 3133 // JoinType: plan.Node_INNER, 3134 // }, nil) 3135 3136 // if i == len(stmt)-1 { 3137 // builder.ctxByNode[leftChildID] = ctx 3138 // err = ctx.mergeContexts(leftCtx, rightCtx) 3139 // if err != nil { 3140 // return 0, err 3141 // } 3142 // } else { 3143 // newCtx := NewBindContext(builder, ctx) 3144 // builder.ctxByNode[leftChildID] = newCtx 3145 // err = newCtx.mergeContexts(leftCtx, rightCtx) 3146 // if err != nil { 3147 // return 0, err 3148 // } 3149 // leftCtx = newCtx 3150 // } 3151 // } 3152 3153 // return leftChildID, err 3154 } 3155 3156 func (builder *QueryBuilder) splitRecursiveMember(stmt *tree.SelectStatement, name string, stmts *[]tree.SelectStatement) (*tree.SelectStatement, error) { 3157 ok, left, err := builder.checkRecursiveCTE(stmt, name, stmts) 3158 if !ok || err != nil { 3159 return left, err 3160 } 3161 return builder.splitRecursiveMember(left, name, stmts) 3162 } 3163 3164 func (builder *QueryBuilder) checkRecursiveCTE(left *tree.SelectStatement, name string, stmts *[]tree.SelectStatement) (bool, *tree.SelectStatement, error) { 3165 if u, ok := (*left).(*tree.UnionClause); ok { 3166 if !u.All { 3167 return false, left, nil 3168 } 3169 3170 rt, ok := u.Right.(*tree.SelectClause) 3171 if !ok { 3172 return false, left, nil 3173 } 3174 3175 count, err := builder.checkRecursiveTable(rt.From.Tables[0], name) 3176 if err != nil && count > 0 { 3177 return false, left, err 3178 } 3179 if count == 0 { 3180 return false, left, nil 3181 } 3182 if count > 1 { 3183 return false, left, moerr.NewParseError(builder.GetContext(), "unsupport multiple recursive table expr in recursive CTE: %T", left) 3184 } 3185 *stmts = append(*stmts, u.Right) 3186 return true, &u.Left, nil 3187 } 3188 return false, left, nil 3189 } 3190 3191 func (builder *QueryBuilder) checkRecursiveTable(stmt tree.TableExpr, name string) (int, error) { 3192 switch tbl := stmt.(type) { 3193 case *tree.Select: 3194 return 0, moerr.NewParseError(builder.GetContext(), "unsupport SUBQUERY in recursive CTE: %T", stmt) 3195 3196 case *tree.TableName: 3197 table := string(tbl.ObjectName) 3198 if table == name { 3199 return 1, nil 3200 } 3201 return 0, nil 3202 3203 case *tree.JoinTableExpr: 3204 var err, err0 error 3205 if tbl.JoinType == tree.JOIN_TYPE_LEFT || tbl.JoinType == tree.JOIN_TYPE_RIGHT || tbl.JoinType == tree.JOIN_TYPE_NATURAL_LEFT || tbl.JoinType == tree.JOIN_TYPE_NATURAL_RIGHT { 3206 err0 = moerr.NewParseError(builder.GetContext(), "unsupport LEFT, RIGHT or OUTER JOIN in recursive CTE: %T", stmt) 3207 } 3208 c1, err1 := builder.checkRecursiveTable(tbl.Left, name) 3209 c2, err2 := builder.checkRecursiveTable(tbl.Right, name) 3210 if err0 != nil { 3211 err = err0 3212 } else if err1 != nil { 3213 err = err1 3214 } else { 3215 err = err2 3216 } 3217 c := c1 + c2 3218 return c, err 3219 3220 case *tree.TableFunction: 3221 return 0, nil 3222 3223 case *tree.ParenTableExpr: 3224 return builder.checkRecursiveTable(tbl.Expr, name) 3225 3226 case *tree.AliasedTableExpr: 3227 return builder.checkRecursiveTable(tbl.Expr, name) 3228 3229 default: 3230 return 0, nil 3231 } 3232 } 3233 3234 func getSelectTree(s *tree.Select) *tree.Select { 3235 switch stmt := s.Select.(type) { 3236 case *tree.ParenSelect: 3237 return getSelectTree(stmt.Select) 3238 default: 3239 return s 3240 } 3241 } 3242 3243 func (builder *QueryBuilder) buildTable(stmt tree.TableExpr, ctx *BindContext, preNodeId int32, leftCtx *BindContext) (nodeID int32, err error) { 3244 switch tbl := stmt.(type) { 3245 case *tree.Select: 3246 if builder.isForUpdate { 3247 return 0, moerr.NewInternalError(builder.GetContext(), "not support select from derived table for update") 3248 } 3249 subCtx := NewBindContext(builder, ctx) 3250 nodeID, err = builder.buildSelect(tbl, subCtx, false) 3251 if subCtx.isCorrelated { 3252 return 0, moerr.NewNYI(builder.GetContext(), "correlated subquery in FROM clause") 3253 } 3254 3255 if subCtx.hasSingleRow { 3256 ctx.hasSingleRow = true 3257 } 3258 ctx.views = append(ctx.views, subCtx.views...) 3259 3260 case *tree.TableName: 3261 schema := string(tbl.SchemaName) 3262 table := string(tbl.ObjectName) 3263 if len(table) == 0 || table == "dual" { //special table name 3264 nodeID = builder.appendNode(&plan.Node{ 3265 NodeType: plan.Node_VALUE_SCAN, 3266 }, ctx) 3267 3268 ctx.hasSingleRow = true 3269 3270 break 3271 } 3272 3273 if len(schema) == 0 && ctx.normalCTE && table == ctx.cteName { 3274 return 0, moerr.NewParseError(builder.GetContext(), "In recursive query block of Recursive Common Table Expression %s, the recursive table must be referenced only once, and not in any subquery", table) 3275 } else if len(schema) == 0 { 3276 cteRef := ctx.findCTE(table) 3277 if cteRef != nil { 3278 if ctx.recSelect { 3279 nodeID = ctx.recRecursiveScanNodeId 3280 return 3281 } 3282 3283 var s *tree.Select 3284 switch stmt := cteRef.ast.Stmt.(type) { 3285 case *tree.Select: 3286 s = getSelectTree(stmt) 3287 case *tree.ParenSelect: 3288 s = getSelectTree(stmt.Select) 3289 default: 3290 err = moerr.NewParseError(builder.GetContext(), "unexpected statement: '%v'", tree.String(stmt, dialect.MYSQL)) 3291 return 3292 } 3293 3294 var left *tree.SelectStatement 3295 var stmts []tree.SelectStatement 3296 left, err = builder.splitRecursiveMember(&s.Select, table, &stmts) 3297 if err != nil { 3298 return 0, err 3299 } 3300 isR := len(stmts) > 0 3301 3302 if isR && !cteRef.isRecursive { 3303 err = moerr.NewParseError(builder.GetContext(), "not declare RECURSIVE: '%v'", tree.String(stmt, dialect.MYSQL)) 3304 } else if !isR { 3305 subCtx := NewBindContext(builder, ctx) 3306 subCtx.maskedCTEs = cteRef.maskedCTEs 3307 subCtx.cteName = table 3308 subCtx.snapshot = cteRef.snapshot 3309 //reset defaultDatabase 3310 if len(cteRef.defaultDatabase) > 0 { 3311 subCtx.defaultDatabase = cteRef.defaultDatabase 3312 } 3313 cteRef.isRecursive = false 3314 3315 oldSnapshot := builder.compCtx.GetSnapshot() 3316 builder.compCtx.SetSnapshot(subCtx.snapshot) 3317 nodeID, err = builder.buildSelect(s, subCtx, false) 3318 builder.compCtx.SetSnapshot(oldSnapshot) 3319 if err != nil { 3320 return 3321 } 3322 3323 if subCtx.hasSingleRow { 3324 ctx.hasSingleRow = true 3325 } 3326 ctx.views = append(ctx.views, subCtx.views...) 3327 3328 cols := cteRef.ast.Name.Cols 3329 3330 if len(cols) > len(subCtx.headings) { 3331 return 0, moerr.NewSyntaxError(builder.GetContext(), "table %q has %d columns available but %d columns specified", table, len(subCtx.headings), len(cols)) 3332 } 3333 3334 for i, col := range cols { 3335 subCtx.headings[i] = string(col) 3336 } 3337 } else { 3338 if len(s.OrderBy) > 0 { 3339 return 0, moerr.NewParseError(builder.GetContext(), "not support ORDER BY in recursive cte") 3340 } 3341 // initial statement 3342 initCtx := NewBindContext(builder, ctx) 3343 initCtx.initSelect = true 3344 initCtx.sinkTag = builder.genNewTag() 3345 initLastNodeID, err1 := builder.buildSelect(&tree.Select{Select: *left}, initCtx, false) 3346 if err1 != nil { 3347 err = err1 3348 return 3349 } 3350 projects := builder.qry.Nodes[builder.qry.Nodes[initLastNodeID].Children[0]].ProjectList 3351 // recursive statement 3352 recursiveLastNodeID := initLastNodeID 3353 initSourceStep := int32(len(builder.qry.Steps)) 3354 recursiveSteps := make([]int32, len(stmts)) 3355 recursiveNodeIDs := make([]int32, len(stmts)) 3356 if len(cteRef.ast.Name.Cols) > 0 { 3357 cteRef.ast.Name.Cols = append(cteRef.ast.Name.Cols, moRecursiveLevelCol) 3358 } 3359 3360 for i, r := range stmts { 3361 subCtx := NewBindContext(builder, ctx) 3362 subCtx.maskedCTEs = cteRef.maskedCTEs 3363 subCtx.cteName = table 3364 if len(cteRef.defaultDatabase) > 0 { 3365 subCtx.defaultDatabase = cteRef.defaultDatabase 3366 } 3367 subCtx.recSelect = true 3368 subCtx.sinkTag = initCtx.sinkTag 3369 subCtx.cteByName = make(map[string]*CTERef) 3370 subCtx.cteByName[table] = cteRef 3371 err = builder.addBinding(initLastNodeID, *cteRef.ast.Name, subCtx) 3372 if err != nil { 3373 return 3374 } 3375 _ = builder.appendStep(recursiveLastNodeID) 3376 subCtx.recRecursiveScanNodeId = appendRecursiveScanNode(builder, subCtx, initSourceStep, subCtx.sinkTag) 3377 recursiveNodeIDs[i] = subCtx.recRecursiveScanNodeId 3378 recursiveSteps[i] = int32(len(builder.qry.Steps)) 3379 recursiveLastNodeID, err = builder.buildSelect(&tree.Select{Select: r}, subCtx, false) 3380 if err != nil { 3381 return 3382 } 3383 // some check 3384 n := builder.qry.Nodes[builder.qry.Nodes[recursiveLastNodeID].Children[0]] 3385 if len(projects) != len(n.ProjectList) { 3386 return 0, moerr.NewParseError(builder.GetContext(), "recursive cte %s projection error", table) 3387 } 3388 for i := range n.ProjectList { 3389 projTyp := projects[i].GetTyp() 3390 n.ProjectList[i], err = makePlan2CastExpr(builder.GetContext(), n.ProjectList[i], projTyp) 3391 if err != nil { 3392 return 3393 } 3394 } 3395 if subCtx.hasSingleRow { 3396 ctx.hasSingleRow = true 3397 } 3398 3399 cols := cteRef.ast.Name.Cols 3400 3401 if len(cols) > len(subCtx.headings) { 3402 return 0, moerr.NewSyntaxError(builder.GetContext(), "table %q has %d columns available but %d columns specified", table, len(subCtx.headings), len(cols)) 3403 } 3404 3405 for i, col := range cols { 3406 subCtx.headings[i] = string(col) 3407 } 3408 } 3409 // union all statement 3410 var limitExpr *Expr 3411 var offsetExpr *Expr 3412 if s.Limit != nil { 3413 limitBinder := NewLimitBinder(builder, ctx) 3414 if s.Limit.Offset != nil { 3415 offsetExpr, err = limitBinder.BindExpr(s.Limit.Offset, 0, true) 3416 if err != nil { 3417 return 0, err 3418 } 3419 3420 if cExpr, ok := offsetExpr.Expr.(*plan.Expr_Lit); ok { 3421 if c, ok := cExpr.Lit.Value.(*plan.Literal_I64Val); ok { 3422 if c.I64Val < 0 { 3423 return 0, moerr.NewSyntaxError(builder.GetContext(), "offset value must be nonnegative") 3424 } 3425 } 3426 } 3427 } 3428 if s.Limit.Count != nil { 3429 limitExpr, err = limitBinder.BindExpr(s.Limit.Count, 0, true) 3430 if err != nil { 3431 return 0, err 3432 } 3433 3434 if cExpr, ok := limitExpr.Expr.(*plan.Expr_Lit); ok { 3435 if c, ok := cExpr.Lit.Value.(*plan.Literal_I64Val); ok { 3436 if c.I64Val < 0 { 3437 return 0, moerr.NewSyntaxError(builder.GetContext(), "limit value must be nonnegative") 3438 } 3439 ctx.hasSingleRow = c.I64Val == 1 3440 } 3441 } 3442 } 3443 } 3444 3445 _ = builder.appendStep(recursiveLastNodeID) 3446 nodeID = appendCTEScanNode(builder, ctx, initSourceStep, initCtx.sinkTag) 3447 if limitExpr != nil || offsetExpr != nil { 3448 node := builder.qry.Nodes[nodeID] 3449 node.Limit = limitExpr 3450 node.Offset = offsetExpr 3451 } 3452 for i := 0; i < len(recursiveSteps); i++ { 3453 builder.qry.Nodes[nodeID].SourceStep = append(builder.qry.Nodes[nodeID].SourceStep, recursiveSteps[i]) 3454 } 3455 curStep := int32(len(builder.qry.Steps)) 3456 for _, id := range recursiveNodeIDs { 3457 // builder.qry.Nodes[id].SourceStep = append(builder.qry.Nodes[id].SourceStep, curStep) 3458 builder.qry.Nodes[id].SourceStep[0] = curStep 3459 } 3460 unionAllLastNodeID := appendSinkNodeWithTag(builder, ctx, nodeID, ctx.sinkTag) 3461 builder.qry.Nodes[unionAllLastNodeID].RecursiveSink = true 3462 3463 // final statement 3464 ctx.finalSelect = true 3465 ctx.sinkTag = initCtx.sinkTag 3466 err = builder.addBinding(initLastNodeID, *cteRef.ast.Name, ctx) 3467 if err != nil { 3468 return 3469 } 3470 sourceStep := builder.appendStep(unionAllLastNodeID) 3471 nodeID = appendSinkScanNodeWithTag(builder, ctx, sourceStep, initCtx.sinkTag) 3472 // builder.qry.Nodes[nodeID].SourceStep = append(builder.qry.Nodes[nodeID].SourceStep, initSourceStep) 3473 } 3474 3475 break 3476 } 3477 schema = ctx.defaultDatabase 3478 } 3479 3480 if tbl.AtTsExpr != nil { 3481 ctx.snapshot, err = builder.resolveTsHint(tbl.AtTsExpr) 3482 if err != nil { 3483 return 0, err 3484 } 3485 } 3486 snapshot := ctx.snapshot 3487 if snapshot == nil { 3488 snapshot = &Snapshot{TS: ×tamp.Timestamp{}} 3489 } 3490 3491 // TODO 3492 schema, err = databaseIsValid(schema, builder.compCtx, *snapshot) 3493 if err != nil { 3494 return 0, err 3495 } 3496 3497 // TODO 3498 obj, tableDef := builder.compCtx.Resolve(schema, table, *snapshot) 3499 if tableDef == nil { 3500 return 0, moerr.NewParseError(builder.GetContext(), "table %q does not exist", table) 3501 } 3502 3503 tableDef.Name2ColIndex = map[string]int32{} 3504 for i := 0; i < len(tableDef.Cols); i++ { 3505 tableDef.Name2ColIndex[tableDef.Cols[i].Name] = int32(i) 3506 } 3507 nodeType := plan.Node_TABLE_SCAN 3508 if tableDef.TableType == catalog.SystemExternalRel { 3509 nodeType = plan.Node_EXTERNAL_SCAN 3510 col := &ColDef{ 3511 Name: catalog.ExternalFilePath, 3512 Typ: plan.Type{ 3513 Id: int32(types.T_varchar), 3514 Width: types.MaxVarcharLen, 3515 Table: table, 3516 }, 3517 } 3518 tableDef.Cols = append(tableDef.Cols, col) 3519 } else if tableDef.TableType == catalog.SystemSourceRel { 3520 nodeType = plan.Node_SOURCE_SCAN 3521 } else if tableDef.TableType == catalog.SystemViewRel { 3522 if yes, dbOfView, nameOfView := builder.compCtx.GetBuildingAlterView(); yes { 3523 currentDB := schema 3524 if currentDB == "" { 3525 currentDB = builder.compCtx.DefaultDatabase() 3526 } 3527 if dbOfView == currentDB && nameOfView == table { 3528 return 0, moerr.NewInternalError(builder.GetContext(), "there is a recursive reference to the view %s", nameOfView) 3529 } 3530 } 3531 // set view statment to CTE 3532 viewDefString := tableDef.ViewSql.View 3533 3534 if viewDefString != "" { 3535 if ctx.cteByName == nil { 3536 ctx.cteByName = make(map[string]*CTERef) 3537 } 3538 3539 viewData := ViewData{} 3540 err := json.Unmarshal([]byte(viewDefString), &viewData) 3541 if err != nil { 3542 return 0, err 3543 } 3544 3545 originStmts, err := mysql.Parse(builder.GetContext(), viewData.Stmt, 1, 0) 3546 defer func() { 3547 for _, s := range originStmts { 3548 s.Free() 3549 } 3550 }() 3551 if err != nil { 3552 return 0, err 3553 } 3554 viewStmt, ok := originStmts[0].(*tree.CreateView) 3555 3556 // No createview stmt, check alterview stmt. 3557 if !ok { 3558 alterstmt, ok := originStmts[0].(*tree.AlterView) 3559 viewStmt = &tree.CreateView{} 3560 if !ok { 3561 return 0, moerr.NewParseError(builder.GetContext(), "can not get view statement") 3562 } 3563 viewStmt.Name = alterstmt.Name 3564 viewStmt.ColNames = alterstmt.ColNames 3565 viewStmt.AsSource = alterstmt.AsSource 3566 } 3567 3568 viewName := viewStmt.Name.ObjectName 3569 var maskedCTEs map[string]bool 3570 if len(ctx.cteByName) > 0 { 3571 maskedCTEs = make(map[string]bool) 3572 for name := range ctx.cteByName { 3573 maskedCTEs[name] = true 3574 } 3575 } 3576 defaultDatabase := viewData.DefaultDatabase 3577 if obj.PubInfo != nil { 3578 defaultDatabase = obj.SubscriptionName 3579 } 3580 ctx.cteByName[string(viewName)] = &CTERef{ 3581 ast: &tree.CTE{ 3582 Name: &tree.AliasClause{ 3583 Alias: viewName, 3584 Cols: viewStmt.ColNames, 3585 }, 3586 Stmt: viewStmt.AsSource, 3587 }, 3588 defaultDatabase: defaultDatabase, 3589 maskedCTEs: maskedCTEs, 3590 snapshot: snapshot, 3591 } 3592 // consist with frontend.genKey() 3593 ctx.views = append(ctx.views, schema+"#"+table) 3594 3595 newTableName := tree.NewTableName(viewName, tree.ObjectNamePrefix{ 3596 CatalogName: tbl.CatalogName, // TODO unused now, if used in some code, that will be save in view 3597 SchemaName: tree.Identifier(""), 3598 ExplicitCatalog: false, 3599 ExplicitSchema: false, 3600 }, nil) 3601 return builder.buildTable(newTableName, ctx, preNodeId, leftCtx) 3602 } 3603 } 3604 3605 nodeID = builder.appendNode(&plan.Node{ 3606 NodeType: nodeType, 3607 Stats: nil, 3608 ObjRef: obj, 3609 TableDef: tableDef, 3610 BindingTags: []int32{builder.genNewTag()}, 3611 ScanSnapshot: snapshot, 3612 }, ctx) 3613 3614 case *tree.JoinTableExpr: 3615 if tbl.Right == nil { 3616 return builder.buildTable(tbl.Left, ctx, preNodeId, leftCtx) 3617 } else if builder.isForUpdate { 3618 return 0, moerr.NewInternalError(builder.GetContext(), "not support select from join table for update") 3619 } 3620 return builder.buildJoinTable(tbl, ctx) 3621 3622 case *tree.TableFunction: 3623 if tbl.Id() == "result_scan" { 3624 return builder.buildResultScan(tbl, ctx) 3625 } 3626 return builder.buildTableFunction(tbl, ctx, preNodeId, leftCtx) 3627 3628 case *tree.ParenTableExpr: 3629 return builder.buildTable(tbl.Expr, ctx, preNodeId, leftCtx) 3630 3631 case *tree.AliasedTableExpr: //allways AliasedTableExpr first 3632 if _, ok := tbl.Expr.(*tree.Select); ok { 3633 if tbl.As.Alias == "" { 3634 return 0, moerr.NewSyntaxError(builder.GetContext(), "subquery in FROM must have an alias: %T", stmt) 3635 } 3636 } 3637 3638 nodeID, err = builder.buildTable(tbl.Expr, ctx, preNodeId, leftCtx) 3639 if err != nil { 3640 return 3641 } 3642 3643 err = builder.addBinding(nodeID, tbl.As, ctx) 3644 3645 //tableDef := builder.qry.Nodes[nodeID].GetTableDef() 3646 midNode := builder.qry.Nodes[nodeID] 3647 //if it is the non-sys account and reads the cluster table, 3648 //we add an account_id filter to make sure that the non-sys account 3649 //can only read its own data. 3650 3651 if midNode.NodeType == plan.Node_TABLE_SCAN { 3652 dbName := midNode.ObjRef.SchemaName 3653 tableName := midNode.TableDef.Name 3654 currentAccountID, err := builder.compCtx.GetAccountId() 3655 if err != nil { 3656 return 0, err 3657 } 3658 acctName := builder.compCtx.GetUserName() 3659 if sub := builder.compCtx.GetQueryingSubscription(); sub != nil { 3660 currentAccountID = uint32(sub.AccountId) 3661 builder.qry.Nodes[nodeID].NotCacheable = true 3662 } 3663 if currentAccountID != catalog.System_Account { 3664 // add account filter for system table scan 3665 if dbName == catalog.MO_CATALOG && tableName == catalog.MO_DATABASE { 3666 modatabaseFilter := util.BuildMoDataBaseFilter(uint64(currentAccountID)) 3667 ctx.binder = NewWhereBinder(builder, ctx) 3668 accountFilterExprs, err := splitAndBindCondition(modatabaseFilter, NoAlias, ctx) 3669 if err != nil { 3670 return 0, err 3671 } 3672 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3673 } else if dbName == catalog.MO_SYSTEM_METRICS && (tableName == catalog.MO_METRIC || tableName == catalog.MO_SQL_STMT_CU) { 3674 motablesFilter := util.BuildSysMetricFilter(acctName) 3675 ctx.binder = NewWhereBinder(builder, ctx) 3676 accountFilterExprs, err := splitAndBindCondition(motablesFilter, NoAlias, ctx) 3677 if err != nil { 3678 return 0, err 3679 } 3680 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3681 } else if dbName == catalog.MO_SYSTEM && tableName == catalog.MO_STATEMENT { 3682 motablesFilter := util.BuildSysStatementInfoFilter(acctName) 3683 ctx.binder = NewWhereBinder(builder, ctx) 3684 accountFilterExprs, err := splitAndBindCondition(motablesFilter, NoAlias, ctx) 3685 if err != nil { 3686 return 0, err 3687 } 3688 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3689 } else if dbName == catalog.MO_CATALOG && tableName == catalog.MO_TABLES { 3690 motablesFilter := util.BuildMoTablesFilter(uint64(currentAccountID)) 3691 ctx.binder = NewWhereBinder(builder, ctx) 3692 accountFilterExprs, err := splitAndBindCondition(motablesFilter, NoAlias, ctx) 3693 if err != nil { 3694 return 0, err 3695 } 3696 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3697 } else if dbName == catalog.MO_CATALOG && tableName == catalog.MO_COLUMNS { 3698 moColumnsFilter := util.BuildMoColumnsFilter(uint64(currentAccountID)) 3699 ctx.binder = NewWhereBinder(builder, ctx) 3700 accountFilterExprs, err := splitAndBindCondition(moColumnsFilter, NoAlias, ctx) 3701 if err != nil { 3702 return 0, err 3703 } 3704 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3705 } else if util.TableIsClusterTable(midNode.GetTableDef().GetTableType()) { 3706 ctx.binder = NewWhereBinder(builder, ctx) 3707 left := &tree.UnresolvedName{ 3708 NumParts: 1, 3709 Parts: tree.NameParts{util.GetClusterTableAttributeName()}, 3710 } 3711 right := tree.NewNumVal(constant.MakeUint64(uint64(currentAccountID)), strconv.Itoa(int(currentAccountID)), false) 3712 right.ValType = tree.P_uint64 3713 //account_id = the accountId of the non-sys account 3714 accountFilter := &tree.ComparisonExpr{ 3715 Op: tree.EQUAL, 3716 Left: left, 3717 Right: right, 3718 } 3719 accountFilterExprs, err := splitAndBindCondition(accountFilter, NoAlias, ctx) 3720 if err != nil { 3721 return 0, err 3722 } 3723 builder.qry.Nodes[nodeID].FilterList = accountFilterExprs 3724 } 3725 } 3726 } 3727 return 3728 case *tree.StatementSource: 3729 return 0, moerr.NewParseError(builder.GetContext(), "unsupport table expr: %T", stmt) 3730 3731 default: 3732 // Values table not support 3733 return 0, moerr.NewParseError(builder.GetContext(), "unsupport table expr: %T", stmt) 3734 } 3735 3736 return 3737 } 3738 3739 func (builder *QueryBuilder) genNewTag() int32 { 3740 builder.nextTag++ 3741 return builder.nextTag 3742 } 3743 3744 func (builder *QueryBuilder) genNewMsgTag() (ret int32) { 3745 // start from 1, and 0 means do not handle with message 3746 builder.nextMsgTag++ 3747 return builder.nextMsgTag 3748 } 3749 3750 func (builder *QueryBuilder) addBinding(nodeID int32, alias tree.AliasClause, ctx *BindContext) error { 3751 node := builder.qry.Nodes[nodeID] 3752 3753 var cols []string 3754 var colIsHidden []bool 3755 var types []*plan.Type 3756 var defaultVals []string 3757 var binding *Binding 3758 var table string 3759 if node.NodeType == plan.Node_TABLE_SCAN || node.NodeType == plan.Node_MATERIAL_SCAN || node.NodeType == plan.Node_EXTERNAL_SCAN || node.NodeType == plan.Node_FUNCTION_SCAN || node.NodeType == plan.Node_VALUE_SCAN || node.NodeType == plan.Node_SINK_SCAN || node.NodeType == plan.Node_RECURSIVE_SCAN || node.NodeType == plan.Node_SOURCE_SCAN { 3760 if (node.NodeType == plan.Node_VALUE_SCAN || node.NodeType == plan.Node_SINK_SCAN || node.NodeType == plan.Node_RECURSIVE_SCAN) && node.TableDef == nil { 3761 return nil 3762 } 3763 if len(alias.Cols) > len(node.TableDef.Cols) { 3764 return moerr.NewSyntaxError(builder.GetContext(), "table %q has %d columns available but %d columns specified", alias.Alias, len(node.TableDef.Cols), len(alias.Cols)) 3765 } 3766 3767 if alias.Alias != "" { 3768 table = string(alias.Alias) 3769 } else { 3770 if node.NodeType == plan.Node_FUNCTION_SCAN { 3771 return moerr.NewSyntaxError(builder.GetContext(), "Every table function must have an alias") 3772 } 3773 if node.NodeType == plan.Node_RECURSIVE_SCAN || node.NodeType == plan.Node_SINK_SCAN { 3774 return nil 3775 } 3776 3777 table = node.TableDef.Name 3778 } 3779 3780 if _, ok := ctx.bindingByTable[table]; ok { 3781 return moerr.NewSyntaxError(builder.GetContext(), "table name %q specified more than once", table) 3782 } 3783 3784 colLength := len(node.TableDef.Cols) 3785 cols = make([]string, colLength) 3786 colIsHidden = make([]bool, colLength) 3787 types = make([]*plan.Type, colLength) 3788 defaultVals = make([]string, colLength) 3789 3790 tag := node.BindingTags[0] 3791 3792 for i, col := range node.TableDef.Cols { 3793 if i < len(alias.Cols) { 3794 cols[i] = string(alias.Cols[i]) 3795 } else { 3796 cols[i] = col.Name 3797 } 3798 colIsHidden[i] = col.Hidden 3799 types[i] = &col.Typ 3800 if col.Default != nil { 3801 defaultVals[i] = col.Default.OriginString 3802 } 3803 name := table + "." + cols[i] 3804 builder.nameByColRef[[2]int32{tag, int32(i)}] = name 3805 } 3806 3807 binding = NewBinding(tag, nodeID, node.TableDef.DbName, table, node.TableDef.TblId, cols, colIsHidden, types, 3808 util.TableIsClusterTable(node.TableDef.TableType), defaultVals) 3809 } else { 3810 // Subquery 3811 subCtx := builder.ctxByNode[nodeID] 3812 tag := subCtx.rootTag() 3813 headings := subCtx.headings 3814 projects := subCtx.projects 3815 3816 if len(alias.Cols) > len(headings) { 3817 return moerr.NewSyntaxError(builder.GetContext(), "11111 table %q has %d columns available but %d columns specified", alias.Alias, len(headings), len(alias.Cols)) 3818 } 3819 3820 table = subCtx.cteName 3821 if len(alias.Alias) > 0 { 3822 table = string(alias.Alias) 3823 } 3824 if len(table) == 0 { 3825 table = fmt.Sprintf("mo_table_subquery_alias_%d", tag) 3826 } 3827 if _, ok := ctx.bindingByTable[table]; ok { 3828 return moerr.NewSyntaxError(builder.GetContext(), "table name %q specified more than once", table) 3829 } 3830 3831 colLength := len(headings) 3832 cols = make([]string, colLength) 3833 colIsHidden = make([]bool, colLength) 3834 types = make([]*plan.Type, colLength) 3835 defaultVals = make([]string, colLength) 3836 3837 for i, col := range headings { 3838 if i < len(alias.Cols) { 3839 cols[i] = string(alias.Cols[i]) 3840 } else { 3841 cols[i] = strings.ToLower(col) 3842 } 3843 types[i] = &projects[i].Typ 3844 colIsHidden[i] = false 3845 defaultVals[i] = "" 3846 name := table + "." + cols[i] 3847 builder.nameByColRef[[2]int32{tag, int32(i)}] = name 3848 } 3849 3850 binding = NewBinding(tag, nodeID, "", table, 0, cols, colIsHidden, types, false, defaultVals) 3851 } 3852 3853 ctx.bindings = append(ctx.bindings, binding) 3854 ctx.bindingByTag[binding.tag] = binding 3855 ctx.bindingByTable[binding.table] = binding 3856 3857 if node.NodeType != plan.Node_RECURSIVE_SCAN && node.NodeType != plan.Node_SINK_SCAN { 3858 for _, col := range binding.cols { 3859 if _, ok := ctx.bindingByCol[col]; ok { 3860 ctx.bindingByCol[col] = nil 3861 } else { 3862 ctx.bindingByCol[col] = binding 3863 } 3864 } 3865 } 3866 3867 ctx.bindingTree = &BindingTreeNode{ 3868 binding: binding, 3869 } 3870 3871 return nil 3872 } 3873 3874 func (builder *QueryBuilder) buildJoinTable(tbl *tree.JoinTableExpr, ctx *BindContext) (int32, error) { 3875 var joinType plan.Node_JoinType 3876 3877 switch tbl.JoinType { 3878 case tree.JOIN_TYPE_CROSS, tree.JOIN_TYPE_INNER, tree.JOIN_TYPE_NATURAL: 3879 joinType = plan.Node_INNER 3880 case tree.JOIN_TYPE_LEFT, tree.JOIN_TYPE_NATURAL_LEFT: 3881 joinType = plan.Node_LEFT 3882 case tree.JOIN_TYPE_RIGHT, tree.JOIN_TYPE_NATURAL_RIGHT: 3883 joinType = plan.Node_RIGHT 3884 case tree.JOIN_TYPE_FULL: 3885 joinType = plan.Node_OUTER 3886 } 3887 3888 leftCtx := NewBindContext(builder, ctx) 3889 rightCtx := NewBindContext(builder, ctx) 3890 3891 leftChildID, err := builder.buildTable(tbl.Left, leftCtx, -1, leftCtx) 3892 if err != nil { 3893 return 0, err 3894 } 3895 ctx.views = append(ctx.views, leftCtx.views...) 3896 3897 if _, ok := tbl.Right.(*tree.TableFunction); ok { 3898 return 0, moerr.NewSyntaxError(builder.GetContext(), "Every table function must have an alias") 3899 } 3900 rightChildID, err := builder.buildTable(tbl.Right, rightCtx, leftChildID, leftCtx) 3901 if err != nil { 3902 return 0, err 3903 } 3904 ctx.views = append(ctx.views, rightCtx.views...) 3905 3906 if builder.qry.Nodes[rightChildID].NodeType == plan.Node_FUNCTION_SCAN { 3907 if joinType != plan.Node_INNER { 3908 return 0, moerr.NewSyntaxError(builder.GetContext(), "table function can only be used in a inner join") 3909 } 3910 } 3911 3912 err = ctx.mergeContexts(builder.GetContext(), leftCtx, rightCtx) 3913 if err != nil { 3914 return 0, err 3915 } 3916 3917 nodeID := builder.appendNode(&plan.Node{ 3918 NodeType: plan.Node_JOIN, 3919 Children: []int32{leftChildID, rightChildID}, 3920 JoinType: joinType, 3921 }, ctx) 3922 node := builder.qry.Nodes[nodeID] 3923 3924 ctx.binder = NewTableBinder(builder, ctx) 3925 3926 switch cond := tbl.Cond.(type) { 3927 case *tree.OnJoinCond: 3928 joinConds, err := splitAndBindCondition(cond.Expr, NoAlias, ctx) 3929 if err != nil { 3930 return 0, err 3931 } 3932 3933 node.OnList = joinConds 3934 3935 case *tree.UsingJoinCond: 3936 for _, col := range cond.Cols { 3937 expr, err := ctx.addUsingCol(string(col), joinType, leftCtx, rightCtx) 3938 if err != nil { 3939 return 0, err 3940 } 3941 3942 node.OnList = append(node.OnList, expr) 3943 } 3944 3945 default: 3946 if tbl.JoinType == tree.JOIN_TYPE_NATURAL || tbl.JoinType == tree.JOIN_TYPE_NATURAL_LEFT || tbl.JoinType == tree.JOIN_TYPE_NATURAL_RIGHT { 3947 leftCols := make(map[string]bool) 3948 for _, binding := range leftCtx.bindings { 3949 for i, col := range binding.cols { 3950 if binding.colIsHidden[i] { 3951 continue 3952 } 3953 leftCols[col] = true 3954 } 3955 } 3956 3957 var usingCols []string 3958 for _, binding := range rightCtx.bindings { 3959 for _, col := range binding.cols { 3960 if leftCols[col] { 3961 usingCols = append(usingCols, col) 3962 } 3963 } 3964 } 3965 3966 for _, col := range usingCols { 3967 expr, err := ctx.addUsingCol(col, joinType, leftCtx, rightCtx) 3968 if err != nil { 3969 return 0, err 3970 } 3971 3972 node.OnList = append(node.OnList, expr) 3973 } 3974 } 3975 } 3976 3977 return nodeID, nil 3978 } 3979 3980 func (builder *QueryBuilder) buildTableFunction(tbl *tree.TableFunction, ctx *BindContext, preNodeId int32, leftCtx *BindContext) (int32, error) { 3981 var ( 3982 childId int32 3983 err error 3984 nodeId int32 3985 ) 3986 3987 if preNodeId == -1 { 3988 scanNode := &plan.Node{ 3989 NodeType: plan.Node_VALUE_SCAN, 3990 } 3991 childId = builder.appendNode(scanNode, ctx) 3992 ctx.binder = NewTableBinder(builder, ctx) 3993 } else { 3994 ctx.binder = NewTableBinder(builder, leftCtx) 3995 childId = builder.copyNode(ctx, preNodeId) 3996 } 3997 3998 exprs := make([]*plan.Expr, 0, len(tbl.Func.Exprs)) 3999 for _, v := range tbl.Func.Exprs { 4000 curExpr, err := ctx.binder.BindExpr(v, 0, false) 4001 if err != nil { 4002 return 0, err 4003 } 4004 exprs = append(exprs, curExpr) 4005 } 4006 id := tbl.Id() 4007 switch id { 4008 case "unnest": 4009 nodeId, err = builder.buildUnnest(tbl, ctx, exprs, childId) 4010 case "generate_series": 4011 nodeId = builder.buildGenerateSeries(tbl, ctx, exprs, childId) 4012 case "meta_scan": 4013 nodeId, err = builder.buildMetaScan(tbl, ctx, exprs, childId) 4014 case "current_account": 4015 nodeId, err = builder.buildCurrentAccount(tbl, ctx, exprs, childId) 4016 case "metadata_scan": 4017 nodeId = builder.buildMetadataScan(tbl, ctx, exprs, childId) 4018 case "processlist", "mo_sessions": 4019 nodeId, err = builder.buildProcesslist(tbl, ctx, exprs, childId) 4020 case "mo_configurations": 4021 nodeId, err = builder.buildMoConfigurations(tbl, ctx, exprs, childId) 4022 case "mo_locks": 4023 nodeId, err = builder.buildMoLocks(tbl, ctx, exprs, childId) 4024 case "mo_transactions": 4025 nodeId, err = builder.buildMoTransactions(tbl, ctx, exprs, childId) 4026 case "mo_cache": 4027 nodeId, err = builder.buildMoCache(tbl, ctx, exprs, childId) 4028 default: 4029 err = moerr.NewNotSupported(builder.GetContext(), "table function '%s' not supported", id) 4030 } 4031 return nodeId, err 4032 } 4033 4034 func (builder *QueryBuilder) GetContext() context.Context { 4035 if builder == nil { 4036 return context.TODO() 4037 } 4038 return builder.compCtx.GetContext() 4039 } 4040 4041 func (builder *QueryBuilder) checkExprCanPushdown(expr *Expr, node *Node) bool { 4042 switch node.NodeType { 4043 case plan.Node_FUNCTION_SCAN: 4044 if onlyContainsTag(expr, node.BindingTags[0]) { 4045 return true 4046 } 4047 for _, childId := range node.Children { 4048 if builder.checkExprCanPushdown(expr, builder.qry.Nodes[childId]) { 4049 return true 4050 } 4051 } 4052 return false 4053 case plan.Node_TABLE_SCAN, plan.Node_EXTERNAL_SCAN, plan.Node_SOURCE_SCAN: 4054 return onlyContainsTag(expr, node.BindingTags[0]) 4055 case plan.Node_JOIN: 4056 if containsTag(expr, builder.qry.Nodes[node.Children[0]].BindingTags[0]) && containsTag(expr, builder.qry.Nodes[node.Children[1]].BindingTags[0]) { 4057 return true 4058 } 4059 for _, childId := range node.Children { 4060 if builder.checkExprCanPushdown(expr, builder.qry.Nodes[childId]) { 4061 return true 4062 } 4063 } 4064 return false 4065 4066 default: 4067 for _, childId := range node.Children { 4068 if builder.checkExprCanPushdown(expr, builder.qry.Nodes[childId]) { 4069 return true 4070 } 4071 } 4072 return false 4073 } 4074 } 4075 4076 func (builder *QueryBuilder) resolveTsHint(tsExpr *tree.AtTimeStamp) (snapshot *Snapshot, err error) { 4077 if tsExpr == nil { 4078 return 4079 } 4080 4081 conds := splitAstConjunction(tsExpr.Expr) 4082 if len(conds) != 1 { 4083 err = moerr.NewParseError(builder.GetContext(), "invalid timestamp hint") 4084 return 4085 } 4086 4087 binder := NewDefaultBinder(builder.GetContext(), nil, nil, plan.Type{}, nil) 4088 binder.builder = builder 4089 defExpr, err := binder.BindExpr(conds[0], 0, false) 4090 if err != nil { 4091 return 4092 } 4093 exprLit, ok := defExpr.Expr.(*plan.Expr_Lit) 4094 if !ok { 4095 err = moerr.NewParseError(builder.GetContext(), "invalid timestamp hint") 4096 return 4097 } 4098 4099 var tenant *SnapshotTenant 4100 if bgSnapshot := builder.compCtx.GetSnapshot(); IsSnapshotValid(bgSnapshot) { 4101 tenant = &SnapshotTenant{ 4102 TenantName: bgSnapshot.Tenant.TenantName, 4103 TenantID: bgSnapshot.Tenant.TenantID, 4104 } 4105 } 4106 4107 switch lit := exprLit.Lit.Value.(type) { 4108 case *plan.Literal_Sval: 4109 if tsExpr.Type == tree.ATTIMESTAMPTIME { 4110 var ts time.Time 4111 if ts, err = time.Parse("2006-01-02 15:04:05.999999999", lit.Sval); err != nil { 4112 return 4113 } 4114 4115 tsNano := ts.UTC().UnixNano() 4116 if tsNano <= 0 { 4117 err = moerr.NewInvalidArg(builder.GetContext(), "invalid timestamp value", lit.Sval) 4118 return 4119 } 4120 4121 if time.Now().UTC().UnixNano()-tsNano <= options.DefaultGCTTL.Nanoseconds() && 0 <= time.Now().UTC().UnixNano()-tsNano { 4122 snapshot = &Snapshot{TS: ×tamp.Timestamp{PhysicalTime: tsNano}, Tenant: tenant} 4123 } else { 4124 var valid bool 4125 if valid, err = builder.compCtx.CheckTimeStampValid(tsNano); err != nil { 4126 return 4127 } 4128 4129 if !valid { 4130 err = moerr.NewInvalidArg(builder.GetContext(), "invalid timestamp value, no corresponding snapshot ", lit.Sval) 4131 return 4132 } 4133 4134 snapshot = &Snapshot{TS: ×tamp.Timestamp{PhysicalTime: tsNano}, Tenant: tenant} 4135 } 4136 } else if tsExpr.Type == tree.ATTIMESTAMPSNAPSHOT { 4137 return builder.compCtx.ResolveSnapshotWithSnapshotName(lit.Sval) 4138 } else if tsExpr.Type == tree.ATMOTIMESTAMP { 4139 var ts timestamp.Timestamp 4140 if ts, err = timestamp.ParseTimestamp(lit.Sval); err != nil { 4141 return 4142 } 4143 4144 snapshot = &Snapshot{TS: &ts, Tenant: tenant} 4145 } else { 4146 err = moerr.NewInvalidArg(builder.GetContext(), "invalid timestamp hint type", tsExpr.Type.String()) 4147 return 4148 } 4149 case *plan.Literal_I64Val: 4150 if tsExpr.Type == tree.ATTIMESTAMPTIME { 4151 if lit.I64Val <= 0 { 4152 err = moerr.NewInvalidArg(builder.GetContext(), "invalid timestamp value", lit.I64Val) 4153 return 4154 } 4155 4156 snapshot = &Snapshot{TS: ×tamp.Timestamp{PhysicalTime: lit.I64Val}, Tenant: tenant} 4157 } else { 4158 err = moerr.NewInvalidArg(builder.GetContext(), "invalid timestamp hint for snapshot hint", lit.I64Val) 4159 return 4160 } 4161 default: 4162 err = moerr.NewInvalidArg(builder.GetContext(), "invalid input expr ", tsExpr.Expr.String()) 4163 } 4164 4165 return 4166 } 4167 4168 func IsSnapshotValid(snapshot *Snapshot) bool { 4169 if snapshot == nil { 4170 return false 4171 } 4172 4173 if snapshot.TS == nil || snapshot.TS.Equal(timestamp.Timestamp{PhysicalTime: 0, LogicalTime: 0}) { 4174 return false 4175 } 4176 4177 return true 4178 }