github.com/timstclair/heapster@v0.20.0-alpha1/Godeps/_workspace/src/google.golang.org/appengine/datastore/query.go (about) 1 // Copyright 2011 Google Inc. All rights reserved. 2 // Use of this source code is governed by the Apache 2.0 3 // license that can be found in the LICENSE file. 4 5 package datastore 6 7 import ( 8 "encoding/base64" 9 "errors" 10 "fmt" 11 "math" 12 "reflect" 13 "strings" 14 15 "github.com/golang/protobuf/proto" 16 "golang.org/x/net/context" 17 18 "google.golang.org/appengine/internal" 19 pb "google.golang.org/appengine/internal/datastore" 20 ) 21 22 type operator int 23 24 const ( 25 lessThan operator = iota 26 lessEq 27 equal 28 greaterEq 29 greaterThan 30 ) 31 32 var operatorToProto = map[operator]*pb.Query_Filter_Operator{ 33 lessThan: pb.Query_Filter_LESS_THAN.Enum(), 34 lessEq: pb.Query_Filter_LESS_THAN_OR_EQUAL.Enum(), 35 equal: pb.Query_Filter_EQUAL.Enum(), 36 greaterEq: pb.Query_Filter_GREATER_THAN_OR_EQUAL.Enum(), 37 greaterThan: pb.Query_Filter_GREATER_THAN.Enum(), 38 } 39 40 // filter is a conditional filter on query results. 41 type filter struct { 42 FieldName string 43 Op operator 44 Value interface{} 45 } 46 47 type sortDirection int 48 49 const ( 50 ascending sortDirection = iota 51 descending 52 ) 53 54 var sortDirectionToProto = map[sortDirection]*pb.Query_Order_Direction{ 55 ascending: pb.Query_Order_ASCENDING.Enum(), 56 descending: pb.Query_Order_DESCENDING.Enum(), 57 } 58 59 // order is a sort order on query results. 60 type order struct { 61 FieldName string 62 Direction sortDirection 63 } 64 65 // NewQuery creates a new Query for a specific entity kind. 66 // 67 // An empty kind means to return all entities, including entities created and 68 // managed by other App Engine features, and is called a kindless query. 69 // Kindless queries cannot include filters or sort orders on property values. 70 func NewQuery(kind string) *Query { 71 return &Query{ 72 kind: kind, 73 limit: -1, 74 } 75 } 76 77 // Query represents a datastore query. 78 type Query struct { 79 kind string 80 ancestor *Key 81 filter []filter 82 order []order 83 projection []string 84 85 distinct bool 86 keysOnly bool 87 eventual bool 88 limit int32 89 offset int32 90 start *pb.CompiledCursor 91 end *pb.CompiledCursor 92 93 err error 94 } 95 96 func (q *Query) clone() *Query { 97 x := *q 98 // Copy the contents of the slice-typed fields to a new backing store. 99 if len(q.filter) > 0 { 100 x.filter = make([]filter, len(q.filter)) 101 copy(x.filter, q.filter) 102 } 103 if len(q.order) > 0 { 104 x.order = make([]order, len(q.order)) 105 copy(x.order, q.order) 106 } 107 return &x 108 } 109 110 // Ancestor returns a derivative query with an ancestor filter. 111 // The ancestor should not be nil. 112 func (q *Query) Ancestor(ancestor *Key) *Query { 113 q = q.clone() 114 if ancestor == nil { 115 q.err = errors.New("datastore: nil query ancestor") 116 return q 117 } 118 q.ancestor = ancestor 119 return q 120 } 121 122 // EventualConsistency returns a derivative query that returns eventually 123 // consistent results. 124 // It only has an effect on ancestor queries. 125 func (q *Query) EventualConsistency() *Query { 126 q = q.clone() 127 q.eventual = true 128 return q 129 } 130 131 // Filter returns a derivative query with a field-based filter. 132 // The filterStr argument must be a field name followed by optional space, 133 // followed by an operator, one of ">", "<", ">=", "<=", or "=". 134 // Fields are compared against the provided value using the operator. 135 // Multiple filters are AND'ed together. 136 func (q *Query) Filter(filterStr string, value interface{}) *Query { 137 q = q.clone() 138 filterStr = strings.TrimSpace(filterStr) 139 if len(filterStr) < 1 { 140 q.err = errors.New("datastore: invalid filter: " + filterStr) 141 return q 142 } 143 f := filter{ 144 FieldName: strings.TrimRight(filterStr, " ><=!"), 145 Value: value, 146 } 147 switch op := strings.TrimSpace(filterStr[len(f.FieldName):]); op { 148 case "<=": 149 f.Op = lessEq 150 case ">=": 151 f.Op = greaterEq 152 case "<": 153 f.Op = lessThan 154 case ">": 155 f.Op = greaterThan 156 case "=": 157 f.Op = equal 158 default: 159 q.err = fmt.Errorf("datastore: invalid operator %q in filter %q", op, filterStr) 160 return q 161 } 162 q.filter = append(q.filter, f) 163 return q 164 } 165 166 // Order returns a derivative query with a field-based sort order. Orders are 167 // applied in the order they are added. The default order is ascending; to sort 168 // in descending order prefix the fieldName with a minus sign (-). 169 func (q *Query) Order(fieldName string) *Query { 170 q = q.clone() 171 fieldName = strings.TrimSpace(fieldName) 172 o := order{ 173 Direction: ascending, 174 FieldName: fieldName, 175 } 176 if strings.HasPrefix(fieldName, "-") { 177 o.Direction = descending 178 o.FieldName = strings.TrimSpace(fieldName[1:]) 179 } else if strings.HasPrefix(fieldName, "+") { 180 q.err = fmt.Errorf("datastore: invalid order: %q", fieldName) 181 return q 182 } 183 if len(o.FieldName) == 0 { 184 q.err = errors.New("datastore: empty order") 185 return q 186 } 187 q.order = append(q.order, o) 188 return q 189 } 190 191 // Project returns a derivative query that yields only the given fields. It 192 // cannot be used with KeysOnly. 193 func (q *Query) Project(fieldNames ...string) *Query { 194 q = q.clone() 195 q.projection = append([]string(nil), fieldNames...) 196 return q 197 } 198 199 // Distinct returns a derivative query that yields de-duplicated entities with 200 // respect to the set of projected fields. It is only used for projection 201 // queries. 202 func (q *Query) Distinct() *Query { 203 q = q.clone() 204 q.distinct = true 205 return q 206 } 207 208 // KeysOnly returns a derivative query that yields only keys, not keys and 209 // entities. It cannot be used with projection queries. 210 func (q *Query) KeysOnly() *Query { 211 q = q.clone() 212 q.keysOnly = true 213 return q 214 } 215 216 // Limit returns a derivative query that has a limit on the number of results 217 // returned. A negative value means unlimited. 218 func (q *Query) Limit(limit int) *Query { 219 q = q.clone() 220 if limit < math.MinInt32 || limit > math.MaxInt32 { 221 q.err = errors.New("datastore: query limit overflow") 222 return q 223 } 224 q.limit = int32(limit) 225 return q 226 } 227 228 // Offset returns a derivative query that has an offset of how many keys to 229 // skip over before returning results. A negative value is invalid. 230 func (q *Query) Offset(offset int) *Query { 231 q = q.clone() 232 if offset < 0 { 233 q.err = errors.New("datastore: negative query offset") 234 return q 235 } 236 if offset > math.MaxInt32 { 237 q.err = errors.New("datastore: query offset overflow") 238 return q 239 } 240 q.offset = int32(offset) 241 return q 242 } 243 244 // Start returns a derivative query with the given start point. 245 func (q *Query) Start(c Cursor) *Query { 246 q = q.clone() 247 if c.cc == nil { 248 q.err = errors.New("datastore: invalid cursor") 249 return q 250 } 251 q.start = c.cc 252 return q 253 } 254 255 // End returns a derivative query with the given end point. 256 func (q *Query) End(c Cursor) *Query { 257 q = q.clone() 258 if c.cc == nil { 259 q.err = errors.New("datastore: invalid cursor") 260 return q 261 } 262 q.end = c.cc 263 return q 264 } 265 266 // toProto converts the query to a protocol buffer. 267 func (q *Query) toProto(dst *pb.Query, appID string) error { 268 if len(q.projection) != 0 && q.keysOnly { 269 return errors.New("datastore: query cannot both project and be keys-only") 270 } 271 dst.Reset() 272 dst.App = proto.String(appID) 273 if q.kind != "" { 274 dst.Kind = proto.String(q.kind) 275 } 276 if q.ancestor != nil { 277 dst.Ancestor = keyToProto(appID, q.ancestor) 278 if q.eventual { 279 dst.Strong = proto.Bool(false) 280 } 281 } 282 if q.projection != nil { 283 dst.PropertyName = q.projection 284 if q.distinct { 285 dst.GroupByPropertyName = q.projection 286 } 287 } 288 if q.keysOnly { 289 dst.KeysOnly = proto.Bool(true) 290 dst.RequirePerfectPlan = proto.Bool(true) 291 } 292 for _, qf := range q.filter { 293 if qf.FieldName == "" { 294 return errors.New("datastore: empty query filter field name") 295 } 296 p, errStr := valueToProto(appID, qf.FieldName, reflect.ValueOf(qf.Value), false) 297 if errStr != "" { 298 return errors.New("datastore: bad query filter value type: " + errStr) 299 } 300 xf := &pb.Query_Filter{ 301 Op: operatorToProto[qf.Op], 302 Property: []*pb.Property{p}, 303 } 304 if xf.Op == nil { 305 return errors.New("datastore: unknown query filter operator") 306 } 307 dst.Filter = append(dst.Filter, xf) 308 } 309 for _, qo := range q.order { 310 if qo.FieldName == "" { 311 return errors.New("datastore: empty query order field name") 312 } 313 xo := &pb.Query_Order{ 314 Property: proto.String(qo.FieldName), 315 Direction: sortDirectionToProto[qo.Direction], 316 } 317 if xo.Direction == nil { 318 return errors.New("datastore: unknown query order direction") 319 } 320 dst.Order = append(dst.Order, xo) 321 } 322 if q.limit >= 0 { 323 dst.Limit = proto.Int32(q.limit) 324 } 325 if q.offset != 0 { 326 dst.Offset = proto.Int32(q.offset) 327 } 328 dst.CompiledCursor = q.start 329 dst.EndCompiledCursor = q.end 330 dst.Compile = proto.Bool(true) 331 return nil 332 } 333 334 // Count returns the number of results for the query. 335 func (q *Query) Count(c context.Context) (int, error) { 336 // Check that the query is well-formed. 337 if q.err != nil { 338 return 0, q.err 339 } 340 341 // Run a copy of the query, with keysOnly true (if we're not a projection, 342 // since the two are incompatible), and an adjusted offset. We also set the 343 // limit to zero, as we don't want any actual entity data, just the number 344 // of skipped results. 345 newQ := q.clone() 346 newQ.keysOnly = len(newQ.projection) == 0 347 newQ.limit = 0 348 if q.limit < 0 { 349 // If the original query was unlimited, set the new query's offset to maximum. 350 newQ.offset = math.MaxInt32 351 } else { 352 newQ.offset = q.offset + q.limit 353 if newQ.offset < 0 { 354 // Do the best we can, in the presence of overflow. 355 newQ.offset = math.MaxInt32 356 } 357 } 358 req := &pb.Query{} 359 if err := newQ.toProto(req, internal.FullyQualifiedAppID(c)); err != nil { 360 return 0, err 361 } 362 res := &pb.QueryResult{} 363 if err := internal.Call(c, "datastore_v3", "RunQuery", req, res); err != nil { 364 return 0, err 365 } 366 367 // n is the count we will return. For example, suppose that our original 368 // query had an offset of 4 and a limit of 2008: the count will be 2008, 369 // provided that there are at least 2012 matching entities. However, the 370 // RPCs will only skip 1000 results at a time. The RPC sequence is: 371 // call RunQuery with (offset, limit) = (2012, 0) // 2012 == newQ.offset 372 // response has (skippedResults, moreResults) = (1000, true) 373 // n += 1000 // n == 1000 374 // call Next with (offset, limit) = (1012, 0) // 1012 == newQ.offset - n 375 // response has (skippedResults, moreResults) = (1000, true) 376 // n += 1000 // n == 2000 377 // call Next with (offset, limit) = (12, 0) // 12 == newQ.offset - n 378 // response has (skippedResults, moreResults) = (12, false) 379 // n += 12 // n == 2012 380 // // exit the loop 381 // n -= 4 // n == 2008 382 var n int32 383 for { 384 // The QueryResult should have no actual entity data, just skipped results. 385 if len(res.Result) != 0 { 386 return 0, errors.New("datastore: internal error: Count request returned too much data") 387 } 388 n += res.GetSkippedResults() 389 if !res.GetMoreResults() { 390 break 391 } 392 if err := callNext(c, res, newQ.offset-n, 0); err != nil { 393 return 0, err 394 } 395 } 396 n -= q.offset 397 if n < 0 { 398 // If the offset was greater than the number of matching entities, 399 // return 0 instead of negative. 400 n = 0 401 } 402 return int(n), nil 403 } 404 405 // callNext issues a datastore_v3/Next RPC to advance a cursor, such as that 406 // returned by a query with more results. 407 func callNext(c context.Context, res *pb.QueryResult, offset, limit int32) error { 408 if res.Cursor == nil { 409 return errors.New("datastore: internal error: server did not return a cursor") 410 } 411 req := &pb.NextRequest{ 412 Cursor: res.Cursor, 413 } 414 if limit >= 0 { 415 req.Count = proto.Int32(limit) 416 } 417 if offset != 0 { 418 req.Offset = proto.Int32(offset) 419 } 420 if res.CompiledCursor != nil { 421 req.Compile = proto.Bool(true) 422 } 423 res.Reset() 424 return internal.Call(c, "datastore_v3", "Next", req, res) 425 } 426 427 // GetAll runs the query in the given context and returns all keys that match 428 // that query, as well as appending the values to dst. 429 // 430 // dst must have type *[]S or *[]*S or *[]P, for some struct type S or some non- 431 // interface, non-pointer type P such that P or *P implements PropertyLoadSaver. 432 // 433 // As a special case, *PropertyList is an invalid type for dst, even though a 434 // PropertyList is a slice of structs. It is treated as invalid to avoid being 435 // mistakenly passed when *[]PropertyList was intended. 436 // 437 // The keys returned by GetAll will be in a 1-1 correspondence with the entities 438 // added to dst. 439 // 440 // If q is a ``keys-only'' query, GetAll ignores dst and only returns the keys. 441 func (q *Query) GetAll(c context.Context, dst interface{}) ([]*Key, error) { 442 var ( 443 dv reflect.Value 444 mat multiArgType 445 elemType reflect.Type 446 errFieldMismatch error 447 ) 448 if !q.keysOnly { 449 dv = reflect.ValueOf(dst) 450 if dv.Kind() != reflect.Ptr || dv.IsNil() { 451 return nil, ErrInvalidEntityType 452 } 453 dv = dv.Elem() 454 mat, elemType = checkMultiArg(dv) 455 if mat == multiArgTypeInvalid || mat == multiArgTypeInterface { 456 return nil, ErrInvalidEntityType 457 } 458 } 459 460 var keys []*Key 461 for t := q.Run(c); ; { 462 k, e, err := t.next() 463 if err == Done { 464 break 465 } 466 if err != nil { 467 return keys, err 468 } 469 if !q.keysOnly { 470 ev := reflect.New(elemType) 471 if elemType.Kind() == reflect.Map { 472 // This is a special case. The zero values of a map type are 473 // not immediately useful; they have to be make'd. 474 // 475 // Funcs and channels are similar, in that a zero value is not useful, 476 // but even a freshly make'd channel isn't useful: there's no fixed 477 // channel buffer size that is always going to be large enough, and 478 // there's no goroutine to drain the other end. Theoretically, these 479 // types could be supported, for example by sniffing for a constructor 480 // method or requiring prior registration, but for now it's not a 481 // frequent enough concern to be worth it. Programmers can work around 482 // it by explicitly using Iterator.Next instead of the Query.GetAll 483 // convenience method. 484 x := reflect.MakeMap(elemType) 485 ev.Elem().Set(x) 486 } 487 if err = loadEntity(ev.Interface(), e); err != nil { 488 if _, ok := err.(*ErrFieldMismatch); ok { 489 // We continue loading entities even in the face of field mismatch errors. 490 // If we encounter any other error, that other error is returned. Otherwise, 491 // an ErrFieldMismatch is returned. 492 errFieldMismatch = err 493 } else { 494 return keys, err 495 } 496 } 497 if mat != multiArgTypeStructPtr { 498 ev = ev.Elem() 499 } 500 dv.Set(reflect.Append(dv, ev)) 501 } 502 keys = append(keys, k) 503 } 504 return keys, errFieldMismatch 505 } 506 507 // Run runs the query in the given context. 508 func (q *Query) Run(c context.Context) *Iterator { 509 if q.err != nil { 510 return &Iterator{err: q.err} 511 } 512 t := &Iterator{ 513 c: c, 514 limit: q.limit, 515 q: q, 516 prevCC: q.start, 517 } 518 var req pb.Query 519 if err := q.toProto(&req, internal.FullyQualifiedAppID(c)); err != nil { 520 t.err = err 521 return t 522 } 523 if err := internal.Call(c, "datastore_v3", "RunQuery", &req, &t.res); err != nil { 524 t.err = err 525 return t 526 } 527 offset := q.offset - t.res.GetSkippedResults() 528 for offset > 0 && t.res.GetMoreResults() { 529 t.prevCC = t.res.CompiledCursor 530 if err := callNext(t.c, &t.res, offset, t.limit); err != nil { 531 t.err = err 532 break 533 } 534 skip := t.res.GetSkippedResults() 535 if skip < 0 { 536 t.err = errors.New("datastore: internal error: negative number of skipped_results") 537 break 538 } 539 offset -= skip 540 } 541 if offset < 0 { 542 t.err = errors.New("datastore: internal error: query offset was overshot") 543 } 544 return t 545 } 546 547 // Iterator is the result of running a query. 548 type Iterator struct { 549 c context.Context 550 err error 551 // res is the result of the most recent RunQuery or Next API call. 552 res pb.QueryResult 553 // i is how many elements of res.Result we have iterated over. 554 i int 555 // limit is the limit on the number of results this iterator should return. 556 // A negative value means unlimited. 557 limit int32 558 // q is the original query which yielded this iterator. 559 q *Query 560 // prevCC is the compiled cursor that marks the end of the previous batch 561 // of results. 562 prevCC *pb.CompiledCursor 563 } 564 565 // Done is returned when a query iteration has completed. 566 var Done = errors.New("datastore: query has no more results") 567 568 // Next returns the key of the next result. When there are no more results, 569 // Done is returned as the error. 570 // 571 // If the query is not keys only and dst is non-nil, it also loads the entity 572 // stored for that key into the struct pointer or PropertyLoadSaver dst, with 573 // the same semantics and possible errors as for the Get function. 574 func (t *Iterator) Next(dst interface{}) (*Key, error) { 575 k, e, err := t.next() 576 if err != nil { 577 return nil, err 578 } 579 if dst != nil && !t.q.keysOnly { 580 err = loadEntity(dst, e) 581 } 582 return k, err 583 } 584 585 func (t *Iterator) next() (*Key, *pb.EntityProto, error) { 586 if t.err != nil { 587 return nil, nil, t.err 588 } 589 590 // Issue datastore_v3/Next RPCs as necessary. 591 for t.i == len(t.res.Result) { 592 if !t.res.GetMoreResults() { 593 t.err = Done 594 return nil, nil, t.err 595 } 596 t.prevCC = t.res.CompiledCursor 597 if err := callNext(t.c, &t.res, 0, t.limit); err != nil { 598 t.err = err 599 return nil, nil, t.err 600 } 601 if t.res.GetSkippedResults() != 0 { 602 t.err = errors.New("datastore: internal error: iterator has skipped results") 603 return nil, nil, t.err 604 } 605 t.i = 0 606 if t.limit >= 0 { 607 t.limit -= int32(len(t.res.Result)) 608 if t.limit < 0 { 609 t.err = errors.New("datastore: internal error: query returned more results than the limit") 610 return nil, nil, t.err 611 } 612 } 613 } 614 615 // Extract the key from the t.i'th element of t.res.Result. 616 e := t.res.Result[t.i] 617 t.i++ 618 if e.Key == nil { 619 return nil, nil, errors.New("datastore: internal error: server did not return a key") 620 } 621 k, err := protoToKey(e.Key) 622 if err != nil || k.Incomplete() { 623 return nil, nil, errors.New("datastore: internal error: server returned an invalid key") 624 } 625 return k, e, nil 626 } 627 628 // Cursor returns a cursor for the iterator's current location. 629 func (t *Iterator) Cursor() (Cursor, error) { 630 if t.err != nil && t.err != Done { 631 return Cursor{}, t.err 632 } 633 // If we are at either end of the current batch of results, 634 // return the compiled cursor at that end. 635 skipped := t.res.GetSkippedResults() 636 if t.i == 0 && skipped == 0 { 637 if t.prevCC == nil { 638 // A nil pointer (of type *pb.CompiledCursor) means no constraint: 639 // passing it as the end cursor of a new query means unlimited results 640 // (glossing over the integer limit parameter for now). 641 // A non-nil pointer to an empty pb.CompiledCursor means the start: 642 // passing it as the end cursor of a new query means 0 results. 643 // If prevCC was nil, then the original query had no start cursor, but 644 // Iterator.Cursor should return "the start" instead of unlimited. 645 return Cursor{&zeroCC}, nil 646 } 647 return Cursor{t.prevCC}, nil 648 } 649 if t.i == len(t.res.Result) { 650 return Cursor{t.res.CompiledCursor}, nil 651 } 652 // Otherwise, re-run the query offset to this iterator's position, starting from 653 // the most recent compiled cursor. This is done on a best-effort basis, as it 654 // is racy; if a concurrent process has added or removed entities, then the 655 // cursor returned may be inconsistent. 656 q := t.q.clone() 657 q.start = t.prevCC 658 q.offset = skipped + int32(t.i) 659 q.limit = 0 660 q.keysOnly = len(q.projection) == 0 661 t1 := q.Run(t.c) 662 _, _, err := t1.next() 663 if err != Done { 664 if err == nil { 665 err = fmt.Errorf("datastore: internal error: zero-limit query did not have zero results") 666 } 667 return Cursor{}, err 668 } 669 return Cursor{t1.res.CompiledCursor}, nil 670 } 671 672 var zeroCC pb.CompiledCursor 673 674 // Cursor is an iterator's position. It can be converted to and from an opaque 675 // string. A cursor can be used from different HTTP requests, but only with a 676 // query with the same kind, ancestor, filter and order constraints. 677 type Cursor struct { 678 cc *pb.CompiledCursor 679 } 680 681 // String returns a base-64 string representation of a cursor. 682 func (c Cursor) String() string { 683 if c.cc == nil { 684 return "" 685 } 686 b, err := proto.Marshal(c.cc) 687 if err != nil { 688 // The only way to construct a Cursor with a non-nil cc field is to 689 // unmarshal from the byte representation. We panic if the unmarshal 690 // succeeds but the marshaling of the unchanged protobuf value fails. 691 panic(fmt.Sprintf("datastore: internal error: malformed cursor: %v", err)) 692 } 693 return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") 694 } 695 696 // Decode decodes a cursor from its base-64 string representation. 697 func DecodeCursor(s string) (Cursor, error) { 698 if s == "" { 699 return Cursor{&zeroCC}, nil 700 } 701 if n := len(s) % 4; n != 0 { 702 s += strings.Repeat("=", 4-n) 703 } 704 b, err := base64.URLEncoding.DecodeString(s) 705 if err != nil { 706 return Cursor{}, err 707 } 708 cc := &pb.CompiledCursor{} 709 if err := proto.Unmarshal(b, cc); err != nil { 710 return Cursor{}, err 711 } 712 return Cursor{cc}, nil 713 }