github.com/timstclair/heapster@v0.20.0-alpha1/Godeps/_workspace/src/google.golang.org/appengine/memcache/memcache.go (about) 1 // Copyright 2011 Google Inc. All rights reserved. 2 // Use of this source code is governed by the Apache 2.0 3 // license that can be found in the LICENSE file. 4 5 // Package memcache provides a client for App Engine's distributed in-memory 6 // key-value store for small chunks of arbitrary data. 7 // 8 // The fundamental operations get and set items, keyed by a string. 9 // 10 // item0, err := memcache.Get(c, "key") 11 // if err != nil && err != memcache.ErrCacheMiss { 12 // return err 13 // } 14 // if err == nil { 15 // fmt.Fprintf(w, "memcache hit: Key=%q Val=[% x]\n", item0.Key, item0.Value) 16 // } else { 17 // fmt.Fprintf(w, "memcache miss\n") 18 // } 19 // 20 // and 21 // 22 // item1 := &memcache.Item{ 23 // Key: "foo", 24 // Value: []byte("bar"), 25 // } 26 // if err := memcache.Set(c, item1); err != nil { 27 // return err 28 // } 29 package memcache 30 31 import ( 32 "bytes" 33 "encoding/gob" 34 "encoding/json" 35 "errors" 36 "time" 37 38 "github.com/golang/protobuf/proto" 39 "golang.org/x/net/context" 40 41 "google.golang.org/appengine" 42 "google.golang.org/appengine/internal" 43 pb "google.golang.org/appengine/internal/memcache" 44 ) 45 46 var ( 47 // ErrCacheMiss means that an operation failed 48 // because the item wasn't present. 49 ErrCacheMiss = errors.New("memcache: cache miss") 50 // ErrCASConflict means that a CompareAndSwap call failed due to the 51 // cached value being modified between the Get and the CompareAndSwap. 52 // If the cached value was simply evicted rather than replaced, 53 // ErrNotStored will be returned instead. 54 ErrCASConflict = errors.New("memcache: compare-and-swap conflict") 55 // ErrNoStats means that no statistics were available. 56 ErrNoStats = errors.New("memcache: no statistics available") 57 // ErrNotStored means that a conditional write operation (i.e. Add or 58 // CompareAndSwap) failed because the condition was not satisfied. 59 ErrNotStored = errors.New("memcache: item not stored") 60 // ErrServerError means that a server error occurred. 61 ErrServerError = errors.New("memcache: server error") 62 ) 63 64 // Item is the unit of memcache gets and sets. 65 type Item struct { 66 // Key is the Item's key (250 bytes maximum). 67 Key string 68 // Value is the Item's value. 69 Value []byte 70 // Object is the Item's value for use with a Codec. 71 Object interface{} 72 // Flags are server-opaque flags whose semantics are entirely up to the 73 // App Engine app. 74 Flags uint32 75 // Expiration is the maximum duration that the item will stay 76 // in the cache. 77 // The zero value means the Item has no expiration time. 78 // Subsecond precision is ignored. 79 // This is not set when getting items. 80 Expiration time.Duration 81 // casID is a client-opaque value used for compare-and-swap operations. 82 // Zero means that compare-and-swap is not used. 83 casID uint64 84 } 85 86 const ( 87 secondsIn30Years = 60 * 60 * 24 * 365 * 30 // from memcache server code 88 thirtyYears = time.Duration(secondsIn30Years) * time.Second 89 ) 90 91 // protoToItem converts a protocol buffer item to a Go struct. 92 func protoToItem(p *pb.MemcacheGetResponse_Item) *Item { 93 return &Item{ 94 Key: string(p.Key), 95 Value: p.Value, 96 Flags: p.GetFlags(), 97 casID: p.GetCasId(), 98 } 99 } 100 101 // If err is an appengine.MultiError, return its first element. Otherwise, return err. 102 func singleError(err error) error { 103 if me, ok := err.(appengine.MultiError); ok { 104 return me[0] 105 } 106 return err 107 } 108 109 // Get gets the item for the given key. ErrCacheMiss is returned for a memcache 110 // cache miss. The key must be at most 250 bytes in length. 111 func Get(c context.Context, key string) (*Item, error) { 112 m, err := GetMulti(c, []string{key}) 113 if err != nil { 114 return nil, err 115 } 116 if _, ok := m[key]; !ok { 117 return nil, ErrCacheMiss 118 } 119 return m[key], nil 120 } 121 122 // GetMulti is a batch version of Get. The returned map from keys to items may 123 // have fewer elements than the input slice, due to memcache cache misses. 124 // Each key must be at most 250 bytes in length. 125 func GetMulti(c context.Context, key []string) (map[string]*Item, error) { 126 if len(key) == 0 { 127 return nil, nil 128 } 129 keyAsBytes := make([][]byte, len(key)) 130 for i, k := range key { 131 keyAsBytes[i] = []byte(k) 132 } 133 req := &pb.MemcacheGetRequest{ 134 Key: keyAsBytes, 135 ForCas: proto.Bool(true), 136 } 137 res := &pb.MemcacheGetResponse{} 138 if err := internal.Call(c, "memcache", "Get", req, res); err != nil { 139 return nil, err 140 } 141 m := make(map[string]*Item, len(res.Item)) 142 for _, p := range res.Item { 143 t := protoToItem(p) 144 m[t.Key] = t 145 } 146 return m, nil 147 } 148 149 // Delete deletes the item for the given key. 150 // ErrCacheMiss is returned if the specified item can not be found. 151 // The key must be at most 250 bytes in length. 152 func Delete(c context.Context, key string) error { 153 return singleError(DeleteMulti(c, []string{key})) 154 } 155 156 // DeleteMulti is a batch version of Delete. 157 // If any keys cannot be found, an appengine.MultiError is returned. 158 // Each key must be at most 250 bytes in length. 159 func DeleteMulti(c context.Context, key []string) error { 160 if len(key) == 0 { 161 return nil 162 } 163 req := &pb.MemcacheDeleteRequest{ 164 Item: make([]*pb.MemcacheDeleteRequest_Item, len(key)), 165 } 166 for i, k := range key { 167 req.Item[i] = &pb.MemcacheDeleteRequest_Item{Key: []byte(k)} 168 } 169 res := &pb.MemcacheDeleteResponse{} 170 if err := internal.Call(c, "memcache", "Delete", req, res); err != nil { 171 return err 172 } 173 if len(res.DeleteStatus) != len(key) { 174 return ErrServerError 175 } 176 me, any := make(appengine.MultiError, len(key)), false 177 for i, s := range res.DeleteStatus { 178 switch s { 179 case pb.MemcacheDeleteResponse_DELETED: 180 // OK 181 case pb.MemcacheDeleteResponse_NOT_FOUND: 182 me[i] = ErrCacheMiss 183 any = true 184 default: 185 me[i] = ErrServerError 186 any = true 187 } 188 } 189 if any { 190 return me 191 } 192 return nil 193 } 194 195 // Increment atomically increments the decimal value in the given key 196 // by delta and returns the new value. The value must fit in a uint64. 197 // Overflow wraps around, and underflow is capped to zero. The 198 // provided delta may be negative. If the key doesn't exist in 199 // memcache, the provided initial value is used to atomically 200 // populate it before the delta is applied. 201 // The key must be at most 250 bytes in length. 202 func Increment(c context.Context, key string, delta int64, initialValue uint64) (newValue uint64, err error) { 203 return incr(c, key, delta, &initialValue) 204 } 205 206 // IncrementExisting works like Increment but assumes that the key 207 // already exists in memcache and doesn't take an initial value. 208 // IncrementExisting can save work if calculating the initial value is 209 // expensive. 210 // An error is returned if the specified item can not be found. 211 func IncrementExisting(c context.Context, key string, delta int64) (newValue uint64, err error) { 212 return incr(c, key, delta, nil) 213 } 214 215 func incr(c context.Context, key string, delta int64, initialValue *uint64) (newValue uint64, err error) { 216 req := &pb.MemcacheIncrementRequest{ 217 Key: []byte(key), 218 InitialValue: initialValue, 219 } 220 if delta >= 0 { 221 req.Delta = proto.Uint64(uint64(delta)) 222 } else { 223 req.Delta = proto.Uint64(uint64(-delta)) 224 req.Direction = pb.MemcacheIncrementRequest_DECREMENT.Enum() 225 } 226 res := &pb.MemcacheIncrementResponse{} 227 err = internal.Call(c, "memcache", "Increment", req, res) 228 if err != nil { 229 return 230 } 231 if res.NewValue == nil { 232 return 0, ErrCacheMiss 233 } 234 return *res.NewValue, nil 235 } 236 237 // set sets the given items using the given conflict resolution policy. 238 // appengine.MultiError may be returned. 239 func set(c context.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error { 240 if len(item) == 0 { 241 return nil 242 } 243 req := &pb.MemcacheSetRequest{ 244 Item: make([]*pb.MemcacheSetRequest_Item, len(item)), 245 } 246 for i, t := range item { 247 p := &pb.MemcacheSetRequest_Item{ 248 Key: []byte(t.Key), 249 } 250 if value == nil { 251 p.Value = t.Value 252 } else { 253 p.Value = value[i] 254 } 255 if t.Flags != 0 { 256 p.Flags = proto.Uint32(t.Flags) 257 } 258 if t.Expiration != 0 { 259 // In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned) 260 // for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed). 261 // Throughout this .go file, we use int32. 262 // Also, in the proto, the expiration value is either a duration (in seconds) 263 // or an absolute Unix timestamp (in seconds), depending on whether the 264 // value is less than or greater than or equal to 30 years, respectively. 265 if t.Expiration < time.Second { 266 // Because an Expiration of 0 means no expiration, we take 267 // care here to translate an item with an expiration 268 // Duration between 0-1 seconds as immediately expiring 269 // (saying it expired a few seconds ago), rather than 270 // rounding it down to 0 and making it live forever. 271 p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5) 272 } else if t.Expiration >= thirtyYears { 273 p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second)) 274 } else { 275 p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second)) 276 } 277 } 278 if t.casID != 0 { 279 p.CasId = proto.Uint64(t.casID) 280 p.ForCas = proto.Bool(true) 281 } 282 p.SetPolicy = policy.Enum() 283 req.Item[i] = p 284 } 285 res := &pb.MemcacheSetResponse{} 286 if err := internal.Call(c, "memcache", "Set", req, res); err != nil { 287 return err 288 } 289 if len(res.SetStatus) != len(item) { 290 return ErrServerError 291 } 292 me, any := make(appengine.MultiError, len(item)), false 293 for i, st := range res.SetStatus { 294 var err error 295 switch st { 296 case pb.MemcacheSetResponse_STORED: 297 // OK 298 case pb.MemcacheSetResponse_NOT_STORED: 299 err = ErrNotStored 300 case pb.MemcacheSetResponse_EXISTS: 301 err = ErrCASConflict 302 default: 303 err = ErrServerError 304 } 305 if err != nil { 306 me[i] = err 307 any = true 308 } 309 } 310 if any { 311 return me 312 } 313 return nil 314 } 315 316 // Set writes the given item, unconditionally. 317 func Set(c context.Context, item *Item) error { 318 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_SET)) 319 } 320 321 // SetMulti is a batch version of Set. 322 // appengine.MultiError may be returned. 323 func SetMulti(c context.Context, item []*Item) error { 324 return set(c, item, nil, pb.MemcacheSetRequest_SET) 325 } 326 327 // Add writes the given item, if no value already exists for its key. 328 // ErrNotStored is returned if that condition is not met. 329 func Add(c context.Context, item *Item) error { 330 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_ADD)) 331 } 332 333 // AddMulti is a batch version of Add. 334 // appengine.MultiError may be returned. 335 func AddMulti(c context.Context, item []*Item) error { 336 return set(c, item, nil, pb.MemcacheSetRequest_ADD) 337 } 338 339 // CompareAndSwap writes the given item that was previously returned by Get, 340 // if the value was neither modified or evicted between the Get and the 341 // CompareAndSwap calls. The item's Key should not change between calls but 342 // all other item fields may differ. 343 // ErrCASConflict is returned if the value was modified in between the calls. 344 // ErrNotStored is returned if the value was evicted in between the calls. 345 func CompareAndSwap(c context.Context, item *Item) error { 346 return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_CAS)) 347 } 348 349 // CompareAndSwapMulti is a batch version of CompareAndSwap. 350 // appengine.MultiError may be returned. 351 func CompareAndSwapMulti(c context.Context, item []*Item) error { 352 return set(c, item, nil, pb.MemcacheSetRequest_CAS) 353 } 354 355 // Codec represents a symmetric pair of functions that implement a codec. 356 // Items stored into or retrieved from memcache using a Codec have their 357 // values marshaled or unmarshaled. 358 // 359 // All the methods provided for Codec behave analogously to the package level 360 // function with same name. 361 type Codec struct { 362 Marshal func(interface{}) ([]byte, error) 363 Unmarshal func([]byte, interface{}) error 364 } 365 366 // Get gets the item for the given key and decodes the obtained value into v. 367 // ErrCacheMiss is returned for a memcache cache miss. 368 // The key must be at most 250 bytes in length. 369 func (cd Codec) Get(c context.Context, key string, v interface{}) (*Item, error) { 370 i, err := Get(c, key) 371 if err != nil { 372 return nil, err 373 } 374 if err := cd.Unmarshal(i.Value, v); err != nil { 375 return nil, err 376 } 377 return i, nil 378 } 379 380 func (cd Codec) set(c context.Context, items []*Item, policy pb.MemcacheSetRequest_SetPolicy) error { 381 var vs [][]byte 382 var me appengine.MultiError 383 for i, item := range items { 384 v, err := cd.Marshal(item.Object) 385 if err != nil { 386 if me == nil { 387 me = make(appengine.MultiError, len(items)) 388 } 389 me[i] = err 390 continue 391 } 392 if me == nil { 393 vs = append(vs, v) 394 } 395 } 396 if me != nil { 397 return me 398 } 399 400 return set(c, items, vs, policy) 401 } 402 403 // Set writes the given item, unconditionally. 404 func (cd Codec) Set(c context.Context, item *Item) error { 405 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_SET)) 406 } 407 408 // SetMulti is a batch version of Set. 409 // appengine.MultiError may be returned. 410 func (cd Codec) SetMulti(c context.Context, items []*Item) error { 411 return cd.set(c, items, pb.MemcacheSetRequest_SET) 412 } 413 414 // Add writes the given item, if no value already exists for its key. 415 // ErrNotStored is returned if that condition is not met. 416 func (cd Codec) Add(c context.Context, item *Item) error { 417 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_ADD)) 418 } 419 420 // AddMulti is a batch version of Add. 421 // appengine.MultiError may be returned. 422 func (cd Codec) AddMulti(c context.Context, items []*Item) error { 423 return cd.set(c, items, pb.MemcacheSetRequest_ADD) 424 } 425 426 // CompareAndSwap writes the given item that was previously returned by Get, 427 // if the value was neither modified or evicted between the Get and the 428 // CompareAndSwap calls. The item's Key should not change between calls but 429 // all other item fields may differ. 430 // ErrCASConflict is returned if the value was modified in between the calls. 431 // ErrNotStored is returned if the value was evicted in between the calls. 432 func (cd Codec) CompareAndSwap(c context.Context, item *Item) error { 433 return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_CAS)) 434 } 435 436 // CompareAndSwapMulti is a batch version of CompareAndSwap. 437 // appengine.MultiError may be returned. 438 func (cd Codec) CompareAndSwapMulti(c context.Context, items []*Item) error { 439 return cd.set(c, items, pb.MemcacheSetRequest_CAS) 440 } 441 442 var ( 443 // Gob is a Codec that uses the gob package. 444 Gob = Codec{gobMarshal, gobUnmarshal} 445 // JSON is a Codec that uses the json package. 446 JSON = Codec{json.Marshal, json.Unmarshal} 447 ) 448 449 func gobMarshal(v interface{}) ([]byte, error) { 450 var buf bytes.Buffer 451 if err := gob.NewEncoder(&buf).Encode(v); err != nil { 452 return nil, err 453 } 454 return buf.Bytes(), nil 455 } 456 457 func gobUnmarshal(data []byte, v interface{}) error { 458 return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v) 459 } 460 461 // Statistics represents a set of statistics about the memcache cache. 462 // This may include items that have expired but have not yet been removed from the cache. 463 type Statistics struct { 464 Hits uint64 // Counter of cache hits 465 Misses uint64 // Counter of cache misses 466 ByteHits uint64 // Counter of bytes transferred for gets 467 468 Items uint64 // Items currently in the cache 469 Bytes uint64 // Size of all items currently in the cache 470 471 Oldest int64 // Age of access of the oldest item, in seconds 472 } 473 474 // Stats retrieves the current memcache statistics. 475 func Stats(c context.Context) (*Statistics, error) { 476 req := &pb.MemcacheStatsRequest{} 477 res := &pb.MemcacheStatsResponse{} 478 if err := internal.Call(c, "memcache", "Stats", req, res); err != nil { 479 return nil, err 480 } 481 if res.Stats == nil { 482 return nil, ErrNoStats 483 } 484 return &Statistics{ 485 Hits: *res.Stats.Hits, 486 Misses: *res.Stats.Misses, 487 ByteHits: *res.Stats.ByteHits, 488 Items: *res.Stats.Items, 489 Bytes: *res.Stats.Bytes, 490 Oldest: int64(*res.Stats.OldestItemAge), 491 }, nil 492 } 493 494 // Flush flushes all items from memcache. 495 func Flush(c context.Context) error { 496 req := &pb.MemcacheFlushRequest{} 497 res := &pb.MemcacheFlushResponse{} 498 return internal.Call(c, "memcache", "FlushAll", req, res) 499 } 500 501 func namespaceMod(m proto.Message, namespace string) { 502 switch m := m.(type) { 503 case *pb.MemcacheDeleteRequest: 504 if m.NameSpace == nil { 505 m.NameSpace = &namespace 506 } 507 case *pb.MemcacheGetRequest: 508 if m.NameSpace == nil { 509 m.NameSpace = &namespace 510 } 511 case *pb.MemcacheIncrementRequest: 512 if m.NameSpace == nil { 513 m.NameSpace = &namespace 514 } 515 case *pb.MemcacheSetRequest: 516 if m.NameSpace == nil { 517 m.NameSpace = &namespace 518 } 519 // MemcacheFlushRequest, MemcacheStatsRequest do not apply namespace. 520 } 521 } 522 523 func init() { 524 internal.RegisterErrorCodeMap("memcache", pb.MemcacheServiceError_ErrorCode_name) 525 internal.NamespaceMods["memcache"] = namespaceMod 526 }