github.com/timstclair/heapster@v0.20.0-alpha1/Godeps/_workspace/src/google.golang.org/appengine/taskqueue/taskqueue.go (about) 1 // Copyright 2011 Google Inc. All rights reserved. 2 // Use of this source code is governed by the Apache 2.0 3 // license that can be found in the LICENSE file. 4 5 /* 6 Package taskqueue provides a client for App Engine's taskqueue service. 7 Using this service, applications may perform work outside a user's request. 8 9 A Task may be constructed manually; alternatively, since the most common 10 taskqueue operation is to add a single POST task, NewPOSTTask makes it easy. 11 12 t := taskqueue.NewPOSTTask("/worker", url.Values{ 13 "key": {key}, 14 }) 15 taskqueue.Add(c, t, "") // add t to the default queue 16 */ 17 package taskqueue 18 19 import ( 20 "errors" 21 "fmt" 22 "net/http" 23 "net/url" 24 "time" 25 26 "github.com/golang/protobuf/proto" 27 "golang.org/x/net/context" 28 29 "google.golang.org/appengine" 30 "google.golang.org/appengine/internal" 31 dspb "google.golang.org/appengine/internal/datastore" 32 pb "google.golang.org/appengine/internal/taskqueue" 33 ) 34 35 var ( 36 // ErrTaskAlreadyAdded is the error returned by Add and AddMulti when a task has already been added with a particular name. 37 ErrTaskAlreadyAdded = errors.New("taskqueue: task has already been added") 38 ) 39 40 // RetryOptions let you control whether to retry a task and the backoff intervals between tries. 41 type RetryOptions struct { 42 // Number of tries/leases after which the task fails permanently and is deleted. 43 // If AgeLimit is also set, both limits must be exceeded for the task to fail permanently. 44 RetryLimit int32 45 46 // Maximum time allowed since the task's first try before the task fails permanently and is deleted (only for push tasks). 47 // If RetryLimit is also set, both limits must be exceeded for the task to fail permanently. 48 AgeLimit time.Duration 49 50 // Minimum time between successive tries (only for push tasks). 51 MinBackoff time.Duration 52 53 // Maximum time between successive tries (only for push tasks). 54 MaxBackoff time.Duration 55 56 // Maximum number of times to double the interval between successive tries before the intervals increase linearly (only for push tasks). 57 MaxDoublings int32 58 59 // If MaxDoublings is zero, set ApplyZeroMaxDoublings to true to override the default non-zero value. 60 // Otherwise a zero MaxDoublings is ignored and the default is used. 61 ApplyZeroMaxDoublings bool 62 } 63 64 // toRetryParameter converts RetryOptions to pb.TaskQueueRetryParameters. 65 func (opt *RetryOptions) toRetryParameters() *pb.TaskQueueRetryParameters { 66 params := &pb.TaskQueueRetryParameters{} 67 if opt.RetryLimit > 0 { 68 params.RetryLimit = proto.Int32(opt.RetryLimit) 69 } 70 if opt.AgeLimit > 0 { 71 params.AgeLimitSec = proto.Int64(int64(opt.AgeLimit.Seconds())) 72 } 73 if opt.MinBackoff > 0 { 74 params.MinBackoffSec = proto.Float64(opt.MinBackoff.Seconds()) 75 } 76 if opt.MaxBackoff > 0 { 77 params.MaxBackoffSec = proto.Float64(opt.MaxBackoff.Seconds()) 78 } 79 if opt.MaxDoublings > 0 || (opt.MaxDoublings == 0 && opt.ApplyZeroMaxDoublings) { 80 params.MaxDoublings = proto.Int32(opt.MaxDoublings) 81 } 82 return params 83 } 84 85 // A Task represents a task to be executed. 86 type Task struct { 87 // Path is the worker URL for the task. 88 // If unset, it will default to /_ah/queue/<queue_name>. 89 Path string 90 91 // Payload is the data for the task. 92 // This will be delivered as the HTTP request body. 93 // It is only used when Method is POST, PUT or PULL. 94 // url.Values' Encode method may be used to generate this for POST requests. 95 Payload []byte 96 97 // Additional HTTP headers to pass at the task's execution time. 98 // To schedule the task to be run with an alternate app version 99 // or backend, set the "Host" header. 100 Header http.Header 101 102 // Method is the HTTP method for the task ("GET", "POST", etc.), 103 // or "PULL" if this is task is destined for a pull-based queue. 104 // If empty, this defaults to "POST". 105 Method string 106 107 // A name for the task. 108 // If empty, a name will be chosen. 109 Name string 110 111 // Delay specifies the duration the task queue service must wait 112 // before executing the task. 113 // Either Delay or ETA may be set, but not both. 114 Delay time.Duration 115 116 // ETA specifies the earliest time a task may be executed (push queues) 117 // or leased (pull queues). 118 // Either Delay or ETA may be set, but not both. 119 ETA time.Time 120 121 // The number of times the task has been dispatched or leased. 122 RetryCount int32 123 124 // Tag for the task. Only used when Method is PULL. 125 Tag string 126 127 // Retry options for this task. May be nil. 128 RetryOptions *RetryOptions 129 } 130 131 func (t *Task) method() string { 132 if t.Method == "" { 133 return "POST" 134 } 135 return t.Method 136 } 137 138 // NewPOSTTask creates a Task that will POST to a path with the given form data. 139 func NewPOSTTask(path string, params url.Values) *Task { 140 h := make(http.Header) 141 h.Set("Content-Type", "application/x-www-form-urlencoded") 142 return &Task{ 143 Path: path, 144 Payload: []byte(params.Encode()), 145 Header: h, 146 Method: "POST", 147 } 148 } 149 150 var ( 151 currentNamespace = http.CanonicalHeaderKey("X-AppEngine-Current-Namespace") 152 defaultNamespace = http.CanonicalHeaderKey("X-AppEngine-Default-Namespace") 153 ) 154 155 func getDefaultNamespace(ctx context.Context) string { 156 return internal.IncomingHeaders(ctx).Get(defaultNamespace) 157 } 158 159 func newAddReq(c context.Context, task *Task, queueName string) (*pb.TaskQueueAddRequest, error) { 160 if queueName == "" { 161 queueName = "default" 162 } 163 path := task.Path 164 if path == "" { 165 path = "/_ah/queue/" + queueName 166 } 167 eta := task.ETA 168 if eta.IsZero() { 169 eta = time.Now().Add(task.Delay) 170 } else if task.Delay != 0 { 171 panic("taskqueue: both Delay and ETA are set") 172 } 173 req := &pb.TaskQueueAddRequest{ 174 QueueName: []byte(queueName), 175 TaskName: []byte(task.Name), 176 EtaUsec: proto.Int64(eta.UnixNano() / 1e3), 177 } 178 method := task.method() 179 if method == "PULL" { 180 // Pull-based task 181 req.Body = task.Payload 182 req.Mode = pb.TaskQueueMode_PULL.Enum() 183 if task.Tag != "" { 184 req.Tag = []byte(task.Tag) 185 } 186 } else { 187 // HTTP-based task 188 if v, ok := pb.TaskQueueAddRequest_RequestMethod_value[method]; ok { 189 req.Method = pb.TaskQueueAddRequest_RequestMethod(v).Enum() 190 } else { 191 return nil, fmt.Errorf("taskqueue: bad method %q", method) 192 } 193 req.Url = []byte(path) 194 for k, vs := range task.Header { 195 for _, v := range vs { 196 req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ 197 Key: []byte(k), 198 Value: []byte(v), 199 }) 200 } 201 } 202 if method == "POST" || method == "PUT" { 203 req.Body = task.Payload 204 } 205 206 // Namespace headers. 207 if _, ok := task.Header[currentNamespace]; !ok { 208 // Fetch the current namespace of this request. 209 ns := internal.NamespaceFromContext(c) 210 req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ 211 Key: []byte(currentNamespace), 212 Value: []byte(ns), 213 }) 214 } 215 if _, ok := task.Header[defaultNamespace]; !ok { 216 // Fetch the X-AppEngine-Default-Namespace header of this request. 217 if ns := getDefaultNamespace(c); ns != "" { 218 req.Header = append(req.Header, &pb.TaskQueueAddRequest_Header{ 219 Key: []byte(defaultNamespace), 220 Value: []byte(ns), 221 }) 222 } 223 } 224 } 225 226 if task.RetryOptions != nil { 227 req.RetryParameters = task.RetryOptions.toRetryParameters() 228 } 229 230 return req, nil 231 } 232 233 var alreadyAddedErrors = map[pb.TaskQueueServiceError_ErrorCode]bool{ 234 pb.TaskQueueServiceError_TASK_ALREADY_EXISTS: true, 235 pb.TaskQueueServiceError_TOMBSTONED_TASK: true, 236 } 237 238 // Add adds the task to a named queue. 239 // An empty queue name means that the default queue will be used. 240 // Add returns an equivalent Task with defaults filled in, including setting 241 // the task's Name field to the chosen name if the original was empty. 242 func Add(c context.Context, task *Task, queueName string) (*Task, error) { 243 req, err := newAddReq(c, task, queueName) 244 if err != nil { 245 return nil, err 246 } 247 res := &pb.TaskQueueAddResponse{} 248 if err := internal.Call(c, "taskqueue", "Add", req, res); err != nil { 249 apiErr, ok := err.(*internal.APIError) 250 if ok && alreadyAddedErrors[pb.TaskQueueServiceError_ErrorCode(apiErr.Code)] { 251 return nil, ErrTaskAlreadyAdded 252 } 253 return nil, err 254 } 255 resultTask := *task 256 resultTask.Method = task.method() 257 if task.Name == "" { 258 resultTask.Name = string(res.ChosenTaskName) 259 } 260 return &resultTask, nil 261 } 262 263 // AddMulti adds multiple tasks to a named queue. 264 // An empty queue name means that the default queue will be used. 265 // AddMulti returns a slice of equivalent tasks with defaults filled in, including setting 266 // each task's Name field to the chosen name if the original was empty. 267 // If a given task is badly formed or could not be added, an appengine.MultiError is returned. 268 func AddMulti(c context.Context, tasks []*Task, queueName string) ([]*Task, error) { 269 req := &pb.TaskQueueBulkAddRequest{ 270 AddRequest: make([]*pb.TaskQueueAddRequest, len(tasks)), 271 } 272 me, any := make(appengine.MultiError, len(tasks)), false 273 for i, t := range tasks { 274 req.AddRequest[i], me[i] = newAddReq(c, t, queueName) 275 any = any || me[i] != nil 276 } 277 if any { 278 return nil, me 279 } 280 res := &pb.TaskQueueBulkAddResponse{} 281 if err := internal.Call(c, "taskqueue", "BulkAdd", req, res); err != nil { 282 return nil, err 283 } 284 if len(res.Taskresult) != len(tasks) { 285 return nil, errors.New("taskqueue: server error") 286 } 287 tasksOut := make([]*Task, len(tasks)) 288 for i, tr := range res.Taskresult { 289 tasksOut[i] = new(Task) 290 *tasksOut[i] = *tasks[i] 291 tasksOut[i].Method = tasksOut[i].method() 292 if tasksOut[i].Name == "" { 293 tasksOut[i].Name = string(tr.ChosenTaskName) 294 } 295 if *tr.Result != pb.TaskQueueServiceError_OK { 296 if alreadyAddedErrors[*tr.Result] { 297 me[i] = ErrTaskAlreadyAdded 298 } else { 299 me[i] = &internal.APIError{ 300 Service: "taskqueue", 301 Code: int32(*tr.Result), 302 } 303 } 304 any = true 305 } 306 } 307 if any { 308 return tasksOut, me 309 } 310 return tasksOut, nil 311 } 312 313 // Delete deletes a task from a named queue. 314 func Delete(c context.Context, task *Task, queueName string) error { 315 err := DeleteMulti(c, []*Task{task}, queueName) 316 if me, ok := err.(appengine.MultiError); ok { 317 return me[0] 318 } 319 return err 320 } 321 322 // DeleteMulti deletes multiple tasks from a named queue. 323 // If a given task could not be deleted, an appengine.MultiError is returned. 324 func DeleteMulti(c context.Context, tasks []*Task, queueName string) error { 325 taskNames := make([][]byte, len(tasks)) 326 for i, t := range tasks { 327 taskNames[i] = []byte(t.Name) 328 } 329 if queueName == "" { 330 queueName = "default" 331 } 332 req := &pb.TaskQueueDeleteRequest{ 333 QueueName: []byte(queueName), 334 TaskName: taskNames, 335 } 336 res := &pb.TaskQueueDeleteResponse{} 337 if err := internal.Call(c, "taskqueue", "Delete", req, res); err != nil { 338 return err 339 } 340 if a, b := len(req.TaskName), len(res.Result); a != b { 341 return fmt.Errorf("taskqueue: internal error: requested deletion of %d tasks, got %d results", a, b) 342 } 343 me, any := make(appengine.MultiError, len(res.Result)), false 344 for i, ec := range res.Result { 345 if ec != pb.TaskQueueServiceError_OK { 346 me[i] = &internal.APIError{ 347 Service: "taskqueue", 348 Code: int32(ec), 349 } 350 any = true 351 } 352 } 353 if any { 354 return me 355 } 356 return nil 357 } 358 359 func lease(c context.Context, maxTasks int, queueName string, leaseTime int, groupByTag bool, tag []byte) ([]*Task, error) { 360 if queueName == "" { 361 queueName = "default" 362 } 363 req := &pb.TaskQueueQueryAndOwnTasksRequest{ 364 QueueName: []byte(queueName), 365 LeaseSeconds: proto.Float64(float64(leaseTime)), 366 MaxTasks: proto.Int64(int64(maxTasks)), 367 GroupByTag: proto.Bool(groupByTag), 368 Tag: tag, 369 } 370 res := &pb.TaskQueueQueryAndOwnTasksResponse{} 371 if err := internal.Call(c, "taskqueue", "QueryAndOwnTasks", req, res); err != nil { 372 return nil, err 373 } 374 tasks := make([]*Task, len(res.Task)) 375 for i, t := range res.Task { 376 tasks[i] = &Task{ 377 Payload: t.Body, 378 Name: string(t.TaskName), 379 Method: "PULL", 380 ETA: time.Unix(0, *t.EtaUsec*1e3), 381 RetryCount: *t.RetryCount, 382 Tag: string(t.Tag), 383 } 384 } 385 return tasks, nil 386 } 387 388 // Lease leases tasks from a queue. 389 // leaseTime is in seconds. 390 // The number of tasks fetched will be at most maxTasks. 391 func Lease(c context.Context, maxTasks int, queueName string, leaseTime int) ([]*Task, error) { 392 return lease(c, maxTasks, queueName, leaseTime, false, nil) 393 } 394 395 // LeaseByTag leases tasks from a queue, grouped by tag. 396 // If tag is empty, then the returned tasks are grouped by the tag of the task with earliest ETA. 397 // leaseTime is in seconds. 398 // The number of tasks fetched will be at most maxTasks. 399 func LeaseByTag(c context.Context, maxTasks int, queueName string, leaseTime int, tag string) ([]*Task, error) { 400 return lease(c, maxTasks, queueName, leaseTime, true, []byte(tag)) 401 } 402 403 // Purge removes all tasks from a queue. 404 func Purge(c context.Context, queueName string) error { 405 if queueName == "" { 406 queueName = "default" 407 } 408 req := &pb.TaskQueuePurgeQueueRequest{ 409 QueueName: []byte(queueName), 410 } 411 res := &pb.TaskQueuePurgeQueueResponse{} 412 return internal.Call(c, "taskqueue", "PurgeQueue", req, res) 413 } 414 415 // ModifyLease modifies the lease of a task. 416 // Used to request more processing time, or to abandon processing. 417 // leaseTime is in seconds and must not be negative. 418 func ModifyLease(c context.Context, task *Task, queueName string, leaseTime int) error { 419 if queueName == "" { 420 queueName = "default" 421 } 422 req := &pb.TaskQueueModifyTaskLeaseRequest{ 423 QueueName: []byte(queueName), 424 TaskName: []byte(task.Name), 425 EtaUsec: proto.Int64(task.ETA.UnixNano() / 1e3), // Used to verify ownership. 426 LeaseSeconds: proto.Float64(float64(leaseTime)), 427 } 428 res := &pb.TaskQueueModifyTaskLeaseResponse{} 429 if err := internal.Call(c, "taskqueue", "ModifyTaskLease", req, res); err != nil { 430 return err 431 } 432 task.ETA = time.Unix(0, *res.UpdatedEtaUsec*1e3) 433 return nil 434 } 435 436 // QueueStatistics represents statistics about a single task queue. 437 type QueueStatistics struct { 438 Tasks int // may be an approximation 439 OldestETA time.Time // zero if there are no pending tasks 440 441 Executed1Minute int // tasks executed in the last minute 442 InFlight int // tasks executing now 443 EnforcedRate float64 // requests per second 444 } 445 446 // QueueStats retrieves statistics about queues. 447 func QueueStats(c context.Context, queueNames []string) ([]QueueStatistics, error) { 448 req := &pb.TaskQueueFetchQueueStatsRequest{ 449 QueueName: make([][]byte, len(queueNames)), 450 } 451 for i, q := range queueNames { 452 if q == "" { 453 q = "default" 454 } 455 req.QueueName[i] = []byte(q) 456 } 457 res := &pb.TaskQueueFetchQueueStatsResponse{} 458 if err := internal.Call(c, "taskqueue", "FetchQueueStats", req, res); err != nil { 459 return nil, err 460 } 461 qs := make([]QueueStatistics, len(res.Queuestats)) 462 for i, qsg := range res.Queuestats { 463 qs[i] = QueueStatistics{ 464 Tasks: int(*qsg.NumTasks), 465 } 466 if eta := *qsg.OldestEtaUsec; eta > -1 { 467 qs[i].OldestETA = time.Unix(0, eta*1e3) 468 } 469 if si := qsg.ScannerInfo; si != nil { 470 qs[i].Executed1Minute = int(*si.ExecutedLastMinute) 471 qs[i].InFlight = int(si.GetRequestsInFlight()) 472 qs[i].EnforcedRate = si.GetEnforcedRate() 473 } 474 } 475 return qs, nil 476 } 477 478 func setTransaction(x *pb.TaskQueueAddRequest, t *dspb.Transaction) { 479 x.Transaction = t 480 } 481 482 func init() { 483 internal.RegisterErrorCodeMap("taskqueue", pb.TaskQueueServiceError_ErrorCode_name) 484 485 // Datastore error codes are shifted by DATASTORE_ERROR when presented through taskqueue. 486 dsCode := int32(pb.TaskQueueServiceError_DATASTORE_ERROR) + int32(dspb.Error_TIMEOUT) 487 internal.RegisterTimeoutErrorCode("taskqueue", dsCode) 488 489 // Transaction registration. 490 internal.RegisterTransactionSetter(setTransaction) 491 internal.RegisterTransactionSetter(func(x *pb.TaskQueueBulkAddRequest, t *dspb.Transaction) { 492 for _, req := range x.AddRequest { 493 setTransaction(req, t) 494 } 495 }) 496 }