github.com/maier/nomad@v0.4.1-0.20161110003312-a9e3d0b8549d/api/fs.go (about) 1 package api 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "strconv" 8 "sync" 9 "time" 10 ) 11 12 const ( 13 // OriginStart and OriginEnd are the available parameters for the origin 14 // argument when streaming a file. They respectively offset from the start 15 // and end of a file. 16 OriginStart = "start" 17 OriginEnd = "end" 18 ) 19 20 // AllocFileInfo holds information about a file inside the AllocDir 21 type AllocFileInfo struct { 22 Name string 23 IsDir bool 24 Size int64 25 FileMode string 26 ModTime time.Time 27 } 28 29 // StreamFrame is used to frame data of a file when streaming 30 type StreamFrame struct { 31 Offset int64 `json:",omitempty"` 32 Data []byte `json:",omitempty"` 33 File string `json:",omitempty"` 34 FileEvent string `json:",omitempty"` 35 } 36 37 // IsHeartbeat returns if the frame is a heartbeat frame 38 func (s *StreamFrame) IsHeartbeat() bool { 39 return len(s.Data) == 0 && s.FileEvent == "" && s.File == "" && s.Offset == 0 40 } 41 42 // AllocFS is used to introspect an allocation directory on a Nomad client 43 type AllocFS struct { 44 client *Client 45 } 46 47 // AllocFS returns an handle to the AllocFS endpoints 48 func (c *Client) AllocFS() *AllocFS { 49 return &AllocFS{client: c} 50 } 51 52 // getNodeClient returns a Client that will dial the node. If the QueryOptions 53 // is set, the function will ensure that it is initalized and that the Params 54 // field is valid. 55 func (a *AllocFS) getNodeClient(node *Node, allocID string, q **QueryOptions) (*Client, error) { 56 if node.HTTPAddr == "" { 57 return nil, fmt.Errorf("http addr of the node where alloc %q is running is not advertised", allocID) 58 } 59 60 // Get an API client for the node 61 nodeClient, err := NewClient(a.client.config.CopyConfig(node.HTTPAddr, node.TLSEnabled)) 62 if err != nil { 63 return nil, err 64 } 65 66 // Set the query params 67 if q == nil { 68 return nodeClient, nil 69 } 70 71 if *q == nil { 72 *q = &QueryOptions{} 73 } 74 if actQ := *q; actQ.Params == nil { 75 actQ.Params = make(map[string]string) 76 } 77 return nodeClient, nil 78 } 79 80 // List is used to list the files at a given path of an allocation directory 81 func (a *AllocFS) List(alloc *Allocation, path string, q *QueryOptions) ([]*AllocFileInfo, *QueryMeta, error) { 82 node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{}) 83 if err != nil { 84 return nil, nil, err 85 } 86 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 87 if err != nil { 88 return nil, nil, err 89 } 90 q.Params["path"] = path 91 92 var resp []*AllocFileInfo 93 qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/ls/%s", alloc.ID), &resp, q) 94 if err != nil { 95 return nil, nil, err 96 } 97 98 return resp, qm, nil 99 } 100 101 // Stat is used to stat a file at a given path of an allocation directory 102 func (a *AllocFS) Stat(alloc *Allocation, path string, q *QueryOptions) (*AllocFileInfo, *QueryMeta, error) { 103 node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{}) 104 if err != nil { 105 return nil, nil, err 106 } 107 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 108 if err != nil { 109 return nil, nil, err 110 } 111 q.Params["path"] = path 112 113 var resp AllocFileInfo 114 qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/stat/%s", alloc.ID), &resp, q) 115 if err != nil { 116 return nil, nil, err 117 } 118 return &resp, qm, nil 119 } 120 121 // ReadAt is used to read bytes at a given offset until limit at the given path 122 // in an allocation directory. If limit is <= 0, there is no limit. 123 func (a *AllocFS) ReadAt(alloc *Allocation, path string, offset int64, limit int64, q *QueryOptions) (io.ReadCloser, error) { 124 node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{}) 125 if err != nil { 126 return nil, err 127 } 128 129 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 130 if err != nil { 131 return nil, err 132 } 133 q.Params["path"] = path 134 q.Params["offset"] = strconv.FormatInt(offset, 10) 135 q.Params["limit"] = strconv.FormatInt(limit, 10) 136 137 r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/readat/%s", alloc.ID), q) 138 if err != nil { 139 return nil, err 140 } 141 return r, nil 142 } 143 144 // Cat is used to read contents of a file at the given path in an allocation 145 // directory 146 func (a *AllocFS) Cat(alloc *Allocation, path string, q *QueryOptions) (io.ReadCloser, error) { 147 node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{}) 148 if err != nil { 149 return nil, err 150 } 151 152 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 153 if err != nil { 154 return nil, err 155 } 156 q.Params["path"] = path 157 158 r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/cat/%s", alloc.ID), q) 159 if err != nil { 160 return nil, err 161 } 162 return r, nil 163 } 164 165 // Stream streams the content of a file blocking on EOF. 166 // The parameters are: 167 // * path: path to file to stream. 168 // * offset: The offset to start streaming data at. 169 // * origin: Either "start" or "end" and defines from where the offset is applied. 170 // * cancel: A channel that when closed, streaming will end. 171 // 172 // The return value is a channel that will emit StreamFrames as they are read. 173 func (a *AllocFS) Stream(alloc *Allocation, path, origin string, offset int64, 174 cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) { 175 176 node, _, err := a.client.Nodes().Info(alloc.NodeID, q) 177 if err != nil { 178 return nil, err 179 } 180 181 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 182 if err != nil { 183 return nil, err 184 } 185 q.Params["path"] = path 186 q.Params["offset"] = strconv.FormatInt(offset, 10) 187 q.Params["origin"] = origin 188 189 r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/stream/%s", alloc.ID), q) 190 if err != nil { 191 return nil, err 192 } 193 194 // Create the output channel 195 frames := make(chan *StreamFrame, 10) 196 197 go func() { 198 // Close the body 199 defer r.Close() 200 201 // Create a decoder 202 dec := json.NewDecoder(r) 203 204 for { 205 // Check if we have been cancelled 206 select { 207 case <-cancel: 208 return 209 default: 210 } 211 212 // Decode the next frame 213 var frame StreamFrame 214 if err := dec.Decode(&frame); err != nil { 215 close(frames) 216 return 217 } 218 219 // Discard heartbeat frames 220 if frame.IsHeartbeat() { 221 continue 222 } 223 224 frames <- &frame 225 } 226 }() 227 228 return frames, nil 229 } 230 231 // Logs streams the content of a tasks logs blocking on EOF. 232 // The parameters are: 233 // * allocation: the allocation to stream from. 234 // * follow: Whether the logs should be followed. 235 // * task: the tasks name to stream logs for. 236 // * logType: Either "stdout" or "stderr" 237 // * origin: Either "start" or "end" and defines from where the offset is applied. 238 // * offset: The offset to start streaming data at. 239 // * cancel: A channel that when closed, streaming will end. 240 // 241 // The return value is a channel that will emit StreamFrames as they are read. 242 func (a *AllocFS) Logs(alloc *Allocation, follow bool, task, logType, origin string, 243 offset int64, cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) { 244 245 node, _, err := a.client.Nodes().Info(alloc.NodeID, q) 246 if err != nil { 247 return nil, err 248 } 249 250 nodeClient, err := a.getNodeClient(node, alloc.ID, &q) 251 if err != nil { 252 return nil, err 253 } 254 q.Params["follow"] = strconv.FormatBool(follow) 255 q.Params["task"] = task 256 q.Params["type"] = logType 257 q.Params["origin"] = origin 258 q.Params["offset"] = strconv.FormatInt(offset, 10) 259 260 r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/logs/%s", alloc.ID), q) 261 if err != nil { 262 return nil, err 263 } 264 265 // Create the output channel 266 frames := make(chan *StreamFrame, 10) 267 268 go func() { 269 // Close the body 270 defer r.Close() 271 272 // Create a decoder 273 dec := json.NewDecoder(r) 274 275 for { 276 // Check if we have been cancelled 277 select { 278 case <-cancel: 279 return 280 default: 281 } 282 283 // Decode the next frame 284 var frame StreamFrame 285 if err := dec.Decode(&frame); err != nil { 286 close(frames) 287 return 288 } 289 290 // Discard heartbeat frames 291 if frame.IsHeartbeat() { 292 continue 293 } 294 295 frames <- &frame 296 } 297 }() 298 299 return frames, nil 300 } 301 302 // FrameReader is used to convert a stream of frames into a read closer. 303 type FrameReader struct { 304 frames <-chan *StreamFrame 305 cancelCh chan struct{} 306 307 closedLock sync.Mutex 308 closed bool 309 310 unblockTime time.Duration 311 312 frame *StreamFrame 313 frameOffset int 314 315 byteOffset int 316 } 317 318 // NewFrameReader takes a channel of frames and returns a FrameReader which 319 // implements io.ReadCloser 320 func NewFrameReader(frames <-chan *StreamFrame, cancelCh chan struct{}) *FrameReader { 321 return &FrameReader{ 322 frames: frames, 323 cancelCh: cancelCh, 324 } 325 } 326 327 // SetUnblockTime sets the time to unblock and return zero bytes read. If the 328 // duration is unset or is zero or less, the read will block til data is read. 329 func (f *FrameReader) SetUnblockTime(d time.Duration) { 330 f.unblockTime = d 331 } 332 333 // Offset returns the offset into the stream. 334 func (f *FrameReader) Offset() int { 335 return f.byteOffset 336 } 337 338 // Read reads the data of the incoming frames into the bytes buffer. Returns EOF 339 // when there are no more frames. 340 func (f *FrameReader) Read(p []byte) (n int, err error) { 341 f.closedLock.Lock() 342 closed := f.closed 343 f.closedLock.Unlock() 344 if closed { 345 return 0, io.EOF 346 } 347 348 if f.frame == nil { 349 var unblock <-chan time.Time 350 if f.unblockTime.Nanoseconds() > 0 { 351 unblock = time.After(f.unblockTime) 352 } 353 354 select { 355 case frame, ok := <-f.frames: 356 if !ok { 357 return 0, io.EOF 358 } 359 f.frame = frame 360 361 // Store the total offset into the file 362 f.byteOffset = int(f.frame.Offset) 363 case <-unblock: 364 return 0, nil 365 case <-f.cancelCh: 366 return 0, io.EOF 367 } 368 } 369 370 // Copy the data out of the frame and update our offset 371 n = copy(p, f.frame.Data[f.frameOffset:]) 372 f.frameOffset += n 373 374 // Clear the frame and its offset once we have read everything 375 if len(f.frame.Data) == f.frameOffset { 376 f.frame = nil 377 f.frameOffset = 0 378 } 379 380 return n, nil 381 } 382 383 // Close cancels the stream of frames 384 func (f *FrameReader) Close() error { 385 f.closedLock.Lock() 386 defer f.closedLock.Unlock() 387 if f.closed { 388 return nil 389 } 390 391 close(f.cancelCh) 392 f.closed = true 393 return nil 394 }