github.com/hspak/nomad@v0.7.2-0.20180309000617-bc4ae22a39a5/api/fs.go (about) 1 package api 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io" 7 "net" 8 "strconv" 9 "sync" 10 "time" 11 ) 12 13 const ( 14 // OriginStart and OriginEnd are the available parameters for the origin 15 // argument when streaming a file. They respectively offset from the start 16 // and end of a file. 17 OriginStart = "start" 18 OriginEnd = "end" 19 ) 20 21 // AllocFileInfo holds information about a file inside the AllocDir 22 type AllocFileInfo struct { 23 Name string 24 IsDir bool 25 Size int64 26 FileMode string 27 ModTime time.Time 28 } 29 30 // StreamFrame is used to frame data of a file when streaming 31 type StreamFrame struct { 32 Offset int64 `json:",omitempty"` 33 Data []byte `json:",omitempty"` 34 File string `json:",omitempty"` 35 FileEvent string `json:",omitempty"` 36 } 37 38 // IsHeartbeat returns if the frame is a heartbeat frame 39 func (s *StreamFrame) IsHeartbeat() bool { 40 return len(s.Data) == 0 && s.FileEvent == "" && s.File == "" && s.Offset == 0 41 } 42 43 // AllocFS is used to introspect an allocation directory on a Nomad client 44 type AllocFS struct { 45 client *Client 46 } 47 48 // AllocFS returns an handle to the AllocFS endpoints 49 func (c *Client) AllocFS() *AllocFS { 50 return &AllocFS{client: c} 51 } 52 53 // List is used to list the files at a given path of an allocation directory 54 func (a *AllocFS) List(alloc *Allocation, path string, q *QueryOptions) ([]*AllocFileInfo, *QueryMeta, error) { 55 if q == nil { 56 q = &QueryOptions{} 57 } 58 if q.Params == nil { 59 q.Params = make(map[string]string) 60 } 61 q.Params["path"] = path 62 63 var resp []*AllocFileInfo 64 qm, err := a.client.query(fmt.Sprintf("/v1/client/fs/ls/%s", alloc.ID), &resp, q) 65 if err != nil { 66 return nil, nil, err 67 } 68 69 return resp, qm, nil 70 } 71 72 // Stat is used to stat a file at a given path of an allocation directory 73 func (a *AllocFS) Stat(alloc *Allocation, path string, q *QueryOptions) (*AllocFileInfo, *QueryMeta, error) { 74 if q == nil { 75 q = &QueryOptions{} 76 } 77 if q.Params == nil { 78 q.Params = make(map[string]string) 79 } 80 81 q.Params["path"] = path 82 83 var resp AllocFileInfo 84 qm, err := a.client.query(fmt.Sprintf("/v1/client/fs/stat/%s", alloc.ID), &resp, q) 85 if err != nil { 86 return nil, nil, err 87 } 88 return &resp, qm, nil 89 } 90 91 // ReadAt is used to read bytes at a given offset until limit at the given path 92 // in an allocation directory. If limit is <= 0, there is no limit. 93 func (a *AllocFS) ReadAt(alloc *Allocation, path string, offset int64, limit int64, q *QueryOptions) (io.ReadCloser, error) { 94 nodeClient, err := a.client.GetNodeClientWithTimeout(alloc.NodeID, ClientConnTimeout, q) 95 if err != nil { 96 return nil, err 97 } 98 99 if q == nil { 100 q = &QueryOptions{} 101 } 102 if q.Params == nil { 103 q.Params = make(map[string]string) 104 } 105 106 q.Params["path"] = path 107 q.Params["offset"] = strconv.FormatInt(offset, 10) 108 q.Params["limit"] = strconv.FormatInt(limit, 10) 109 110 reqPath := fmt.Sprintf("/v1/client/fs/readat/%s", alloc.ID) 111 r, err := nodeClient.rawQuery(reqPath, q) 112 if err != nil { 113 // There was a networking error when talking directly to the client. 114 if _, ok := err.(net.Error); !ok { 115 return nil, err 116 } 117 118 // Try via the server 119 r, err = a.client.rawQuery(reqPath, q) 120 if err != nil { 121 return nil, err 122 } 123 } 124 125 return r, nil 126 } 127 128 // Cat is used to read contents of a file at the given path in an allocation 129 // directory 130 func (a *AllocFS) Cat(alloc *Allocation, path string, q *QueryOptions) (io.ReadCloser, error) { 131 nodeClient, err := a.client.GetNodeClientWithTimeout(alloc.NodeID, ClientConnTimeout, q) 132 if err != nil { 133 return nil, err 134 } 135 136 if q == nil { 137 q = &QueryOptions{} 138 } 139 if q.Params == nil { 140 q.Params = make(map[string]string) 141 } 142 143 q.Params["path"] = path 144 reqPath := fmt.Sprintf("/v1/client/fs/cat/%s", alloc.ID) 145 r, err := nodeClient.rawQuery(reqPath, q) 146 if err != nil { 147 // There was a networking error when talking directly to the client. 148 if _, ok := err.(net.Error); !ok { 149 return nil, err 150 } 151 152 // Try via the server 153 r, err = a.client.rawQuery(reqPath, q) 154 if err != nil { 155 return nil, err 156 } 157 } 158 159 return r, nil 160 } 161 162 // Stream streams the content of a file blocking on EOF. 163 // The parameters are: 164 // * path: path to file to stream. 165 // * offset: The offset to start streaming data at. 166 // * origin: Either "start" or "end" and defines from where the offset is applied. 167 // * cancel: A channel that when closed, streaming will end. 168 // 169 // The return value is a channel that will emit StreamFrames as they are read. 170 func (a *AllocFS) Stream(alloc *Allocation, path, origin string, offset int64, 171 cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, <-chan error) { 172 173 errCh := make(chan error, 1) 174 nodeClient, err := a.client.GetNodeClientWithTimeout(alloc.NodeID, ClientConnTimeout, q) 175 if err != nil { 176 errCh <- err 177 return nil, errCh 178 } 179 180 if q == nil { 181 q = &QueryOptions{} 182 } 183 if q.Params == nil { 184 q.Params = make(map[string]string) 185 } 186 187 q.Params["path"] = path 188 q.Params["offset"] = strconv.FormatInt(offset, 10) 189 q.Params["origin"] = origin 190 191 reqPath := fmt.Sprintf("/v1/client/fs/stream/%s", alloc.ID) 192 r, err := nodeClient.rawQuery(reqPath, q) 193 if err != nil { 194 // There was a networking error when talking directly to the client. 195 if _, ok := err.(net.Error); !ok { 196 errCh <- err 197 return nil, errCh 198 } 199 200 // Try via the server 201 r, err = a.client.rawQuery(reqPath, q) 202 if err != nil { 203 errCh <- err 204 return nil, errCh 205 } 206 } 207 208 // Create the output channel 209 frames := make(chan *StreamFrame, 10) 210 211 go func() { 212 // Close the body 213 defer r.Close() 214 215 // Create a decoder 216 dec := json.NewDecoder(r) 217 218 for { 219 // Check if we have been cancelled 220 select { 221 case <-cancel: 222 return 223 default: 224 } 225 226 // Decode the next frame 227 var frame StreamFrame 228 if err := dec.Decode(&frame); err != nil { 229 errCh <- err 230 close(frames) 231 return 232 } 233 234 // Discard heartbeat frames 235 if frame.IsHeartbeat() { 236 continue 237 } 238 239 frames <- &frame 240 } 241 }() 242 243 return frames, errCh 244 } 245 246 // Logs streams the content of a tasks logs blocking on EOF. 247 // The parameters are: 248 // * allocation: the allocation to stream from. 249 // * follow: Whether the logs should be followed. 250 // * task: the tasks name to stream logs for. 251 // * logType: Either "stdout" or "stderr" 252 // * origin: Either "start" or "end" and defines from where the offset is applied. 253 // * offset: The offset to start streaming data at. 254 // * cancel: A channel that when closed, streaming will end. 255 // 256 // The return value is a channel that will emit StreamFrames as they are read. 257 func (a *AllocFS) Logs(alloc *Allocation, follow bool, task, logType, origin string, 258 offset int64, cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, <-chan error) { 259 260 errCh := make(chan error, 1) 261 nodeClient, err := a.client.GetNodeClientWithTimeout(alloc.NodeID, ClientConnTimeout, q) 262 if err != nil { 263 errCh <- err 264 return nil, errCh 265 } 266 267 if q == nil { 268 q = &QueryOptions{} 269 } 270 if q.Params == nil { 271 q.Params = make(map[string]string) 272 } 273 274 q.Params["follow"] = strconv.FormatBool(follow) 275 q.Params["task"] = task 276 q.Params["type"] = logType 277 q.Params["origin"] = origin 278 q.Params["offset"] = strconv.FormatInt(offset, 10) 279 280 reqPath := fmt.Sprintf("/v1/client/fs/logs/%s", alloc.ID) 281 r, err := nodeClient.rawQuery(reqPath, q) 282 if err != nil { 283 // There was a networking error when talking directly to the client. 284 if _, ok := err.(net.Error); !ok { 285 errCh <- err 286 return nil, errCh 287 } 288 289 // Try via the server 290 r, err = a.client.rawQuery(reqPath, q) 291 if err != nil { 292 errCh <- err 293 return nil, errCh 294 } 295 } 296 297 // Create the output channel 298 frames := make(chan *StreamFrame, 10) 299 300 go func() { 301 // Close the body 302 defer r.Close() 303 304 // Create a decoder 305 dec := json.NewDecoder(r) 306 307 for { 308 // Check if we have been cancelled 309 select { 310 case <-cancel: 311 return 312 default: 313 } 314 315 // Decode the next frame 316 var frame StreamFrame 317 if err := dec.Decode(&frame); err != nil { 318 errCh <- err 319 close(frames) 320 return 321 } 322 323 // Discard heartbeat frames 324 if frame.IsHeartbeat() { 325 continue 326 } 327 328 frames <- &frame 329 } 330 }() 331 332 return frames, errCh 333 } 334 335 // FrameReader is used to convert a stream of frames into a read closer. 336 type FrameReader struct { 337 frames <-chan *StreamFrame 338 errCh <-chan error 339 cancelCh chan struct{} 340 341 closedLock sync.Mutex 342 closed bool 343 344 unblockTime time.Duration 345 346 frame *StreamFrame 347 frameOffset int 348 349 byteOffset int 350 } 351 352 // NewFrameReader takes a channel of frames and returns a FrameReader which 353 // implements io.ReadCloser 354 func NewFrameReader(frames <-chan *StreamFrame, errCh <-chan error, cancelCh chan struct{}) *FrameReader { 355 return &FrameReader{ 356 frames: frames, 357 errCh: errCh, 358 cancelCh: cancelCh, 359 } 360 } 361 362 // SetUnblockTime sets the time to unblock and return zero bytes read. If the 363 // duration is unset or is zero or less, the read will block til data is read. 364 func (f *FrameReader) SetUnblockTime(d time.Duration) { 365 f.unblockTime = d 366 } 367 368 // Offset returns the offset into the stream. 369 func (f *FrameReader) Offset() int { 370 return f.byteOffset 371 } 372 373 // Read reads the data of the incoming frames into the bytes buffer. Returns EOF 374 // when there are no more frames. 375 func (f *FrameReader) Read(p []byte) (n int, err error) { 376 f.closedLock.Lock() 377 closed := f.closed 378 f.closedLock.Unlock() 379 if closed { 380 return 0, io.EOF 381 } 382 383 if f.frame == nil { 384 var unblock <-chan time.Time 385 if f.unblockTime.Nanoseconds() > 0 { 386 unblock = time.After(f.unblockTime) 387 } 388 389 select { 390 case frame, ok := <-f.frames: 391 if !ok { 392 return 0, io.EOF 393 } 394 f.frame = frame 395 396 // Store the total offset into the file 397 f.byteOffset = int(f.frame.Offset) 398 case <-unblock: 399 return 0, nil 400 case err := <-f.errCh: 401 return 0, err 402 case <-f.cancelCh: 403 return 0, io.EOF 404 } 405 } 406 407 // Copy the data out of the frame and update our offset 408 n = copy(p, f.frame.Data[f.frameOffset:]) 409 f.frameOffset += n 410 411 // Clear the frame and its offset once we have read everything 412 if len(f.frame.Data) == f.frameOffset { 413 f.frame = nil 414 f.frameOffset = 0 415 } 416 417 return n, nil 418 } 419 420 // Close cancels the stream of frames 421 func (f *FrameReader) Close() error { 422 f.closedLock.Lock() 423 defer f.closedLock.Unlock() 424 if f.closed { 425 return nil 426 } 427 428 close(f.cancelCh) 429 f.closed = true 430 return nil 431 }