github.com/maier/nomad@v0.4.1-0.20161110003312-a9e3d0b8549d/api/fs.go (about)

     1  package api
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"io"
     7  	"strconv"
     8  	"sync"
     9  	"time"
    10  )
    11  
    12  const (
    13  	// OriginStart and OriginEnd are the available parameters for the origin
    14  	// argument when streaming a file. They respectively offset from the start
    15  	// and end of a file.
    16  	OriginStart = "start"
    17  	OriginEnd   = "end"
    18  )
    19  
    20  // AllocFileInfo holds information about a file inside the AllocDir
    21  type AllocFileInfo struct {
    22  	Name     string
    23  	IsDir    bool
    24  	Size     int64
    25  	FileMode string
    26  	ModTime  time.Time
    27  }
    28  
    29  // StreamFrame is used to frame data of a file when streaming
    30  type StreamFrame struct {
    31  	Offset    int64  `json:",omitempty"`
    32  	Data      []byte `json:",omitempty"`
    33  	File      string `json:",omitempty"`
    34  	FileEvent string `json:",omitempty"`
    35  }
    36  
    37  // IsHeartbeat returns if the frame is a heartbeat frame
    38  func (s *StreamFrame) IsHeartbeat() bool {
    39  	return len(s.Data) == 0 && s.FileEvent == "" && s.File == "" && s.Offset == 0
    40  }
    41  
    42  // AllocFS is used to introspect an allocation directory on a Nomad client
    43  type AllocFS struct {
    44  	client *Client
    45  }
    46  
    47  // AllocFS returns an handle to the AllocFS endpoints
    48  func (c *Client) AllocFS() *AllocFS {
    49  	return &AllocFS{client: c}
    50  }
    51  
    52  // getNodeClient returns a Client that will dial the node. If the QueryOptions
    53  // is set, the function will ensure that it is initalized and that the Params
    54  // field is valid.
    55  func (a *AllocFS) getNodeClient(node *Node, allocID string, q **QueryOptions) (*Client, error) {
    56  	if node.HTTPAddr == "" {
    57  		return nil, fmt.Errorf("http addr of the node where alloc %q is running is not advertised", allocID)
    58  	}
    59  
    60  	// Get an API client for the node
    61  	nodeClient, err := NewClient(a.client.config.CopyConfig(node.HTTPAddr, node.TLSEnabled))
    62  	if err != nil {
    63  		return nil, err
    64  	}
    65  
    66  	// Set the query params
    67  	if q == nil {
    68  		return nodeClient, nil
    69  	}
    70  
    71  	if *q == nil {
    72  		*q = &QueryOptions{}
    73  	}
    74  	if actQ := *q; actQ.Params == nil {
    75  		actQ.Params = make(map[string]string)
    76  	}
    77  	return nodeClient, nil
    78  }
    79  
    80  // List is used to list the files at a given path of an allocation directory
    81  func (a *AllocFS) List(alloc *Allocation, path string, q *QueryOptions) ([]*AllocFileInfo, *QueryMeta, error) {
    82  	node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
    83  	if err != nil {
    84  		return nil, nil, err
    85  	}
    86  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
    87  	if err != nil {
    88  		return nil, nil, err
    89  	}
    90  	q.Params["path"] = path
    91  
    92  	var resp []*AllocFileInfo
    93  	qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/ls/%s", alloc.ID), &resp, q)
    94  	if err != nil {
    95  		return nil, nil, err
    96  	}
    97  
    98  	return resp, qm, nil
    99  }
   100  
   101  // Stat is used to stat a file at a given path of an allocation directory
   102  func (a *AllocFS) Stat(alloc *Allocation, path string, q *QueryOptions) (*AllocFileInfo, *QueryMeta, error) {
   103  	node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
   104  	if err != nil {
   105  		return nil, nil, err
   106  	}
   107  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
   108  	if err != nil {
   109  		return nil, nil, err
   110  	}
   111  	q.Params["path"] = path
   112  
   113  	var resp AllocFileInfo
   114  	qm, err := nodeClient.query(fmt.Sprintf("/v1/client/fs/stat/%s", alloc.ID), &resp, q)
   115  	if err != nil {
   116  		return nil, nil, err
   117  	}
   118  	return &resp, qm, nil
   119  }
   120  
   121  // ReadAt is used to read bytes at a given offset until limit at the given path
   122  // in an allocation directory. If limit is <= 0, there is no limit.
   123  func (a *AllocFS) ReadAt(alloc *Allocation, path string, offset int64, limit int64, q *QueryOptions) (io.ReadCloser, error) {
   124  	node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
   125  	if err != nil {
   126  		return nil, err
   127  	}
   128  
   129  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
   130  	if err != nil {
   131  		return nil, err
   132  	}
   133  	q.Params["path"] = path
   134  	q.Params["offset"] = strconv.FormatInt(offset, 10)
   135  	q.Params["limit"] = strconv.FormatInt(limit, 10)
   136  
   137  	r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/readat/%s", alloc.ID), q)
   138  	if err != nil {
   139  		return nil, err
   140  	}
   141  	return r, nil
   142  }
   143  
   144  // Cat is used to read contents of a file at the given path in an allocation
   145  // directory
   146  func (a *AllocFS) Cat(alloc *Allocation, path string, q *QueryOptions) (io.ReadCloser, error) {
   147  	node, _, err := a.client.Nodes().Info(alloc.NodeID, &QueryOptions{})
   148  	if err != nil {
   149  		return nil, err
   150  	}
   151  
   152  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
   153  	if err != nil {
   154  		return nil, err
   155  	}
   156  	q.Params["path"] = path
   157  
   158  	r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/cat/%s", alloc.ID), q)
   159  	if err != nil {
   160  		return nil, err
   161  	}
   162  	return r, nil
   163  }
   164  
   165  // Stream streams the content of a file blocking on EOF.
   166  // The parameters are:
   167  // * path: path to file to stream.
   168  // * offset: The offset to start streaming data at.
   169  // * origin: Either "start" or "end" and defines from where the offset is applied.
   170  // * cancel: A channel that when closed, streaming will end.
   171  //
   172  // The return value is a channel that will emit StreamFrames as they are read.
   173  func (a *AllocFS) Stream(alloc *Allocation, path, origin string, offset int64,
   174  	cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) {
   175  
   176  	node, _, err := a.client.Nodes().Info(alloc.NodeID, q)
   177  	if err != nil {
   178  		return nil, err
   179  	}
   180  
   181  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
   182  	if err != nil {
   183  		return nil, err
   184  	}
   185  	q.Params["path"] = path
   186  	q.Params["offset"] = strconv.FormatInt(offset, 10)
   187  	q.Params["origin"] = origin
   188  
   189  	r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/stream/%s", alloc.ID), q)
   190  	if err != nil {
   191  		return nil, err
   192  	}
   193  
   194  	// Create the output channel
   195  	frames := make(chan *StreamFrame, 10)
   196  
   197  	go func() {
   198  		// Close the body
   199  		defer r.Close()
   200  
   201  		// Create a decoder
   202  		dec := json.NewDecoder(r)
   203  
   204  		for {
   205  			// Check if we have been cancelled
   206  			select {
   207  			case <-cancel:
   208  				return
   209  			default:
   210  			}
   211  
   212  			// Decode the next frame
   213  			var frame StreamFrame
   214  			if err := dec.Decode(&frame); err != nil {
   215  				close(frames)
   216  				return
   217  			}
   218  
   219  			// Discard heartbeat frames
   220  			if frame.IsHeartbeat() {
   221  				continue
   222  			}
   223  
   224  			frames <- &frame
   225  		}
   226  	}()
   227  
   228  	return frames, nil
   229  }
   230  
   231  // Logs streams the content of a tasks logs blocking on EOF.
   232  // The parameters are:
   233  // * allocation: the allocation to stream from.
   234  // * follow: Whether the logs should be followed.
   235  // * task: the tasks name to stream logs for.
   236  // * logType: Either "stdout" or "stderr"
   237  // * origin: Either "start" or "end" and defines from where the offset is applied.
   238  // * offset: The offset to start streaming data at.
   239  // * cancel: A channel that when closed, streaming will end.
   240  //
   241  // The return value is a channel that will emit StreamFrames as they are read.
   242  func (a *AllocFS) Logs(alloc *Allocation, follow bool, task, logType, origin string,
   243  	offset int64, cancel <-chan struct{}, q *QueryOptions) (<-chan *StreamFrame, error) {
   244  
   245  	node, _, err := a.client.Nodes().Info(alloc.NodeID, q)
   246  	if err != nil {
   247  		return nil, err
   248  	}
   249  
   250  	nodeClient, err := a.getNodeClient(node, alloc.ID, &q)
   251  	if err != nil {
   252  		return nil, err
   253  	}
   254  	q.Params["follow"] = strconv.FormatBool(follow)
   255  	q.Params["task"] = task
   256  	q.Params["type"] = logType
   257  	q.Params["origin"] = origin
   258  	q.Params["offset"] = strconv.FormatInt(offset, 10)
   259  
   260  	r, err := nodeClient.rawQuery(fmt.Sprintf("/v1/client/fs/logs/%s", alloc.ID), q)
   261  	if err != nil {
   262  		return nil, err
   263  	}
   264  
   265  	// Create the output channel
   266  	frames := make(chan *StreamFrame, 10)
   267  
   268  	go func() {
   269  		// Close the body
   270  		defer r.Close()
   271  
   272  		// Create a decoder
   273  		dec := json.NewDecoder(r)
   274  
   275  		for {
   276  			// Check if we have been cancelled
   277  			select {
   278  			case <-cancel:
   279  				return
   280  			default:
   281  			}
   282  
   283  			// Decode the next frame
   284  			var frame StreamFrame
   285  			if err := dec.Decode(&frame); err != nil {
   286  				close(frames)
   287  				return
   288  			}
   289  
   290  			// Discard heartbeat frames
   291  			if frame.IsHeartbeat() {
   292  				continue
   293  			}
   294  
   295  			frames <- &frame
   296  		}
   297  	}()
   298  
   299  	return frames, nil
   300  }
   301  
   302  // FrameReader is used to convert a stream of frames into a read closer.
   303  type FrameReader struct {
   304  	frames   <-chan *StreamFrame
   305  	cancelCh chan struct{}
   306  
   307  	closedLock sync.Mutex
   308  	closed     bool
   309  
   310  	unblockTime time.Duration
   311  
   312  	frame       *StreamFrame
   313  	frameOffset int
   314  
   315  	byteOffset int
   316  }
   317  
   318  // NewFrameReader takes a channel of frames and returns a FrameReader which
   319  // implements io.ReadCloser
   320  func NewFrameReader(frames <-chan *StreamFrame, cancelCh chan struct{}) *FrameReader {
   321  	return &FrameReader{
   322  		frames:   frames,
   323  		cancelCh: cancelCh,
   324  	}
   325  }
   326  
   327  // SetUnblockTime sets the time to unblock and return zero bytes read. If the
   328  // duration is unset or is zero or less, the read will block til data is read.
   329  func (f *FrameReader) SetUnblockTime(d time.Duration) {
   330  	f.unblockTime = d
   331  }
   332  
   333  // Offset returns the offset into the stream.
   334  func (f *FrameReader) Offset() int {
   335  	return f.byteOffset
   336  }
   337  
   338  // Read reads the data of the incoming frames into the bytes buffer. Returns EOF
   339  // when there are no more frames.
   340  func (f *FrameReader) Read(p []byte) (n int, err error) {
   341  	f.closedLock.Lock()
   342  	closed := f.closed
   343  	f.closedLock.Unlock()
   344  	if closed {
   345  		return 0, io.EOF
   346  	}
   347  
   348  	if f.frame == nil {
   349  		var unblock <-chan time.Time
   350  		if f.unblockTime.Nanoseconds() > 0 {
   351  			unblock = time.After(f.unblockTime)
   352  		}
   353  
   354  		select {
   355  		case frame, ok := <-f.frames:
   356  			if !ok {
   357  				return 0, io.EOF
   358  			}
   359  			f.frame = frame
   360  
   361  			// Store the total offset into the file
   362  			f.byteOffset = int(f.frame.Offset)
   363  		case <-unblock:
   364  			return 0, nil
   365  		case <-f.cancelCh:
   366  			return 0, io.EOF
   367  		}
   368  	}
   369  
   370  	// Copy the data out of the frame and update our offset
   371  	n = copy(p, f.frame.Data[f.frameOffset:])
   372  	f.frameOffset += n
   373  
   374  	// Clear the frame and its offset once we have read everything
   375  	if len(f.frame.Data) == f.frameOffset {
   376  		f.frame = nil
   377  		f.frameOffset = 0
   378  	}
   379  
   380  	return n, nil
   381  }
   382  
   383  // Close cancels the stream of frames
   384  func (f *FrameReader) Close() error {
   385  	f.closedLock.Lock()
   386  	defer f.closedLock.Unlock()
   387  	if f.closed {
   388  		return nil
   389  	}
   390  
   391  	close(f.cancelCh)
   392  	f.closed = true
   393  	return nil
   394  }