github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/spyglass/lenses/lenses.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package lenses provides interfaces and methods necessary for implementing custom artifact viewers
    18  package lenses
    19  
    20  import (
    21  	"bufio"
    22  	"bytes"
    23  	"errors"
    24  	"fmt"
    25  	"github.com/sirupsen/logrus"
    26  	"io"
    27  	"path/filepath"
    28  )
    29  
    30  var (
    31  	lensReg = map[string]Lens{}
    32  
    33  	// ErrGzipOffsetRead will be thrown when an offset read is attempted on a gzip-compressed object
    34  	ErrGzipOffsetRead = errors.New("offset read on gzipped files unsupported")
    35  	// ErrInvalidLensName will be thrown when a viewer method is called on a view name that has not
    36  	// been registered. Ensure your viewer is registered using RegisterViewer and that you are
    37  	// providing the correct viewer name.
    38  	ErrInvalidLensName = errors.New("invalid lens name")
    39  	// ErrFileTooLarge will be thrown when a size-limited operation (ex. ReadAll) is called on an
    40  	// artifact whose size exceeds the configured limit.
    41  	ErrFileTooLarge = errors.New("file size over specified limit")
    42  	// ErrContextUnsupported is thrown when attempting to use a context with an artifact that
    43  	// does not support context operations (cancel, withtimeout, etc.)
    44  	ErrContextUnsupported = errors.New("artifact does not support context operations")
    45  )
    46  
    47  // Lens defines the interface that lenses are required to implement in order to be used by Spyglass.
    48  type Lens interface {
    49  	// Name returns the name of the lens. It must match the package name.
    50  	Name() string
    51  	// Title returns a human-readable title for the lens.
    52  	Title() string
    53  	// Priority returns a number used to sort viewers. Lower is more important.
    54  	Priority() int
    55  	// Header returns a a string that is injected into the rendered lens's <head>
    56  	Header(artifacts []Artifact, resourceDir string) string
    57  	// Body returns a string that is initially injected into the rendered lens's <body>.
    58  	// The lens's front-end code may call back to Body again, passing in some data string of its choosing.
    59  	Body(artifacts []Artifact, resourceDir string, data string) string
    60  	// Callback receives a string sent by the lens's front-end code and returns another string to be returned
    61  	// to that frontend code.
    62  	Callback(artifacts []Artifact, resourceDir string, data string) string
    63  }
    64  
    65  // Artifact represents some output of a prow job
    66  type Artifact interface {
    67  	// ReadAt reads len(p) bytes of the artifact at offset off. (unsupported on some compressed files)
    68  	ReadAt(p []byte, off int64) (n int, err error)
    69  	// ReadAtMost reads at most n bytes from the beginning of the artifact
    70  	ReadAtMost(n int64) ([]byte, error)
    71  	// CanonicalLink gets a link to viewing this artifact in storage
    72  	CanonicalLink() string
    73  	// JobPath is the path to the artifact within the job (i.e. without the job prefix)
    74  	JobPath() string
    75  	// ReadAll reads all bytes from the artifact up to a limit specified by the artifact
    76  	ReadAll() ([]byte, error)
    77  	// ReadTail reads the last n bytes from the artifact (unsupported on some compressed files)
    78  	ReadTail(n int64) ([]byte, error)
    79  	// Size gets the size of the artifact in bytes, may make a network call
    80  	Size() (int64, error)
    81  }
    82  
    83  // ResourceDirForLens returns the path to a lens's public resource directory.
    84  func ResourceDirForLens(baseDir, name string) string {
    85  	return filepath.Join(baseDir, name)
    86  }
    87  
    88  // RegisterLens registers new viewers
    89  func RegisterLens(lens Lens) error {
    90  	_, ok := lensReg[lens.Name()]
    91  	if ok {
    92  		return fmt.Errorf("viewer already registered with name %s", lens.Name())
    93  	}
    94  
    95  	if lens.Title() == "" {
    96  		return errors.New("empty title field in view metadata")
    97  	}
    98  	if lens.Priority() < 0 {
    99  		return errors.New("priority must be >=0")
   100  	}
   101  	lensReg[lens.Name()] = lens
   102  	logrus.Infof("Spyglass registered viewer %s with title %s.", lens.Name(), lens.Title())
   103  	return nil
   104  }
   105  
   106  // GetLens returns a Lens by name, if it exists; otherwise it returns an error.
   107  func GetLens(name string) (Lens, error) {
   108  	lens, ok := lensReg[name]
   109  	if !ok {
   110  		return nil, ErrInvalidLensName
   111  	}
   112  	return lens, nil
   113  }
   114  
   115  // UnregisterLens unregisters lenses
   116  func UnregisterLens(viewerName string) {
   117  	delete(lensReg, viewerName)
   118  	logrus.Infof("Spyglass unregistered viewer %s.", viewerName)
   119  }
   120  
   121  // LastNLines reads the last n lines from an artifact.
   122  func LastNLines(a Artifact, n int64) ([]string, error) {
   123  	// 300B, a reasonable log line length, probably a bit more scalable than a hard-coded value
   124  	return LastNLinesChunked(a, n, 300*n+1)
   125  }
   126  
   127  // LastNLinesChunked reads the last n lines from an artifact by reading chunks of size chunkSize
   128  // from the end of the artifact. Best performance is achieved by:
   129  // argmin 0<chunkSize<INTMAX, f(chunkSize) = chunkSize - n * avgLineLength
   130  func LastNLinesChunked(a Artifact, n, chunkSize int64) ([]string, error) {
   131  	toRead := chunkSize + 1 // Add 1 for exclusive upper bound read range
   132  	chunks := int64(1)
   133  	var contents []byte
   134  	var linesInContents int64
   135  	artifactSize, err := a.Size()
   136  	if err != nil {
   137  		return nil, fmt.Errorf("error getting artifact size: %v", err)
   138  	}
   139  	offset := artifactSize - chunks*chunkSize
   140  	lastOffset := offset
   141  	var lastRead int64
   142  	for linesInContents < n && offset != 0 {
   143  		offset = lastOffset - lastRead
   144  		if offset < 0 {
   145  			toRead = offset + chunkSize + 1
   146  			offset = 0
   147  		}
   148  		bytesRead := make([]byte, toRead)
   149  		numBytesRead, err := a.ReadAt(bytesRead, offset)
   150  		if err != nil && err != io.EOF {
   151  			return nil, fmt.Errorf("error reading artifact: %v", err)
   152  		}
   153  		lastRead = int64(numBytesRead)
   154  		lastOffset = offset
   155  		bytesRead = bytes.Trim(bytesRead, "\x00")
   156  		linesInContents += int64(bytes.Count(bytesRead, []byte("\n")))
   157  		contents = append(bytesRead, contents...)
   158  		chunks++
   159  	}
   160  
   161  	var lines []string
   162  	scanner := bufio.NewScanner(bytes.NewReader(contents))
   163  	scanner.Split(bufio.ScanLines)
   164  	for scanner.Scan() {
   165  		line := scanner.Text()
   166  		lines = append(lines, line)
   167  	}
   168  	l := int64(len(lines))
   169  	if l < n {
   170  		return lines, nil
   171  	}
   172  	return lines[l-n:], nil
   173  }