github.com/munnerz/test-infra@v0.0.0-20190108210205-ce3d181dc989/prow/spyglass/lenses/buildlog/lens.go (about)

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package buildlog provides a build log viewer for Spyglass
    18  package buildlog
    19  
    20  import (
    21  	"bytes"
    22  	"encoding/json"
    23  	"fmt"
    24  	"html/template"
    25  	"io"
    26  	"path/filepath"
    27  	"regexp"
    28  	"strings"
    29  
    30  	"github.com/sirupsen/logrus"
    31  	"k8s.io/test-infra/prow/spyglass/lenses"
    32  )
    33  
    34  const (
    35  	name            = "buildlog"
    36  	title           = "Build Log"
    37  	priority        = 10
    38  	neighborLines   = 5 // number of "important" lines to be displayed in either direction
    39  	minLinesSkipped = 5
    40  )
    41  
    42  // Lens implements the build lens.
    43  type Lens struct{}
    44  
    45  // Name returns the name.
    46  func (lens Lens) Name() string {
    47  	return name
    48  }
    49  
    50  // Title returns the title.
    51  func (lens Lens) Title() string {
    52  	return title
    53  }
    54  
    55  // Priority returns the priority.
    56  func (lens Lens) Priority() int {
    57  	return priority
    58  }
    59  
    60  // Header executes the "header" section of the template.
    61  func (lens Lens) Header(artifacts []lenses.Artifact, resourceDir string) string {
    62  	return executeTemplate(resourceDir, "header", BuildLogsView{})
    63  }
    64  
    65  // errRE matches keywords and glog error messages
    66  var errRE = regexp.MustCompile(`(?i)(\s|^)timed out\b|(\s|^)error(s)?\b|(\s|^)fail(ure|ed)?\b|(\s|^)fatal\b|(\s|^)panic\b|^E\d{4} \d\d:\d\d:\d\d\.\d\d\d]`)
    67  
    68  func init() {
    69  	lenses.RegisterLens(Lens{})
    70  }
    71  
    72  // SubLine represents an substring within a LogLine. It it used so error terms can be highlighted.
    73  type SubLine struct {
    74  	Highlighted bool
    75  	Text        string
    76  }
    77  
    78  // LogLine represents a line displayed in the LogArtifactView.
    79  type LogLine struct {
    80  	Number      int
    81  	Length      int
    82  	Highlighted bool
    83  	Skip        bool
    84  	SubLines    []SubLine
    85  }
    86  
    87  // LineGroup holds multiple lines that can be collapsed/expanded as a block
    88  type LineGroup struct {
    89  	Skip                   bool
    90  	Start, End             int // closed, open
    91  	ByteOffset, ByteLength int
    92  	LogLines               []LogLine
    93  }
    94  
    95  // LineRequest represents a request for output lines from an artifact. If Offset is 0 and Length
    96  // is -1, all lines will be fetched.
    97  type LineRequest struct {
    98  	Artifact  string `json:"artifact"`
    99  	Offset    int64  `json:"offset"`
   100  	Length    int64  `json:"length"`
   101  	StartLine int    `json:"startLine"`
   102  }
   103  
   104  // LinesSkipped returns the number of lines skipped in a line group.
   105  func (g LineGroup) LinesSkipped() int {
   106  	return g.End - g.Start
   107  }
   108  
   109  // LogArtifactView holds a single log file's view
   110  type LogArtifactView struct {
   111  	ArtifactName string
   112  	ArtifactLink string
   113  	LineGroups   []LineGroup
   114  	ViewAll      bool
   115  }
   116  
   117  // BuildLogsView holds each log file view
   118  type BuildLogsView struct {
   119  	LogViews           []LogArtifactView
   120  	RawGetAllRequests  map[string]string
   121  	RawGetMoreRequests map[string]string
   122  }
   123  
   124  // Body returns the <body> content for a build log (or multiple build logs)
   125  func (lens Lens) Body(artifacts []lenses.Artifact, resourceDir string, data string) string {
   126  	buildLogsView := BuildLogsView{
   127  		LogViews:           []LogArtifactView{},
   128  		RawGetAllRequests:  make(map[string]string),
   129  		RawGetMoreRequests: make(map[string]string),
   130  	}
   131  
   132  	// Read log artifacts and construct template structs
   133  	for _, a := range artifacts {
   134  		av := LogArtifactView{
   135  			ArtifactName: a.JobPath(),
   136  			ArtifactLink: a.CanonicalLink(),
   137  		}
   138  		lines, err := logLinesAll(a)
   139  		if err != nil {
   140  			logrus.WithError(err).Error("Error reading log.")
   141  			continue
   142  		}
   143  		av.LineGroups = groupLines(highlightLines(lines, 0))
   144  		av.ViewAll = true
   145  		buildLogsView.LogViews = append(buildLogsView.LogViews, av)
   146  	}
   147  
   148  	return executeTemplate(resourceDir, "body", buildLogsView)
   149  }
   150  
   151  // Callback is used to retrieve new log segments
   152  func (lens Lens) Callback(artifacts []lenses.Artifact, resourceDir string, data string) string {
   153  	var request LineRequest
   154  	err := json.Unmarshal([]byte(data), &request)
   155  	if err != nil {
   156  		return "failed to unmarshal request"
   157  	}
   158  	artifact, ok := artifactByName(artifacts, request.Artifact)
   159  	if !ok {
   160  		return "no artifact named " + request.Artifact
   161  	}
   162  
   163  	var lines []string
   164  	if request.Offset == 0 && request.Length == -1 {
   165  		lines, err = logLinesAll(artifact)
   166  	} else {
   167  		lines, err = logLines(artifact, request.Offset, request.Length)
   168  	}
   169  	if err != nil {
   170  		return fmt.Sprintf("failed to retrieve log lines: %v", err)
   171  	}
   172  
   173  	logLines := highlightLines(lines, request.StartLine)
   174  	return executeTemplate(resourceDir, "line group", logLines)
   175  }
   176  
   177  func artifactByName(artifacts []lenses.Artifact, name string) (lenses.Artifact, bool) {
   178  	for _, a := range artifacts {
   179  		if a.JobPath() == name {
   180  			return a, true
   181  		}
   182  	}
   183  	return nil, false
   184  }
   185  
   186  // logLinesAll reads all of an artifact and splits it into lines.
   187  func logLinesAll(artifact lenses.Artifact) ([]string, error) {
   188  	read, err := artifact.ReadAll()
   189  	if err != nil {
   190  		return nil, fmt.Errorf("failed to read log %q: %v", artifact.JobPath(), err)
   191  	}
   192  	logLines := strings.Split(string(read), "\n")
   193  
   194  	return logLines, nil
   195  }
   196  
   197  func logLines(artifact lenses.Artifact, offset, length int64) ([]string, error) {
   198  	b := make([]byte, length)
   199  	_, err := artifact.ReadAt(b, offset)
   200  	if err != nil && err != io.EOF {
   201  		if err != lenses.ErrGzipOffsetRead {
   202  			return nil, fmt.Errorf("couldn't read requested bytes: %v", err)
   203  		}
   204  		moreBytes, err := artifact.ReadAtMost(offset + length)
   205  		if err != nil && err != io.EOF {
   206  			return nil, fmt.Errorf("couldn't handle reading gzipped file: %v", err)
   207  		}
   208  		b = moreBytes[offset:]
   209  	}
   210  	return strings.Split(string(b), "\n"), nil
   211  }
   212  
   213  func highlightLines(lines []string, startLine int) []LogLine {
   214  	// mark highlighted lines
   215  	logLines := make([]LogLine, 0, len(lines))
   216  	for i, text := range lines {
   217  		length := len(text)
   218  		subLines := []SubLine{}
   219  		loc := errRE.FindStringIndex(text)
   220  		for loc != nil {
   221  			subLines = append(subLines, SubLine{false, text[:loc[0]]})
   222  			subLines = append(subLines, SubLine{true, text[loc[0]:loc[1]]})
   223  			text = text[loc[1]:]
   224  			loc = errRE.FindStringIndex(text)
   225  		}
   226  		subLines = append(subLines, SubLine{false, text})
   227  		logLines = append(logLines, LogLine{
   228  			Length:      length + 1, // counting the "\n"
   229  			SubLines:    subLines,
   230  			Number:      startLine + i + 1,
   231  			Highlighted: len(subLines) > 1,
   232  			Skip:        true,
   233  		})
   234  	}
   235  	return logLines
   236  }
   237  
   238  // breaks lines into important/unimportant groups
   239  func groupLines(logLines []LogLine) []LineGroup {
   240  	// show highlighted lines and their neighboring lines
   241  	for i, line := range logLines {
   242  		if line.Highlighted {
   243  			for d := -neighborLines; d <= neighborLines; d++ {
   244  				if i+d < 0 {
   245  					continue
   246  				}
   247  				if i+d >= len(logLines) {
   248  					break
   249  				}
   250  				logLines[i+d].Skip = false
   251  			}
   252  		}
   253  	}
   254  	// break into groups
   255  	currentOffset := 0
   256  	previousOffset := 0
   257  	var lineGroups []LineGroup
   258  	curGroup := LineGroup{}
   259  	for i, line := range logLines {
   260  		if line.Skip == curGroup.Skip {
   261  			curGroup.LogLines = append(curGroup.LogLines, line)
   262  			currentOffset += line.Length
   263  		} else {
   264  			curGroup.End = i
   265  			curGroup.ByteLength = currentOffset - previousOffset - 1 // -1 for trailing newline
   266  			previousOffset = currentOffset
   267  			if curGroup.Skip {
   268  				if curGroup.LinesSkipped() < minLinesSkipped {
   269  					curGroup.Skip = false
   270  				}
   271  			}
   272  			if len(curGroup.LogLines) > 0 {
   273  				lineGroups = append(lineGroups, curGroup)
   274  			}
   275  			curGroup = LineGroup{
   276  				Skip:       line.Skip,
   277  				Start:      i,
   278  				LogLines:   []LogLine{line},
   279  				ByteOffset: currentOffset,
   280  			}
   281  			currentOffset += line.Length
   282  		}
   283  	}
   284  	curGroup.End = len(logLines)
   285  	curGroup.ByteLength = currentOffset - previousOffset - 1
   286  	if curGroup.Skip {
   287  		if curGroup.LinesSkipped() < minLinesSkipped {
   288  			curGroup.Skip = false
   289  		}
   290  	}
   291  	if len(curGroup.LogLines) > 0 {
   292  		lineGroups = append(lineGroups, curGroup)
   293  	}
   294  	return lineGroups
   295  }
   296  
   297  // LogViewTemplate executes the log viewer template ready for rendering
   298  func executeTemplate(resourceDir, templateName string, data interface{}) string {
   299  	t := template.New("template.html")
   300  	_, err := t.ParseFiles(filepath.Join(resourceDir, "template.html"))
   301  	if err != nil {
   302  		return fmt.Sprintf("Failed to load template: %v", err)
   303  	}
   304  	var buf bytes.Buffer
   305  	if err := t.ExecuteTemplate(&buf, templateName, data); err != nil {
   306  		logrus.WithError(err).Error("Error executing template.")
   307  	}
   308  	return buf.String()
   309  }