github.com/google/cadvisor@v0.49.1/utils/oomparser/oomparser.go (about)

     1  // Copyright 2014 Google Inc. All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package oomparser
    16  
    17  import (
    18  	"path"
    19  	"regexp"
    20  	"strconv"
    21  	"time"
    22  
    23  	"github.com/euank/go-kmsg-parser/kmsgparser"
    24  
    25  	"k8s.io/klog/v2"
    26  )
    27  
    28  var (
    29  	legacyContainerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`)
    30  	// Starting in 5.0 linux kernels, the OOM message changed
    31  	containerRegexp = regexp.MustCompile(`oom-kill:constraint=(.*),nodemask=(.*),cpuset=(.*),mems_allowed=(.*),oom_memcg=(.*),task_memcg=(.*),task=(.*),pid=(.*),uid=(.*)`)
    32  	lastLineRegexp  = regexp.MustCompile(`Killed process ([0-9]+) \((.+)\)`)
    33  	firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`)
    34  )
    35  
    36  // OomParser wraps a kmsgparser in order to extract OOM events from the
    37  // individual kernel ring buffer messages.
    38  type OomParser struct {
    39  	parser kmsgparser.Parser
    40  }
    41  
    42  // struct that contains information related to an OOM kill instance
    43  type OomInstance struct {
    44  	// process id of the killed process
    45  	Pid int
    46  	// the name of the killed process
    47  	ProcessName string
    48  	// the time that the process was reported to be killed,
    49  	// accurate to the minute
    50  	TimeOfDeath time.Time
    51  	// the absolute name of the container that OOMed
    52  	ContainerName string
    53  	// the absolute name of the container that was killed
    54  	// due to the OOM.
    55  	VictimContainerName string
    56  	// the constraint that triggered the OOM.  One of CONSTRAINT_NONE,
    57  	// CONSTRAINT_CPUSET, CONSTRAINT_MEMORY_POLICY, CONSTRAINT_MEMCG
    58  	Constraint string
    59  }
    60  
    61  // gets the container name from a line and adds it to the oomInstance.
    62  func getLegacyContainerName(line string, currentOomInstance *OomInstance) error {
    63  	parsedLine := legacyContainerRegexp.FindStringSubmatch(line)
    64  	if parsedLine == nil {
    65  		return nil
    66  	}
    67  	currentOomInstance.ContainerName = path.Join("/", parsedLine[1])
    68  	currentOomInstance.VictimContainerName = path.Join("/", parsedLine[2])
    69  	return nil
    70  }
    71  
    72  // gets the container name from a line and adds it to the oomInstance.
    73  func getContainerName(line string, currentOomInstance *OomInstance) (bool, error) {
    74  	parsedLine := containerRegexp.FindStringSubmatch(line)
    75  	if parsedLine == nil {
    76  		// Fall back to the legacy format if it isn't found here.
    77  		return false, getLegacyContainerName(line, currentOomInstance)
    78  	}
    79  	currentOomInstance.ContainerName = parsedLine[6]
    80  	currentOomInstance.VictimContainerName = parsedLine[5]
    81  	currentOomInstance.Constraint = parsedLine[1]
    82  	pid, err := strconv.Atoi(parsedLine[8])
    83  	if err != nil {
    84  		return false, err
    85  	}
    86  	currentOomInstance.Pid = pid
    87  	currentOomInstance.ProcessName = parsedLine[7]
    88  	return true, nil
    89  }
    90  
    91  // gets the pid, name, and date from a line and adds it to oomInstance
    92  func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) {
    93  	reList := lastLineRegexp.FindStringSubmatch(line)
    94  
    95  	if reList == nil {
    96  		return false, nil
    97  	}
    98  
    99  	pid, err := strconv.Atoi(reList[1])
   100  	if err != nil {
   101  		return false, err
   102  	}
   103  	currentOomInstance.Pid = pid
   104  	currentOomInstance.ProcessName = reList[2]
   105  	return true, nil
   106  }
   107  
   108  // uses regex to see if line is the start of a kernel oom log
   109  func checkIfStartOfOomMessages(line string) bool {
   110  	potentialOomStart := firstLineRegexp.MatchString(line)
   111  	return potentialOomStart
   112  }
   113  
   114  // StreamOoms writes to a provided a stream of OomInstance objects representing
   115  // OOM events that are found in the logs.
   116  // It will block and should be called from a goroutine.
   117  func (p *OomParser) StreamOoms(outStream chan<- *OomInstance) {
   118  	kmsgEntries := p.parser.Parse()
   119  	defer p.parser.Close()
   120  
   121  	for msg := range kmsgEntries {
   122  		isOomMessage := checkIfStartOfOomMessages(msg.Message)
   123  		if isOomMessage {
   124  			oomCurrentInstance := &OomInstance{
   125  				ContainerName:       "/",
   126  				VictimContainerName: "/",
   127  				TimeOfDeath:         msg.Timestamp,
   128  			}
   129  			for msg := range kmsgEntries {
   130  				finished, err := getContainerName(msg.Message, oomCurrentInstance)
   131  				if err != nil {
   132  					klog.Errorf("%v", err)
   133  				}
   134  				if !finished {
   135  					finished, err = getProcessNamePid(msg.Message, oomCurrentInstance)
   136  					if err != nil {
   137  						klog.Errorf("%v", err)
   138  					}
   139  				}
   140  				if finished {
   141  					oomCurrentInstance.TimeOfDeath = msg.Timestamp
   142  					break
   143  				}
   144  			}
   145  			outStream <- oomCurrentInstance
   146  		}
   147  	}
   148  	// Should not happen
   149  	klog.Errorf("exiting analyzeLines. OOM events will not be reported.")
   150  }
   151  
   152  // initializes an OomParser object. Returns an OomParser object and an error.
   153  func New() (*OomParser, error) {
   154  	parser, err := kmsgparser.NewParser()
   155  	if err != nil {
   156  		return nil, err
   157  	}
   158  	parser.SetLogger(glogAdapter{})
   159  	return &OomParser{parser: parser}, nil
   160  }
   161  
   162  type glogAdapter struct{}
   163  
   164  var _ kmsgparser.Logger = glogAdapter{}
   165  
   166  func (glogAdapter) Infof(format string, args ...interface{}) {
   167  	klog.V(4).Infof(format, args...)
   168  }
   169  func (glogAdapter) Warningf(format string, args ...interface{}) {
   170  	klog.V(2).Infof(format, args...)
   171  }
   172  func (glogAdapter) Errorf(format string, args ...interface{}) {
   173  	klog.Warningf(format, args...)
   174  }