k8s.io/kubernetes@v1.29.3/test/integration/logs/benchmark/load.go (about) 1 /* 2 Copyright 2021 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package benchmark 18 19 import ( 20 "bufio" 21 "bytes" 22 "encoding/json" 23 "errors" 24 "fmt" 25 "os" 26 "reflect" 27 "regexp" 28 "sort" 29 "strings" 30 "text/template" 31 32 v1 "k8s.io/api/core/v1" 33 runtimev1 "k8s.io/cri-api/pkg/apis/runtime/v1" 34 "k8s.io/klog/v2" 35 ) 36 37 type logMessage struct { 38 msg string 39 verbosity int 40 err error 41 isError bool 42 kvs []interface{} 43 } 44 45 const ( 46 stringArg = "string" 47 multiLineStringArg = "multiLineString" 48 objectStringArg = "objectString" 49 numberArg = "number" 50 krefArg = "kref" 51 otherArg = "other" 52 totalArg = "total" 53 ) 54 55 type logStats struct { 56 TotalLines, JsonLines, SplitLines, ErrorMessages int 57 58 ArgCounts map[string]int 59 OtherLines []string 60 OtherArgs []interface{} 61 MultiLineArgs [][]string 62 ObjectTypes map[string]int 63 } 64 65 var ( 66 logStatsTemplate = template.Must(template.New("format").Funcs(template.FuncMap{ 67 "percent": func(x, y int) string { 68 if y == 0 { 69 return "NA" 70 } 71 return fmt.Sprintf("%d%%", x*100/y) 72 }, 73 "sub": func(x, y int) int { 74 return x - y 75 }, 76 }).Parse(`Total number of lines: {{.TotalLines}} 77 JSON line continuation: {{.SplitLines}} 78 Valid JSON messages: {{.JsonLines}} ({{percent .JsonLines .TotalLines}} of total lines) 79 Error messages: {{.ErrorMessages}} ({{percent .ErrorMessages .JsonLines}} of valid JSON messages) 80 Unrecognized lines: {{sub (sub .TotalLines .JsonLines) .SplitLines}} 81 {{range .OtherLines}} {{if gt (len .) 80}}{{slice . 0 80}}{{else}}{{.}}{{end}} 82 {{end}} 83 Args: 84 total: {{if .ArgCounts.total}}{{.ArgCounts.total}}{{else}}0{{end}}{{if .ArgCounts.string}} 85 strings: {{.ArgCounts.string}} ({{percent .ArgCounts.string .ArgCounts.total}}){{end}} {{if .ArgCounts.multiLineString}} 86 with line breaks: {{.ArgCounts.multiLineString}} ({{percent .ArgCounts.multiLineString .ArgCounts.total}} of all arguments) 87 {{range .MultiLineArgs}} ===== {{index . 0}} ===== 88 {{index . 1}} 89 90 {{end}}{{end}}{{if .ArgCounts.objectString}} 91 with API objects: {{.ArgCounts.objectString}} ({{percent .ArgCounts.objectString .ArgCounts.total}} of all arguments) 92 types and their number of usage:{{range $key, $value := .ObjectTypes}} {{ $key }}:{{ $value }}{{end}}{{end}}{{if .ArgCounts.number}} 93 numbers: {{.ArgCounts.number}} ({{percent .ArgCounts.number .ArgCounts.total}}){{end}}{{if .ArgCounts.kref}} 94 ObjectRef: {{.ArgCounts.kref}} ({{percent .ArgCounts.kref .ArgCounts.total}}){{end}}{{if .ArgCounts.other}} 95 others: {{.ArgCounts.other}} ({{percent .ArgCounts.other .ArgCounts.total}}){{end}} 96 `)) 97 ) 98 99 // This produces too much output: 100 // {{range .OtherArgs}} {{.}} 101 // {{end}} 102 103 // Doesn't work? 104 // Unrecognized lines: {{with $delta := sub .TotalLines .JsonLines}}{{$delta}} ({{percent $delta .TotalLines}} of total lines){{end}} 105 106 func (s logStats) String() string { 107 var buffer bytes.Buffer 108 err := logStatsTemplate.Execute(&buffer, &s) 109 if err != nil { 110 return err.Error() 111 } 112 return buffer.String() 113 } 114 115 func loadLog(path string) (messages []logMessage, stats logStats, err error) { 116 file, err := os.Open(path) 117 if err != nil { 118 return nil, logStats{}, err 119 } 120 defer file.Close() 121 122 stats.ArgCounts = map[string]int{} 123 scanner := bufio.NewScanner(file) 124 var buffer bytes.Buffer 125 for lineNo := 0; scanner.Scan(); lineNo++ { 126 stats.TotalLines++ 127 line := scanner.Bytes() 128 buffer.Write(line) 129 msg, err := parseLine(buffer.Bytes(), &stats) 130 if err != nil { 131 // JSON might have been split across multiple lines. 132 var jsonErr *json.SyntaxError 133 if errors.As(err, &jsonErr) && jsonErr.Offset > 1 { 134 // The start of the buffer was okay. Keep the 135 // data and add the next line to it. 136 stats.SplitLines++ 137 continue 138 } 139 stats.OtherLines = append(stats.OtherLines, fmt.Sprintf("%d: %s", lineNo, string(line))) 140 buffer.Reset() 141 continue 142 } 143 stats.JsonLines++ 144 messages = append(messages, msg) 145 buffer.Reset() 146 } 147 148 if err := scanner.Err(); err != nil { 149 return nil, logStats{}, fmt.Errorf("reading %s failed: %v", path, err) 150 } 151 152 return 153 } 154 155 // String format for API structs from generated.pb.go. 156 // &Container{...} 157 var objectRE = regexp.MustCompile(`^&([a-zA-Z]*)\{`) 158 159 func parseLine(line []byte, stats *logStats) (item logMessage, err error) { 160 161 content := map[string]interface{}{} 162 if err := json.Unmarshal(line, &content); err != nil { 163 return logMessage{}, fmt.Errorf("JSON parsing failed: %w", err) 164 } 165 166 kvs := map[string]interface{}{} 167 item.isError = true 168 for key, value := range content { 169 switch key { 170 case "v": 171 verbosity, ok := value.(float64) 172 if !ok { 173 return logMessage{}, fmt.Errorf("expected number for v, got: %T %v", value, value) 174 } 175 item.verbosity = int(verbosity) 176 item.isError = false 177 case "msg": 178 msg, ok := value.(string) 179 if !ok { 180 return logMessage{}, fmt.Errorf("expected string for msg, got: %T %v", value, value) 181 } 182 item.msg = msg 183 case "ts", "caller": 184 // ignore 185 case "err": 186 errStr, ok := value.(string) 187 if !ok { 188 return logMessage{}, fmt.Errorf("expected string for err, got: %T %v", value, value) 189 } 190 item.err = errors.New(errStr) 191 stats.ArgCounts[stringArg]++ 192 stats.ArgCounts[totalArg]++ 193 default: 194 if obj := toObject(value); obj != nil { 195 value = obj 196 } 197 switch value := value.(type) { 198 case string: 199 stats.ArgCounts[stringArg]++ 200 if strings.Contains(value, "\n") { 201 stats.ArgCounts[multiLineStringArg]++ 202 stats.MultiLineArgs = append(stats.MultiLineArgs, []string{key, value}) 203 } 204 match := objectRE.FindStringSubmatch(value) 205 if match != nil { 206 if stats.ObjectTypes == nil { 207 stats.ObjectTypes = map[string]int{} 208 } 209 stats.ArgCounts[objectStringArg]++ 210 stats.ObjectTypes[match[1]]++ 211 } 212 case float64: 213 stats.ArgCounts[numberArg]++ 214 case klog.ObjectRef: 215 stats.ArgCounts[krefArg]++ 216 default: 217 stats.ArgCounts[otherArg]++ 218 stats.OtherArgs = append(stats.OtherArgs, value) 219 } 220 stats.ArgCounts[totalArg]++ 221 kvs[key] = value 222 } 223 } 224 225 // Sort by key. 226 var keys []string 227 for key := range kvs { 228 keys = append(keys, key) 229 } 230 sort.Strings(keys) 231 for _, key := range keys { 232 item.kvs = append(item.kvs, key, kvs[key]) 233 } 234 235 if !item.isError && item.err != nil { 236 // Error is a normal key/value. 237 item.kvs = append(item.kvs, "err", item.err) 238 item.err = nil 239 } 240 if item.isError { 241 stats.ErrorMessages++ 242 } 243 return 244 } 245 246 // This is a list of objects that might have been dumped. The simple ones must 247 // come first because unmarshaling will try one after the after and an 248 // ObjectRef would unmarshal fine into any of the others whereas any of the 249 // other types hopefully have enough extra fields that they won't fit (unknown 250 // fields are an error). 251 var objectTypes = []reflect.Type{ 252 reflect.TypeOf(klog.ObjectRef{}), 253 reflect.TypeOf(&runtimev1.VersionResponse{}), 254 reflect.TypeOf(&v1.Pod{}), 255 reflect.TypeOf(&v1.Container{}), 256 } 257 258 func toObject(value interface{}) interface{} { 259 data, ok := value.(map[string]interface{}) 260 if !ok { 261 return nil 262 } 263 jsonData, err := json.Marshal(data) 264 if err != nil { 265 return nil 266 } 267 for _, t := range objectTypes { 268 obj := reflect.New(t) 269 decoder := json.NewDecoder(bytes.NewBuffer(jsonData)) 270 decoder.DisallowUnknownFields() 271 if err := decoder.Decode(obj.Interface()); err == nil { 272 return reflect.Indirect(obj).Interface() 273 } 274 } 275 return nil 276 }