k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/experiment/logviewer/gcsreader/main.go (about)

     1  /*
     2  Copyright 2019 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package main
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  	"os"
    23  	"regexp"
    24  
    25  	"cloud.google.com/go/storage"
    26  	gzip "github.com/klauspost/pgzip"
    27  	"google.golang.org/api/option"
    28  	"k8s.io/klog/v2"
    29  )
    30  
    31  func main() {
    32  	logPath, targetSubstring := setupFromConsole()
    33  	regex := regexp.MustCompile(targetSubstring)
    34  
    35  	r, err := downloadAndDecompress(logPath)
    36  	if err != nil {
    37  		klog.Fatal(err)
    38  	}
    39  	parsed, err := processLines(r, regex)
    40  	if err != nil {
    41  		klog.Fatal(err)
    42  	}
    43  
    44  	if len(parsed) == 0 {
    45  		klog.Info("No lines found")
    46  	} else {
    47  		for _, line := range parsed {
    48  			klog.Infoln(*line)
    49  		}
    50  	}
    51  }
    52  
    53  const testLogPath = "logs/ci-kubernetes-e2e-gce-scale-performance/290/artifacts/gce-scale-cluster-master/kube-apiserver-audit.log-20190102-1546444805.gz"
    54  const testTargetSubstring = "\"auditID\":\"07ff64df-fcfe-4cdc-83a5-0c6a09237698\""
    55  
    56  func setupFromConsole() (string, string) {
    57  	var logPath, targetSubstring string
    58  	if len(os.Args) < 2 || os.Args[1] == "" {
    59  		logPath = testLogPath
    60  	} else {
    61  		logPath = os.Args[1]
    62  	}
    63  
    64  	if len(os.Args) < 3 || os.Args[2] == "" {
    65  		targetSubstring = testTargetSubstring
    66  	} else {
    67  		targetSubstring = os.Args[2]
    68  	}
    69  
    70  	return logPath, targetSubstring
    71  }
    72  
    73  func downloadAndDecompress(objectPath string) (io.Reader, error) {
    74  	reader, err := download(objectPath)
    75  	if err != nil {
    76  		return nil, err
    77  	}
    78  
    79  	decompressed, err := decompress(reader)
    80  	if err != nil {
    81  		return nil, err
    82  	}
    83  	return decompressed, nil
    84  }
    85  
    86  const bucketName = "kubernetes-jenkins"
    87  
    88  func download(objectPath string) (io.Reader, error) {
    89  	context := context.Background()
    90  	client, err := storage.NewClient(context, option.WithoutAuthentication())
    91  	if err != nil {
    92  		return nil, err
    93  	}
    94  
    95  	bucket := client.Bucket(bucketName)
    96  
    97  	remoteFile := bucket.Object(objectPath).ReadCompressed(true)
    98  	reader, err := remoteFile.NewReader(context)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  
   103  	return reader, err
   104  }
   105  
   106  func decompress(reader io.Reader) (io.Reader, error) {
   107  	newReader, err := gzip.NewReader(reader)
   108  	if err != nil {
   109  		return nil, err
   110  	}
   111  	return newReader, nil
   112  }