k8s.io/perf-tests/clusterloader2@v0.0.0-20240304094227-64bdb12da87e/pkg/prometheus/gce_windows_util.go (about)

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package prometheus
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os/exec"
    23  	"strings"
    24  	"time"
    25  
    26  	corev1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/client-go/kubernetes"
    29  	"k8s.io/klog/v2"
    30  	"k8s.io/perf-tests/clusterloader2/pkg/config"
    31  	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
    32  )
    33  
    34  const (
    35  	installWmiExporterRetryInterval = 10 * time.Second
    36  	installWmiExporterRetryTimes    = 6
    37  )
    38  
    39  // gceNode is a struct storing gce node info
    40  type gceNode struct {
    41  	projectID string
    42  	zone      string
    43  	nodeName  string
    44  }
    45  
    46  func setUpWindowsNodeAndTemplate(k8sClient kubernetes.Interface, mapping map[string]interface{}) error {
    47  	// Get the gce windows node info
    48  	windowsNode, err := getGceWindowsNodeFromKubernetesService(k8sClient)
    49  	if err != nil {
    50  		return err
    51  	}
    52  	// Install the wmi exporter onto windows node
    53  	if err := installWmiExporter(k8sClient, windowsNode, mapping); err != nil {
    54  		return err
    55  	}
    56  	mapping["WINDOWS_NODE_NAME"] = windowsNode.nodeName
    57  	mapping["PROMETHEUS_SCRAPE_WINDOWS_NODES"] = true
    58  	return nil
    59  }
    60  
    61  func isWindowsNodeScrapingEnabled(mapping map[string]interface{}, clusterLoaderConfig *config.ClusterLoaderConfig) bool {
    62  	if windowsNodeTest, exists := mapping["WINDOWS_NODE_TEST"]; exists && windowsNodeTest.(bool) && clusterLoaderConfig.ClusterConfig.Provider.Features().SupportWindowsNodeScraping {
    63  		return true
    64  	}
    65  	return false
    66  }
    67  
    68  func installWmiExporter(k8sClient kubernetes.Interface, windowsNode *gceNode, mapping map[string]interface{}) error {
    69  	wmiExporterURL, ok := mapping["CL2_WMI_EXPORTER_URL"]
    70  	if !ok {
    71  		return fmt.Errorf("missing setting up wmi exporter download url")
    72  	}
    73  	wmiExporterCollectors, ok := mapping["CL2_WMI_EXPORTER_ENABLED_COLLECTORS"]
    74  	if !ok {
    75  		return fmt.Errorf("missing setting up wmi exporter enabled collectors")
    76  	}
    77  	// Tried executing the invoke-webrequest cmd directly in gcloud ssh bash, but get this error:
    78  	// 		invoke-webrequest : Win32 internal error "Access is denied" 0x5 occurred while reading the console output buffer.
    79  	// 		Contact Microsoft Customer Support Services.
    80  	// After wrap it up in script block, it works well, also support timeout setting.
    81  	installWmiCmdTemplate := `Remove-Job -Name InstallWmiExporter 2>&1 | Out-Null; Start-Job -Name InstallWmiExporter -ScriptBlock {invoke-webrequest -uri %s -outfile C:\wmi_exporter.exe; New-Service -Name wmi_exporter -BinaryPathName 'C:\wmi_exporter.exe --collectors.enabled="%s" --telemetry.addr=":5000"'; Start-Service wmi_exporter}; Wait-Job -Timeout 300 -Name InstallWmiExporter`
    82  	installWmiExporterCmd := fmt.Sprintf(installWmiCmdTemplate, wmiExporterURL, wmiExporterCollectors)
    83  	powershellCmd := fmt.Sprintf(`powershell.exe -Command "%s"`, installWmiExporterCmd)
    84  	if windowsNode == nil {
    85  		return fmt.Errorf("no windows nodes available to install wmi exporter")
    86  	}
    87  	klog.V(2).Infof("Installing wmi exporter onto projectId: %s, zone: %s, nodeName: %s with cmd: %s", windowsNode.projectID, windowsNode.zone, windowsNode.nodeName, installWmiExporterCmd)
    88  	var err error
    89  	for i := 0; i < installWmiExporterRetryTimes; i++ {
    90  		cmd := exec.Command("gcloud", "compute", "ssh", "--project", windowsNode.projectID, "--zone", windowsNode.zone, windowsNode.nodeName, "--command", powershellCmd)
    91  		err = cmd.Run()
    92  		if err == nil {
    93  			break
    94  		} else {
    95  			klog.V(2).Infof("Retried %d times to install wmi exporter with error: %+v", i, err)
    96  		}
    97  		time.Sleep(installWmiExporterRetryInterval)
    98  	}
    99  	return err
   100  }
   101  
   102  func getGceWindowsNodeFromKubernetesService(k8sClient kubernetes.Interface) (*gceNode, error) {
   103  	var nodeList *corev1.NodeList
   104  	f := func() error {
   105  		var err error
   106  		nodeList, err = k8sClient.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{LabelSelector: "kubernetes.io/os=windows"})
   107  		return err
   108  	}
   109  
   110  	if err := client.RetryWithExponentialBackOff(client.RetryFunction(f)); err != nil {
   111  		return nil, err
   112  	}
   113  
   114  	if len(nodeList.Items) == 0 {
   115  		return nil, fmt.Errorf("no windows nodes available in kubernetes service")
   116  	}
   117  	// TODO: Modify to support multiple windows nodes soon.
   118  	providerStrs := strings.Split(nodeList.Items[0].Spec.ProviderID, "/")
   119  	if len(providerStrs) < 5 {
   120  		return nil, fmt.Errorf("no valid gce provider ID available")
   121  	}
   122  
   123  	// Gce providerID format: gce://{projectID}/{zone}/{nodeName}
   124  	return &gceNode{
   125  		projectID: providerStrs[2],
   126  		zone:      providerStrs[3],
   127  		nodeName:  providerStrs[4],
   128  	}, nil
   129  }