github.com/kubewharf/katalyst-core@v0.5.3/pkg/agent/sysadvisor/plugin/inference/inference.go (about)

     1  /*
     2  Copyright 2022 The Katalyst Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package inference
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"sync"
    23  	"time"
    24  
    25  	"k8s.io/apimachinery/pkg/util/wait"
    26  
    27  	"github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/metacache"
    28  	"github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/plugin"
    29  	"github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/plugin/inference/modelresultfetcher"
    30  	borweinfetcher "github.com/kubewharf/katalyst-core/pkg/agent/sysadvisor/plugin/inference/modelresultfetcher/borwein"
    31  	"github.com/kubewharf/katalyst-core/pkg/config"
    32  	metricemitter "github.com/kubewharf/katalyst-core/pkg/config/agent/sysadvisor/metric-emitter"
    33  	"github.com/kubewharf/katalyst-core/pkg/config/generic"
    34  	"github.com/kubewharf/katalyst-core/pkg/metaserver"
    35  	"github.com/kubewharf/katalyst-core/pkg/metrics"
    36  	metricspool "github.com/kubewharf/katalyst-core/pkg/metrics/metrics-pool"
    37  	"github.com/kubewharf/katalyst-core/pkg/util/general"
    38  )
    39  
    40  func init() {
    41  	modelresultfetcher.RegisterModelResultFetcherInitFunc(borweinfetcher.BorweinModelResultFetcherName,
    42  		borweinfetcher.NewBorweinModelResultFetcher)
    43  }
    44  
    45  type InferencePlugin struct {
    46  	name string
    47  	// conf config.Configuration
    48  
    49  	period               time.Duration
    50  	modelsResultFetchers map[string]modelresultfetcher.ModelResultFetcher
    51  
    52  	metaServer    *metaserver.MetaServer
    53  	metricEmitter metrics.MetricEmitter
    54  
    55  	emitterConf *metricemitter.MetricEmitterPluginConfiguration
    56  	qosConf     *generic.QoSConfiguration
    57  
    58  	metaReader metacache.MetaReader
    59  	metaWriter metacache.MetaWriter
    60  }
    61  
    62  func NewInferencePlugin(pluginName string, conf *config.Configuration, extraConf interface{},
    63  	emitterPool metricspool.MetricsEmitterPool, metaServer *metaserver.MetaServer,
    64  	metaCache metacache.MetaCache,
    65  ) (plugin.SysAdvisorPlugin, error) {
    66  	if conf == nil || conf.InferencePluginConfiguration == nil {
    67  		return nil, fmt.Errorf("nil conf")
    68  	} else if metaServer == nil {
    69  		return nil, fmt.Errorf("nil metaServer")
    70  	} else if metaCache == nil {
    71  		return nil, fmt.Errorf("nil metaCache")
    72  	} else if emitterPool == nil {
    73  		return nil, fmt.Errorf("nil emitterPool")
    74  	}
    75  
    76  	metricEmitter := emitterPool.GetDefaultMetricsEmitter().WithTags("advisor-inference")
    77  
    78  	inferencePlugin := InferencePlugin{
    79  		name:                 pluginName,
    80  		period:               conf.InferencePluginConfiguration.SyncPeriod,
    81  		modelsResultFetchers: make(map[string]modelresultfetcher.ModelResultFetcher),
    82  		metaServer:           metaServer,
    83  		metricEmitter:        metricEmitter,
    84  		emitterConf:          conf.AgentConfiguration.MetricEmitterPluginConfiguration,
    85  		qosConf:              conf.GenericConfiguration.QoSConfiguration,
    86  		metaReader:           metaCache,
    87  		metaWriter:           metaCache,
    88  	}
    89  
    90  	for fetcherName, initFn := range modelresultfetcher.GetRegisteredModelResultFetcherInitFuncs() {
    91  		// todo: support only enabling part of fetchers
    92  		general.Infof("try init fetcher: %s", fetcherName)
    93  		fetcher, err := initFn(fetcherName, conf, extraConf, emitterPool, metaServer, metaCache)
    94  		if err != nil {
    95  			return nil, fmt.Errorf("failed to start sysadvisor plugin %v: %v", pluginName, err)
    96  		} else if fetcher == nil {
    97  			general.Infof("fetcher: %s isn't enabled", fetcherName)
    98  			continue
    99  		}
   100  
   101  		inferencePlugin.modelsResultFetchers[fetcherName] = fetcher
   102  	}
   103  
   104  	return &inferencePlugin, nil
   105  }
   106  
   107  func (infp *InferencePlugin) Run(ctx context.Context) {
   108  	wait.UntilWithContext(ctx, infp.fetchModelResult, infp.period)
   109  }
   110  
   111  // Name returns the name of inference plugin
   112  func (infp *InferencePlugin) Name() string {
   113  	return infp.name
   114  }
   115  
   116  // Init initializes the inference plugin
   117  func (infp *InferencePlugin) Init() error {
   118  	return nil
   119  }
   120  
   121  func (infp *InferencePlugin) fetchModelResult(ctx context.Context) {
   122  	var wg sync.WaitGroup
   123  
   124  	for modelName, fetcher := range infp.modelsResultFetchers {
   125  		wg.Add(1)
   126  		go func(modelName string, fetcher modelresultfetcher.ModelResultFetcher) {
   127  			defer wg.Done()
   128  			err := fetcher.FetchModelResult(ctx, infp.metaReader, infp.metaWriter, infp.metaServer)
   129  			if err != nil {
   130  				general.Errorf("FetchModelResult for model: %s failed with error: %v", modelName, err)
   131  			}
   132  		}(modelName, fetcher)
   133  	}
   134  
   135  	wg.Wait()
   136  }