github.com/pinpoint-apm/pinpoint-go-agent@v1.4.1-0.20240110120318-a50c2eb18c8c/grpc.go (about)

     1  package pinpoint
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"math"
     7  	"math/rand"
     8  	"net"
     9  	"os"
    10  	"runtime"
    11  	"runtime/debug"
    12  	"strconv"
    13  	"time"
    14  
    15  	"github.com/golang/protobuf/ptypes/wrappers"
    16  	pb "github.com/pinpoint-apm/pinpoint-go-agent/protobuf"
    17  	"github.com/sirupsen/logrus"
    18  	"google.golang.org/grpc"
    19  	"google.golang.org/grpc/codes"
    20  	"google.golang.org/grpc/connectivity"
    21  	"google.golang.org/grpc/keepalive"
    22  	"google.golang.org/grpc/metadata"
    23  	"google.golang.org/grpc/status"
    24  )
    25  
    26  const (
    27  	headerAppName   = "applicationname"
    28  	headerAgentID   = "agentid"
    29  	headerAgentName = "agentname"
    30  	headerStartTime = "starttime"
    31  	headerSocketID  = "socketid"
    32  )
    33  
    34  func grpcMetadataContext(agent *agent, socketId int64) context.Context {
    35  	m := map[string]string{}
    36  
    37  	m[headerAppName] = agent.appName
    38  	m[headerAgentID] = agent.agentID
    39  	m[headerStartTime] = strconv.FormatInt(agent.startTime, 10)
    40  	if agent.agentName != "" {
    41  		m[headerAgentName] = agent.agentName
    42  	}
    43  	if socketId > 0 {
    44  		m[headerSocketID] = strconv.FormatInt(socketId, 10)
    45  	}
    46  
    47  	md := metadata.New(m)
    48  	return metadata.NewOutgoingContext(context.Background(), md)
    49  }
    50  
    51  func backOffSleep(attempt int) time.Duration {
    52  	base := float64(1 * time.Second)
    53  	max := float64(60 * time.Second)
    54  
    55  	dur := base * math.Pow(2, float64(attempt))
    56  	if dur > max {
    57  		dur = max
    58  	}
    59  
    60  	return time.Duration(rand.Float64()*(dur-base) + base)
    61  }
    62  
    63  type agentClient interface {
    64  	RequestAgentInfo(ctx context.Context, agentInfo *pb.PAgentInfo) (*pb.PResult, error)
    65  	PingSession(ctx context.Context) (pb.Agent_PingSessionClient, error)
    66  }
    67  
    68  type agentGrpcClient struct {
    69  	client pb.AgentClient
    70  }
    71  
    72  func (agentGrpcClient *agentGrpcClient) RequestAgentInfo(ctx context.Context, agentInfo *pb.PAgentInfo) (*pb.PResult, error) {
    73  	result, err := agentGrpcClient.client.RequestAgentInfo(ctx, agentInfo)
    74  	return result, err
    75  }
    76  
    77  func (agentGrpcClient *agentGrpcClient) PingSession(ctx context.Context) (pb.Agent_PingSessionClient, error) {
    78  	session, err := agentGrpcClient.client.PingSession(ctx)
    79  	return session, err
    80  }
    81  
    82  type metaClient interface {
    83  	RequestApiMetaData(ctx context.Context, in *pb.PApiMetaData) (*pb.PResult, error)
    84  	RequestSqlMetaData(ctx context.Context, in *pb.PSqlMetaData) (*pb.PResult, error)
    85  	RequestSqlUidMetaData(ctx context.Context, in *pb.PSqlUidMetaData) (*pb.PResult, error)
    86  	RequestStringMetaData(ctx context.Context, in *pb.PStringMetaData) (*pb.PResult, error)
    87  	RequestExceptionMetaData(ctx context.Context, in *pb.PExceptionMetaData) (*pb.PResult, error)
    88  }
    89  
    90  type metaGrpcClient struct {
    91  	client pb.MetadataClient
    92  }
    93  
    94  func (metaGrpcClient *metaGrpcClient) RequestApiMetaData(ctx context.Context, in *pb.PApiMetaData) (*pb.PResult, error) {
    95  	result, err := metaGrpcClient.client.RequestApiMetaData(ctx, in)
    96  	return result, err
    97  }
    98  
    99  func (metaGrpcClient *metaGrpcClient) RequestSqlMetaData(ctx context.Context, in *pb.PSqlMetaData) (*pb.PResult, error) {
   100  	result, err := metaGrpcClient.client.RequestSqlMetaData(ctx, in)
   101  	return result, err
   102  }
   103  
   104  func (metaGrpcClient *metaGrpcClient) RequestSqlUidMetaData(ctx context.Context, in *pb.PSqlUidMetaData) (*pb.PResult, error) {
   105  	result, err := metaGrpcClient.client.RequestSqlUidMetaData(ctx, in)
   106  	return result, err
   107  }
   108  
   109  func (metaGrpcClient *metaGrpcClient) RequestStringMetaData(ctx context.Context, in *pb.PStringMetaData) (*pb.PResult, error) {
   110  	result, err := metaGrpcClient.client.RequestStringMetaData(ctx, in)
   111  	return result, err
   112  }
   113  
   114  func (metaGrpcClient *metaGrpcClient) RequestExceptionMetaData(ctx context.Context, in *pb.PExceptionMetaData) (*pb.PResult, error) {
   115  	result, err := metaGrpcClient.client.RequestExceptionMetaData(ctx, in)
   116  	return result, err
   117  }
   118  
   119  const (
   120  	agentGrpcTimeOut  = 60 * time.Second
   121  	sendStreamTimeOut = 5 * time.Second
   122  )
   123  
   124  var kacp = keepalive.ClientParameters{
   125  	Time:                30 * time.Second,
   126  	Timeout:             60 * time.Second,
   127  	PermitWithoutStream: true,
   128  }
   129  
   130  func connectCollector(config *Config, portOption string) (*grpc.ClientConn, error) {
   131  	opts := make([]grpc.DialOption, 0)
   132  	opts = append(opts, grpc.WithKeepaliveParams(kacp))
   133  	opts = append(opts, grpc.WithInsecure())
   134  
   135  	addr := serverAddr(config, portOption)
   136  	Log("grpc").Infof("connect to collector: %s", addr)
   137  	conn, err := grpc.Dial(addr, opts...)
   138  	if err != nil {
   139  		Log("grpc").Errorf("connect to collector - %s, %v", addr, err)
   140  	}
   141  	return conn, err
   142  }
   143  
   144  func serverAddr(config *Config, portOption string) string {
   145  	return fmt.Sprintf("%s:%d", config.String(CfgCollectorHost), config.Int(portOption))
   146  }
   147  
   148  type agentGrpc struct {
   149  	agentConn    *grpc.ClientConn
   150  	agentClient  agentClient
   151  	metaClient   metaClient
   152  	pingSocketId int64
   153  	pingStream   *pingStream
   154  	agent        *agent
   155  }
   156  
   157  func newAgentGrpc(agent *agent) (*agentGrpc, error) {
   158  	conn, err := connectCollector(agent.config, CfgCollectorAgentPort)
   159  	if err != nil {
   160  		return nil, err
   161  	}
   162  
   163  	return &agentGrpc{
   164  		agentConn:   conn,
   165  		agentClient: &agentGrpcClient{pb.NewAgentClient(conn)},
   166  		metaClient:  &metaGrpcClient{pb.NewMetadataClient(conn)},
   167  		agent:       agent,
   168  	}, nil
   169  }
   170  
   171  func getHostName() string {
   172  	if hostName, err := os.Hostname(); err == nil {
   173  		return hostName
   174  	}
   175  	return "unknown host"
   176  }
   177  
   178  func getOutboundIP() string {
   179  	conn, err := net.Dial("udp", "8.8.8.8:80")
   180  	if err != nil {
   181  		return ""
   182  	}
   183  	defer conn.Close()
   184  
   185  	localAddr := conn.LocalAddr().(*net.UDPAddr)
   186  
   187  	return localAddr.IP.String()
   188  }
   189  
   190  func makeGoLibraryInfo() *pb.PServiceInfo {
   191  	libs := make([]string, 0)
   192  	if bi, ok := debug.ReadBuildInfo(); ok {
   193  		for _, dep := range bi.Deps {
   194  			libs = append(libs, dep.Path+" ("+dep.Version+")")
   195  		}
   196  	}
   197  
   198  	return &pb.PServiceInfo{
   199  		ServiceName: "Go (" + runtime.GOOS + ", " + runtime.GOARCH + ", " + runtime.GOROOT() + ")",
   200  		ServiceLib:  libs,
   201  	}
   202  }
   203  
   204  func (agentGrpc *agentGrpc) makeAgentInfo() (context.Context, *pb.PAgentInfo) {
   205  	agentInfo := &pb.PAgentInfo{
   206  		Hostname:     getHostName(),
   207  		Ip:           getOutboundIP(),
   208  		ServiceType:  agentGrpc.agent.appType,
   209  		Pid:          int32(os.Getpid()),
   210  		AgentVersion: Version,
   211  		VmVersion:    runtime.Version(),
   212  
   213  		ServerMetaData: &pb.PServerMetaData{
   214  			ServerInfo:  "Go Application",
   215  			VmArg:       os.Args[1:],
   216  			ServiceInfo: []*pb.PServiceInfo{makeGoLibraryInfo()},
   217  		},
   218  
   219  		JvmInfo: &pb.PJvmInfo{
   220  			Version:   0,
   221  			VmVersion: runtime.Version(),
   222  			GcType:    pb.PJvmGcType_JVM_GC_TYPE_CMS,
   223  		},
   224  		Container: agentGrpc.agent.config.Bool(CfgIsContainerEnv),
   225  	}
   226  
   227  	if IsLogLevelEnabled(logrus.DebugLevel) {
   228  		Log("grpc").Debugf("agent info: %s", agentInfo.String())
   229  	}
   230  
   231  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   232  	return ctx, agentInfo
   233  }
   234  
   235  func (agentGrpc *agentGrpc) sendAgentInfo(ctx context.Context, agentInfo *pb.PAgentInfo) (*pb.PResult, error) {
   236  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   237  	defer cancel()
   238  
   239  	result, err := agentGrpc.agentClient.RequestAgentInfo(ctx, agentInfo)
   240  	if err != nil {
   241  		Log("grpc").Errorf("send agent info - %v", err)
   242  	}
   243  
   244  	return result, err
   245  }
   246  
   247  func (agentGrpc *agentGrpc) registerAgentWithRetry() bool {
   248  	ctx, agentInfo := agentGrpc.makeAgentInfo()
   249  
   250  	for !agentGrpc.agent.shutdown {
   251  		if res, err := agentGrpc.sendAgentInfo(ctx, agentInfo); err == nil {
   252  			if res.Success {
   253  				Log("agent").Infof("success to register agent")
   254  				return true
   255  			} else {
   256  				Log("agent").Errorf("register agent - %s", res.Message)
   257  				break
   258  			}
   259  		}
   260  
   261  		for retry := 1; !agentGrpc.agent.shutdown; retry++ {
   262  			if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   263  				break
   264  			}
   265  		}
   266  		if agentInfo.Ip == "" {
   267  			agentInfo.Ip = getOutboundIP()
   268  		}
   269  	}
   270  	return false
   271  }
   272  
   273  func (agentGrpc *agentGrpc) sendApiMetadata(ctx context.Context, in *pb.PApiMetaData) error {
   274  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   275  	defer cancel()
   276  
   277  	_, err := agentGrpc.metaClient.RequestApiMetaData(ctx, in)
   278  	if err != nil {
   279  		Log("grpc").Errorf("send api metadata - %v", err)
   280  	}
   281  	return err
   282  }
   283  
   284  func (agentGrpc *agentGrpc) sendApiMetadataWithRetry(apiId int32, api string, line int, apiType int) bool {
   285  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   286  	apiMeta := pb.PApiMetaData{
   287  		ApiId:   apiId,
   288  		ApiInfo: api,
   289  		Line:    int32(line),
   290  		Type:    int32(apiType),
   291  	}
   292  
   293  	if IsLogLevelEnabled(logrus.DebugLevel) {
   294  		Log("grpc").Debugf("api metadata: %s", apiMeta.String())
   295  	}
   296  
   297  	for agentGrpc.agent.Enable() {
   298  		if err := agentGrpc.sendApiMetadata(ctx, &apiMeta); err == nil {
   299  			return true
   300  		}
   301  
   302  		if !agentGrpc.agent.config.offGrpc {
   303  			for retry := 1; agentGrpc.agent.Enable(); retry++ {
   304  				if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   305  					break
   306  				}
   307  			}
   308  		}
   309  	}
   310  	return false
   311  }
   312  
   313  func (agentGrpc *agentGrpc) sendStringMetadata(ctx context.Context, in *pb.PStringMetaData) error {
   314  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   315  	defer cancel()
   316  
   317  	_, err := agentGrpc.metaClient.RequestStringMetaData(ctx, in)
   318  	if err != nil {
   319  		Log("grpc").Errorf("send string metadata - %v", err)
   320  	}
   321  	return err
   322  }
   323  
   324  func (agentGrpc *agentGrpc) sendStringMetadataWithRetry(strId int32, str string) bool {
   325  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   326  	strMeta := pb.PStringMetaData{
   327  		StringId:    strId,
   328  		StringValue: str,
   329  	}
   330  
   331  	if IsLogLevelEnabled(logrus.DebugLevel) {
   332  		Log("grpc").Debugf("string metadata: %s", strMeta.String())
   333  	}
   334  
   335  	for agentGrpc.agent.Enable() {
   336  		if err := agentGrpc.sendStringMetadata(ctx, &strMeta); err == nil {
   337  			return true
   338  		}
   339  
   340  		if !agentGrpc.agent.config.offGrpc {
   341  			for retry := 1; agentGrpc.agent.Enable(); retry++ {
   342  				if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   343  					break
   344  				}
   345  			}
   346  		}
   347  	}
   348  	return false
   349  }
   350  
   351  func (agentGrpc *agentGrpc) sendSqlMetadata(ctx context.Context, in *pb.PSqlMetaData) error {
   352  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   353  	defer cancel()
   354  
   355  	_, err := agentGrpc.metaClient.RequestSqlMetaData(ctx, in)
   356  	if err != nil {
   357  		Log("grpc").Errorf("send sql metadata - %v", err)
   358  	}
   359  
   360  	return err
   361  }
   362  
   363  func (agentGrpc *agentGrpc) sendSqlMetadataWithRetry(sqlId int32, sql string) bool {
   364  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   365  	sqlMeta := pb.PSqlMetaData{
   366  		SqlId: sqlId,
   367  		Sql:   sql,
   368  	}
   369  
   370  	if IsLogLevelEnabled(logrus.DebugLevel) {
   371  		Log("grpc").Debugf("sql metadata: %s", sqlMeta.String())
   372  	}
   373  
   374  	for agentGrpc.agent.Enable() {
   375  		if err := agentGrpc.sendSqlMetadata(ctx, &sqlMeta); err == nil {
   376  			return true
   377  		}
   378  
   379  		if !agentGrpc.agent.config.offGrpc {
   380  			for retry := 1; agentGrpc.agent.Enable(); retry++ {
   381  				if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   382  					break
   383  				}
   384  			}
   385  		}
   386  	}
   387  	return false
   388  }
   389  
   390  func (agentGrpc *agentGrpc) sendSqlUidMetadata(ctx context.Context, in *pb.PSqlUidMetaData) error {
   391  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   392  	defer cancel()
   393  
   394  	_, err := agentGrpc.metaClient.RequestSqlUidMetaData(ctx, in)
   395  	if err != nil {
   396  		Log("grpc").Errorf("send sql uid metadata - %v", err)
   397  	}
   398  
   399  	return err
   400  }
   401  
   402  func (agentGrpc *agentGrpc) sendSqlUidMetadataWithRetry(sqlUid []byte, sql string) bool {
   403  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   404  	sqlUidMeta := pb.PSqlUidMetaData{
   405  		SqlUid: sqlUid,
   406  		Sql:    sql,
   407  	}
   408  
   409  	if IsLogLevelEnabled(logrus.DebugLevel) {
   410  		Log("grpc").Debugf("sql uid metadata: %s", sqlUidMeta.String())
   411  	}
   412  
   413  	for agentGrpc.agent.Enable() {
   414  		if err := agentGrpc.sendSqlUidMetadata(ctx, &sqlUidMeta); err == nil {
   415  			return true
   416  		}
   417  
   418  		if !agentGrpc.agent.config.offGrpc {
   419  			for retry := 1; agentGrpc.agent.Enable(); retry++ {
   420  				if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   421  					break
   422  				}
   423  			}
   424  		}
   425  	}
   426  	return false
   427  }
   428  
   429  func (agentGrpc *agentGrpc) sendExceptionMetadata(ctx context.Context, in *pb.PExceptionMetaData) error {
   430  	ctx, cancel := context.WithTimeout(ctx, agentGrpcTimeOut)
   431  	defer cancel()
   432  
   433  	_, err := agentGrpc.metaClient.RequestExceptionMetaData(ctx, in)
   434  	if err != nil {
   435  		Log("grpc").Errorf("send sql metadata - %v", err)
   436  	}
   437  
   438  	return err
   439  }
   440  
   441  func (agentGrpc *agentGrpc) sendExceptionMetadataWithRetry(exception *exceptionMeta) bool {
   442  	ctx := grpcMetadataContext(agentGrpc.agent, -1)
   443  	exceptMeta := makePExceptionMetaData(exception)
   444  
   445  	if IsLogLevelEnabled(logrus.DebugLevel) {
   446  		Log("grpc").Debugf("exception metadata: %s", exceptMeta.String())
   447  	}
   448  
   449  	for agentGrpc.agent.Enable() {
   450  		if err := agentGrpc.sendExceptionMetadata(ctx, exceptMeta); err == nil {
   451  			return true
   452  		}
   453  
   454  		if !agentGrpc.agent.config.offGrpc {
   455  			for retry := 1; agentGrpc.agent.Enable(); retry++ {
   456  				if waitUntilReady(agentGrpc.agentConn, backOffSleep(retry), "agent") {
   457  					break
   458  				}
   459  			}
   460  		}
   461  	}
   462  	return false
   463  }
   464  
   465  func makePExceptionMetaData(e *exceptionMeta) *pb.PExceptionMetaData {
   466  	return &pb.PExceptionMetaData{
   467  		TransactionId: &pb.PTransactionId{
   468  			AgentId:        e.txId.AgentId,
   469  			AgentStartTime: e.txId.StartTime,
   470  			Sequence:       e.txId.Sequence,
   471  		},
   472  		SpanId:      e.spanId,
   473  		UriTemplate: e.uriTemplate,
   474  		Exceptions:  makePExceptionList(e.exceptions),
   475  	}
   476  }
   477  
   478  func makePExceptionList(exceptions []*exception) []*pb.PException {
   479  	list := make([]*pb.PException, 0)
   480  	for _, e := range exceptions {
   481  		list = append(list, makePException(e))
   482  	}
   483  	return list
   484  }
   485  
   486  func makePException(e *exception) *pb.PException {
   487  	frames := e.callstack.stackTrace()
   488  	return &pb.PException{
   489  		ExceptionClassName: frames[0].moduleName,
   490  		ExceptionMessage:   e.callstack.err.Error(),
   491  		StartTime:          e.callstack.errorTime.UnixNano() / int64(time.Millisecond),
   492  		ExceptionId:        e.exceptionId,
   493  		ExceptionDepth:     1,
   494  		StackTraceElement:  makePStackTraceElementList(frames),
   495  	}
   496  }
   497  
   498  func makePStackTraceElementList(frames []frame) []*pb.PStackTraceElement {
   499  	list := make([]*pb.PStackTraceElement, 0)
   500  	for _, f := range frames {
   501  		list = append(list, &pb.PStackTraceElement{
   502  			ClassName:  f.moduleName,
   503  			FileName:   f.file,
   504  			LineNumber: f.line,
   505  			MethodName: f.funcName,
   506  		})
   507  	}
   508  	return list
   509  }
   510  
   511  func sendStreamWithTimeout(send func() error, timeout time.Duration, which string) error {
   512  	errChan := make(chan error, 1)
   513  	go func() {
   514  		errChan <- send()
   515  		close(errChan)
   516  	}()
   517  
   518  	t := time.NewTimer(timeout)
   519  	select {
   520  	case <-t.C:
   521  		return status.Errorf(codes.DeadlineExceeded, which+" stream.Send() - too slow or blocked")
   522  	case err := <-errChan:
   523  		if !t.Stop() {
   524  			<-t.C
   525  		}
   526  		return err
   527  	}
   528  }
   529  
   530  func waitUntilReady(grpcConn *grpc.ClientConn, timeout time.Duration, which string) bool {
   531  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
   532  	defer cancel()
   533  
   534  	state := grpcConn.GetState()
   535  	Log("grpc").Infof("wait %s connection ready - state: %s, timeout: %s", which, state.String(), timeout.String())
   536  
   537  	for state != connectivity.Ready {
   538  		if grpcConn.WaitForStateChange(ctx, state) {
   539  			state = grpcConn.GetState()
   540  		} else {
   541  			// To exit IDLE state, it needs to try grpc action.
   542  			if state == connectivity.Idle && timeout.Seconds() > 30 {
   543  				return true
   544  			} else {
   545  				return false
   546  			}
   547  		}
   548  	}
   549  
   550  	return true
   551  }
   552  
   553  func newStreamWithRetry(agent *agent, grpcConn *grpc.ClientConn, newStreamFunc func() bool, which string) bool {
   554  	for agent.Enable() {
   555  		if newStreamFunc() {
   556  			return true
   557  		}
   558  		if !agent.config.offGrpc {
   559  			for retry := 1; agent.Enable(); retry++ {
   560  				if waitUntilReady(grpcConn, backOffSleep(retry), which) {
   561  					break
   562  				}
   563  			}
   564  		}
   565  	}
   566  	return false
   567  }
   568  
   569  type pingStream struct {
   570  	stream pb.Agent_PingSessionClient
   571  }
   572  
   573  func (agentGrpc *agentGrpc) newPingStream() bool {
   574  	agentGrpc.pingSocketId++
   575  	ctx := grpcMetadataContext(agentGrpc.agent, agentGrpc.pingSocketId)
   576  	stream, err := agentGrpc.agentClient.PingSession(ctx)
   577  	if err != nil {
   578  		Log("grpc").Errorf("make ping stream - %v", err)
   579  		return false
   580  	}
   581  
   582  	agentGrpc.pingStream = &pingStream{stream}
   583  	return true
   584  }
   585  
   586  func (agentGrpc *agentGrpc) newPingStreamWithRetry() *pingStream {
   587  	if newStreamWithRetry(agentGrpc.agent, agentGrpc.agentConn, agentGrpc.newPingStream, "ping") {
   588  		Log("grpc").Infof("success to make ping stream")
   589  		return agentGrpc.pingStream
   590  	}
   591  	return &pingStream{}
   592  }
   593  
   594  var ping = pb.PPing{}
   595  
   596  func (s *pingStream) sendPing() error {
   597  	if s.stream == nil {
   598  		return status.Errorf(codes.Unavailable, "ping stream is nil")
   599  	}
   600  	err := sendStreamWithTimeout(func() error { return s.stream.Send(&ping) }, sendStreamTimeOut, "ping")
   601  	if err != nil {
   602  		return err
   603  	}
   604  
   605  	errChan := make(chan error, 1)
   606  	go func() {
   607  		_, err = s.stream.Recv()
   608  		errChan <- err
   609  		close(errChan)
   610  	}()
   611  
   612  	t := time.NewTimer(sendStreamTimeOut)
   613  	select {
   614  	case <-t.C:
   615  		return status.Errorf(codes.DeadlineExceeded, "ping stream.Recv() - too slow or blocked")
   616  	case err = <-errChan:
   617  		if !t.Stop() {
   618  			<-t.C
   619  		}
   620  		return err
   621  	}
   622  }
   623  
   624  func (s *pingStream) close() {
   625  	if s.stream == nil {
   626  		return
   627  	}
   628  	s.stream.CloseSend()
   629  	s.stream = nil
   630  	Log("grpc").Infof("close ping stream")
   631  }
   632  
   633  func (agentGrpc *agentGrpc) close() {
   634  	if agentGrpc.agentConn != nil {
   635  		agentGrpc.agentConn.Close()
   636  	}
   637  }
   638  
   639  type spanClient interface {
   640  	SendSpan(ctx context.Context) (pb.Span_SendSpanClient, error)
   641  }
   642  
   643  type spanGrpcClient struct {
   644  	client pb.SpanClient
   645  }
   646  
   647  func (spanGrpcClient *spanGrpcClient) SendSpan(ctx context.Context) (pb.Span_SendSpanClient, error) {
   648  	return spanGrpcClient.client.SendSpan(ctx)
   649  }
   650  
   651  type spanGrpc struct {
   652  	spanConn   *grpc.ClientConn
   653  	spanClient spanClient
   654  	stream     *spanStream
   655  	agent      *agent
   656  }
   657  
   658  type spanStream struct {
   659  	stream pb.Span_SendSpanClient
   660  }
   661  
   662  func newSpanGrpc(agent *agent) (*spanGrpc, error) {
   663  	conn, err := connectCollector(agent.config, CfgCollectorSpanPort)
   664  	if err != nil {
   665  		return nil, err
   666  	}
   667  
   668  	return &spanGrpc{
   669  		spanConn:   conn,
   670  		spanClient: &spanGrpcClient{pb.NewSpanClient(conn)},
   671  		agent:      agent,
   672  	}, nil
   673  }
   674  
   675  func (spanGrpc *spanGrpc) close() {
   676  	if spanGrpc.spanConn != nil {
   677  		spanGrpc.spanConn.Close()
   678  	}
   679  }
   680  
   681  func (spanGrpc *spanGrpc) newSpanStream() bool {
   682  	ctx := grpcMetadataContext(spanGrpc.agent, -1)
   683  	stream, err := spanGrpc.spanClient.SendSpan(ctx)
   684  	if err != nil {
   685  		Log("grpc").Errorf("make span stream - %v", err)
   686  		return false
   687  	}
   688  
   689  	spanGrpc.stream = &spanStream{stream}
   690  	return true
   691  }
   692  
   693  func (spanGrpc *spanGrpc) newSpanStreamWithRetry() *spanStream {
   694  	if newStreamWithRetry(spanGrpc.agent, spanGrpc.spanConn, spanGrpc.newSpanStream, "span") {
   695  		Log("grpc").Infof("success to make span stream")
   696  		return spanGrpc.stream
   697  	}
   698  	return &spanStream{}
   699  }
   700  
   701  func (s *spanStream) close() {
   702  	if s.stream == nil {
   703  		return
   704  	}
   705  	s.stream.CloseAndRecv()
   706  	s.stream = nil
   707  	Log("grpc").Infof("close span stream")
   708  }
   709  
   710  func (s *spanStream) sendSpan(span *span) error {
   711  	var gspan *pb.PSpanMessage
   712  
   713  	if s.stream == nil {
   714  		return status.Errorf(codes.Unavailable, "span stream is nil")
   715  	}
   716  
   717  	if span.isAsyncSpan() {
   718  		gspan = makePSpanChunk(span)
   719  	} else {
   720  		gspan = makePSpan(span)
   721  	}
   722  
   723  	if IsLogLevelEnabled(logrus.TraceLevel) {
   724  		Log("grpc").Tracef("PSpanMessage: %s", gspan.String())
   725  	}
   726  
   727  	return sendStreamWithTimeout(func() error { return s.stream.Send(gspan) }, sendStreamTimeOut, "span")
   728  }
   729  
   730  func makePSpan(span *span) *pb.PSpanMessage {
   731  	if span.apiId == 0 && span.operationName != "" {
   732  		span.annotations.AppendString(AnnotationApi, span.operationName)
   733  	}
   734  
   735  	spanEventList := make([]*pb.PSpanEvent, 0)
   736  	for _, event := range span.spanEvents {
   737  		aSpanEvent := makePSpanEvent(event)
   738  		spanEventList = append(spanEventList, aSpanEvent)
   739  	}
   740  
   741  	gspan := &pb.PSpanMessage{
   742  		Field: &pb.PSpanMessage_Span{
   743  			Span: &pb.PSpan{
   744  				Version: 1,
   745  				TransactionId: &pb.PTransactionId{
   746  					AgentId:        span.txId.AgentId,
   747  					AgentStartTime: span.txId.StartTime,
   748  					Sequence:       span.txId.Sequence,
   749  				},
   750  				SpanId:       span.spanId,
   751  				ParentSpanId: span.parentSpanId,
   752  				StartTime:    span.startTime.UnixNano() / int64(time.Millisecond),
   753  				Elapsed:      int32(span.elapsed),
   754  				ServiceType:  span.serviceType,
   755  				AcceptEvent: &pb.PAcceptEvent{
   756  					Rpc:        span.rpcName,
   757  					EndPoint:   span.endPoint,
   758  					RemoteAddr: span.remoteAddr,
   759  					ParentInfo: &pb.PParentInfo{
   760  						ParentApplicationName: span.parentAppName,
   761  						ParentApplicationType: int32(span.parentAppType),
   762  						AcceptorHost:          span.acceptorHost,
   763  					},
   764  				},
   765  				Annotation:             span.annotations.list,
   766  				ApiId:                  span.apiId,
   767  				Flag:                   int32(span.flags),
   768  				SpanEvent:              spanEventList,
   769  				Err:                    int32(span.err),
   770  				ExceptionInfo:          nil,
   771  				ApplicationServiceType: span.agent.appType,
   772  				LoggingTransactionInfo: span.loggingInfo,
   773  			},
   774  		},
   775  	}
   776  
   777  	if span.errorString != "" {
   778  		gspan.GetSpan().ExceptionInfo = &pb.PIntStringValue{
   779  			IntValue:    span.errorFuncId,
   780  			StringValue: &wrappers.StringValue{Value: span.errorString},
   781  		}
   782  	}
   783  
   784  	return gspan
   785  }
   786  
   787  func makePSpanChunk(span *span) *pb.PSpanMessage {
   788  	spanEventList := make([]*pb.PSpanEvent, 0)
   789  	for _, event := range span.spanEvents {
   790  		aSpanEvent := makePSpanEvent(event)
   791  		spanEventList = append(spanEventList, aSpanEvent)
   792  	}
   793  
   794  	gspan := &pb.PSpanMessage{
   795  		Field: &pb.PSpanMessage_SpanChunk{
   796  			SpanChunk: &pb.PSpanChunk{
   797  				Version: 1,
   798  				TransactionId: &pb.PTransactionId{
   799  					AgentId:        span.txId.AgentId,
   800  					AgentStartTime: span.txId.StartTime,
   801  					Sequence:       span.txId.Sequence,
   802  				},
   803  				SpanId:                 span.spanId,
   804  				KeyTime:                span.startTime.UnixNano() / int64(time.Millisecond),
   805  				EndPoint:               span.endPoint,
   806  				SpanEvent:              spanEventList,
   807  				ApplicationServiceType: span.agent.appType,
   808  				LocalAsyncId: &pb.PLocalAsyncId{
   809  					AsyncId:  span.asyncId,
   810  					Sequence: span.asyncSequence,
   811  				},
   812  			},
   813  		},
   814  	}
   815  
   816  	return gspan
   817  }
   818  
   819  func makePSpanEvent(event *spanEvent) *pb.PSpanEvent {
   820  	if event.apiId == 0 && event.operationName != "" {
   821  		event.annotations.AppendString(AnnotationApi, event.operationName)
   822  	}
   823  
   824  	aSpanEvent := pb.PSpanEvent{
   825  		Sequence:      event.sequence,
   826  		Depth:         event.depth,
   827  		StartElapsed:  int32(event.startElapsed),
   828  		EndElapsed:    int32(event.endElapsed),
   829  		ServiceType:   event.serviceType,
   830  		Annotation:    event.annotations.list,
   831  		ApiId:         event.apiId,
   832  		ExceptionInfo: nil,
   833  		NextEvent:     nil,
   834  		AsyncEvent:    event.asyncId,
   835  	}
   836  
   837  	if event.errorString != "" {
   838  		aSpanEvent.ExceptionInfo = &pb.PIntStringValue{
   839  			IntValue:    event.errorFuncId,
   840  			StringValue: &wrappers.StringValue{Value: event.errorString},
   841  		}
   842  	}
   843  
   844  	if event.destinationId != "" {
   845  		next := &pb.PNextEvent{
   846  			Field: &pb.PNextEvent_MessageEvent{
   847  				MessageEvent: &pb.PMessageEvent{
   848  					NextSpanId:    event.nextSpanId,
   849  					EndPoint:      event.endPoint,
   850  					DestinationId: event.destinationId,
   851  				},
   852  			},
   853  		}
   854  
   855  		aSpanEvent.NextEvent = next
   856  	}
   857  
   858  	return &aSpanEvent
   859  }
   860  
   861  type statClient interface {
   862  	SendAgentStat(ctx context.Context) (pb.Stat_SendAgentStatClient, error)
   863  }
   864  
   865  type statGrpcClient struct {
   866  	client pb.StatClient
   867  }
   868  
   869  func (statGrpcClient *statGrpcClient) SendAgentStat(ctx context.Context) (pb.Stat_SendAgentStatClient, error) {
   870  	return statGrpcClient.client.SendAgentStat(ctx)
   871  }
   872  
   873  type statGrpc struct {
   874  	statConn   *grpc.ClientConn
   875  	statClient statClient
   876  	stream     *statStream
   877  	agent      *agent
   878  }
   879  
   880  type statStream struct {
   881  	stream pb.Stat_SendAgentStatClient
   882  }
   883  
   884  func newStatGrpc(agent *agent) (*statGrpc, error) {
   885  	conn, err := connectCollector(agent.config, CfgCollectorStatPort)
   886  	if err != nil {
   887  		return nil, err
   888  	}
   889  
   890  	return &statGrpc{
   891  		statConn:   conn,
   892  		statClient: &statGrpcClient{pb.NewStatClient(conn)},
   893  		agent:      agent,
   894  	}, nil
   895  }
   896  
   897  func (statGrpc *statGrpc) close() {
   898  	if statGrpc.statConn != nil {
   899  		statGrpc.statConn.Close()
   900  	}
   901  }
   902  
   903  func (statGrpc *statGrpc) newStatStream() bool {
   904  	ctx := grpcMetadataContext(statGrpc.agent, -1)
   905  	stream, err := statGrpc.statClient.SendAgentStat(ctx)
   906  	if err != nil {
   907  		Log("grpc").Errorf("make stat stream - %v", err)
   908  		return false
   909  	}
   910  
   911  	statGrpc.stream = &statStream{stream}
   912  	return true
   913  }
   914  
   915  func (statGrpc *statGrpc) newStatStreamWithRetry() *statStream {
   916  	if newStreamWithRetry(statGrpc.agent, statGrpc.statConn, statGrpc.newStatStream, "stat") {
   917  		Log("grpc").Infof("success to make stat stream")
   918  		return statGrpc.stream
   919  	}
   920  	return &statStream{}
   921  }
   922  
   923  func (s *statStream) close() {
   924  	if s.stream == nil {
   925  		return
   926  	}
   927  	s.stream.CloseAndRecv()
   928  	s.stream = nil
   929  	Log("grpc").Infof("close stat stream")
   930  }
   931  
   932  func (s *statStream) sendStats(stats *pb.PStatMessage) error {
   933  	if s.stream == nil {
   934  		return status.Errorf(codes.Unavailable, "stat stream is nil")
   935  	}
   936  	if IsLogLevelEnabled(logrus.TraceLevel) {
   937  		Log("grpc").Tracef("PStatMessage: %s", stats.String())
   938  	}
   939  	return sendStreamWithTimeout(func() error { return s.stream.Send(stats) }, sendStreamTimeOut, "stat")
   940  }
   941  
   942  func makePAgentStatBatch(stats []*inspectorStats) *pb.PStatMessage {
   943  	l := make([]*pb.PAgentStat, 0)
   944  	for _, s := range stats {
   945  		l = append(l, makePAgentStat(s))
   946  	}
   947  	return &pb.PStatMessage{
   948  		Field: &pb.PStatMessage_AgentStatBatch{
   949  			AgentStatBatch: &pb.PAgentStatBatch{
   950  				AgentStat: l,
   951  			},
   952  		},
   953  	}
   954  }
   955  
   956  func makePAgentStat(stat *inspectorStats) *pb.PAgentStat {
   957  	return &pb.PAgentStat{
   958  		Timestamp:       stat.sampleTime.UnixNano() / int64(time.Millisecond),
   959  		CollectInterval: stat.interval,
   960  		Gc: &pb.PJvmGc{
   961  			Type:                 pb.PJvmGcType_JVM_GC_TYPE_CMS,
   962  			JvmMemoryHeapUsed:    stat.heapUsed,
   963  			JvmMemoryHeapMax:     stat.heapMax,
   964  			JvmMemoryNonHeapUsed: stat.nonHeapUsed,
   965  			JvmMemoryNonHeapMax:  stat.nonHeapMax,
   966  			JvmGcOldCount:        stat.gcNum,
   967  			JvmGcOldTime:         stat.gcTime,
   968  			JvmGcDetailed:        nil,
   969  		},
   970  		CpuLoad: &pb.PCpuLoad{
   971  			JvmCpuLoad:    stat.cpuProcLoad,
   972  			SystemCpuLoad: stat.cpuSysLoad,
   973  		},
   974  		Transaction: &pb.PTransaction{
   975  			SampledNewCount:            stat.sampleNew,
   976  			SampledContinuationCount:   stat.sampleCont,
   977  			UnsampledNewCount:          stat.unSampleNew,
   978  			UnsampledContinuationCount: stat.unSampleCont,
   979  			SkippedNewCount:            stat.skipNew,
   980  			SkippedContinuationCount:   stat.skipCont,
   981  		},
   982  		ActiveTrace: &pb.PActiveTrace{
   983  			Histogram: &pb.PActiveTraceHistogram{
   984  				Version:             1,
   985  				HistogramSchemaType: 2, //NORMAL SCHEMA
   986  				ActiveTraceCount:    stat.activeSpan,
   987  			},
   988  		},
   989  		DataSourceList: nil,
   990  		ResponseTime: &pb.PResponseTime{
   991  			Avg: stat.responseAvg,
   992  			Max: stat.responseMax,
   993  		},
   994  		Deadlock: nil,
   995  		FileDescriptor: &pb.PFileDescriptor{
   996  			OpenFileDescriptorCount: stat.numOpenFD,
   997  		},
   998  		DirectBuffer: nil,
   999  		Metadata:     "",
  1000  		TotalThread: &pb.PTotalThread{
  1001  			TotalThreadCount: stat.numThreads,
  1002  		},
  1003  		LoadedClass: nil,
  1004  	}
  1005  }
  1006  
  1007  func makePAgentUriStat(stat *urlStatSnapshot) *pb.PStatMessage {
  1008  	return &pb.PStatMessage{
  1009  		Field: &pb.PStatMessage_AgentUriStat{
  1010  			AgentUriStat: &pb.PAgentUriStat{
  1011  				BucketVersion: urlStatBucketVersion,
  1012  				EachUriStat:   makePEachUriStatList(stat),
  1013  			},
  1014  		},
  1015  	}
  1016  }
  1017  
  1018  func makePEachUriStatList(stat *urlStatSnapshot) []*pb.PEachUriStat {
  1019  	l := make([]*pb.PEachUriStat, 0)
  1020  	for _, e := range stat.urlMap {
  1021  		l = append(l, makePEachUriStat(e))
  1022  	}
  1023  	return l
  1024  }
  1025  
  1026  func makePEachUriStat(e *eachUrlStat) *pb.PEachUriStat {
  1027  	return &pb.PEachUriStat{
  1028  		Uri:             e.url,
  1029  		TotalHistogram:  makePUriHistogram(e.totalHistogram),
  1030  		FailedHistogram: makePUriHistogram(e.failedHistogram),
  1031  		Timestamp:       e.tickTime.UnixNano() / int64(time.Millisecond),
  1032  	}
  1033  }
  1034  
  1035  func makePUriHistogram(h *urlStatHistogram) *pb.PUriHistogram {
  1036  	return &pb.PUriHistogram{
  1037  		Total:     h.total,
  1038  		Max:       h.max,
  1039  		Histogram: h.histogram,
  1040  	}
  1041  }
  1042  
  1043  type cmdGrpc struct {
  1044  	cmdConn   *grpc.ClientConn
  1045  	cmdClient pb.ProfilerCommandServiceClient
  1046  	stream    *cmdStream
  1047  	agent     *agent
  1048  }
  1049  
  1050  type cmdStream struct {
  1051  	stream pb.ProfilerCommandService_HandleCommandClient
  1052  }
  1053  
  1054  func newCommandGrpc(agent *agent) (*cmdGrpc, error) {
  1055  	conn, err := connectCollector(agent.config, CfgCollectorAgentPort)
  1056  	if err != nil {
  1057  		return nil, err
  1058  	}
  1059  
  1060  	cmdClient := pb.NewProfilerCommandServiceClient(conn)
  1061  	return &cmdGrpc{cmdConn: conn, cmdClient: cmdClient, agent: agent}, nil
  1062  }
  1063  
  1064  func (cmdGrpc *cmdGrpc) close() {
  1065  	if cmdGrpc.cmdConn != nil {
  1066  		cmdGrpc.cmdConn.Close()
  1067  	}
  1068  }
  1069  
  1070  func (cmdGrpc *cmdGrpc) newHandleCommandStream() bool {
  1071  	ctx := grpcMetadataContext(cmdGrpc.agent, -1)
  1072  	stream, err := cmdGrpc.cmdClient.HandleCommand(ctx)
  1073  	if err != nil {
  1074  		Log("grpc").Errorf("make command stream - %v", err)
  1075  		return false
  1076  	}
  1077  
  1078  	cmdGrpc.stream = &cmdStream{stream}
  1079  	return true
  1080  }
  1081  
  1082  func (cmdGrpc *cmdGrpc) newCommandStreamWithRetry() *cmdStream {
  1083  	if newStreamWithRetry(cmdGrpc.agent, cmdGrpc.cmdConn, cmdGrpc.newHandleCommandStream, "command") {
  1084  		Log("grpc").Infof("success to make command stream")
  1085  		return cmdGrpc.stream
  1086  	}
  1087  	return &cmdStream{}
  1088  }
  1089  
  1090  func (s *cmdStream) close() {
  1091  	if s.stream == nil {
  1092  		return
  1093  	}
  1094  	s.stream.CloseSend()
  1095  	s.stream = nil
  1096  	Log("grpc").Infof("close command stream")
  1097  }
  1098  
  1099  func (s *cmdStream) sendCommandMessage() error {
  1100  	var gCmd *pb.PCmdMessage
  1101  
  1102  	if s.stream == nil {
  1103  		return status.Errorf(codes.Unavailable, "command stream is nil")
  1104  	}
  1105  
  1106  	sKeys := make([]int32, 0)
  1107  	sKeys = append(sKeys, int32(pb.PCommandType_ECHO))
  1108  	sKeys = append(sKeys, int32(pb.PCommandType_ACTIVE_THREAD_COUNT))
  1109  	sKeys = append(sKeys, int32(pb.PCommandType_ACTIVE_THREAD_DUMP))
  1110  	sKeys = append(sKeys, int32(pb.PCommandType_ACTIVE_THREAD_LIGHT_DUMP))
  1111  
  1112  	gCmd = &pb.PCmdMessage{
  1113  		Message: &pb.PCmdMessage_HandshakeMessage{
  1114  			HandshakeMessage: &pb.PCmdServiceHandshake{
  1115  				SupportCommandServiceKey: sKeys,
  1116  			},
  1117  		},
  1118  	}
  1119  
  1120  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1121  		Log("grpc").Debugf("PCmdMessage: %s", gCmd.String())
  1122  	}
  1123  	return sendStreamWithTimeout(func() error { return s.stream.Send(gCmd) }, sendStreamTimeOut, "cmd")
  1124  }
  1125  
  1126  func (s *cmdStream) recvCommandRequest() (*pb.PCmdRequest, error) {
  1127  	var gCmdReq *pb.PCmdRequest
  1128  
  1129  	if s.stream == nil {
  1130  		return nil, status.Errorf(codes.Unavailable, "command stream is nil")
  1131  	}
  1132  
  1133  	gCmdReq, err := s.stream.Recv()
  1134  	if err != nil {
  1135  		return nil, err
  1136  	}
  1137  
  1138  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1139  		Log("grpc").Debugf("PCmdRequest: %s", gCmdReq.String())
  1140  	}
  1141  	return gCmdReq, nil
  1142  }
  1143  
  1144  type activeThreadCountStream struct {
  1145  	stream   pb.ProfilerCommandService_CommandStreamActiveThreadCountClient
  1146  	reqId    int32
  1147  	actCount int32
  1148  }
  1149  
  1150  func (cmdGrpc *cmdGrpc) newActiveThreadCountStream(reqId int32) *activeThreadCountStream {
  1151  	ctx := grpcMetadataContext(cmdGrpc.agent, -1)
  1152  	stream, err := cmdGrpc.cmdClient.CommandStreamActiveThreadCount(ctx)
  1153  	if err != nil {
  1154  		Log("grpc").Errorf("make active thread count stream - %v", err)
  1155  		return &activeThreadCountStream{nil, -1, 0}
  1156  	}
  1157  
  1158  	return &activeThreadCountStream{stream, reqId, 0}
  1159  }
  1160  
  1161  func (s *activeThreadCountStream) close() {
  1162  	if s.stream == nil {
  1163  		return
  1164  	}
  1165  	s.stream.CloseSend()
  1166  	s.stream = nil
  1167  }
  1168  
  1169  func (s *activeThreadCountStream) sendActiveThreadCount() error {
  1170  	var gRes *pb.PCmdActiveThreadCountRes
  1171  
  1172  	if s.stream == nil {
  1173  		return status.Errorf(codes.Unavailable, "active thread count stream is nil")
  1174  	}
  1175  
  1176  	now := time.Now()
  1177  	activeThreadCount := getActiveSpanCount(now)
  1178  	s.actCount++
  1179  
  1180  	gRes = &pb.PCmdActiveThreadCountRes{
  1181  		CommonStreamResponse: &pb.PCmdStreamResponse{
  1182  			ResponseId: s.reqId,
  1183  			SequenceId: s.actCount,
  1184  			Message:    &wrappers.StringValue{Value: ""},
  1185  		},
  1186  		HistogramSchemaType: 2,
  1187  		ActiveThreadCount:   activeThreadCount,
  1188  		TimeStamp:           now.UnixNano() / int64(time.Millisecond),
  1189  	}
  1190  
  1191  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1192  		Log("grpc").Debugf("PCmdActiveThreadCountRes: %s", gRes.String())
  1193  	}
  1194  	return sendStreamWithTimeout(func() error { return s.stream.Send(gRes) }, sendStreamTimeOut, "arc")
  1195  }
  1196  
  1197  func (cmdGrpc *cmdGrpc) sendActiveThreadDump(reqId int32, limit int32, threadName []string, localId []int64, dump *goroutineDump) {
  1198  	var gRes *pb.PCmdActiveThreadDumpRes
  1199  
  1200  	status := int32(0)
  1201  	msg := ""
  1202  
  1203  	if dump == nil {
  1204  		status = -1
  1205  		msg = "An error occurred while dumping Goroutine"
  1206  	}
  1207  
  1208  	gRes = &pb.PCmdActiveThreadDumpRes{
  1209  		CommonResponse: &pb.PCmdResponse{
  1210  			ResponseId: reqId,
  1211  			Status:     status,
  1212  			Message:    &wrappers.StringValue{Value: msg},
  1213  		},
  1214  		ThreadDump: makePActiveThreadDumpList(dump, int(limit), threadName, localId),
  1215  		Type:       "Go",
  1216  		SubType:    "",
  1217  		Version:    runtime.Version(),
  1218  	}
  1219  
  1220  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1221  		Log("grpc").Debugf("send PCmdActiveThreadDumpRes: %s", gRes.String())
  1222  	}
  1223  
  1224  	ctx := grpcMetadataContext(cmdGrpc.agent, -1)
  1225  	_, err := cmdGrpc.cmdClient.CommandActiveThreadDump(ctx, gRes)
  1226  	if err != nil {
  1227  		Log("grpc").Errorf("send active thread dump - %v", err)
  1228  	}
  1229  }
  1230  
  1231  func makePActiveThreadDumpList(dump *goroutineDump, limit int, threadName []string, localId []int64) []*pb.PActiveThreadDump {
  1232  	dumpList := make([]*pb.PActiveThreadDump, 0)
  1233  
  1234  	if dump != nil {
  1235  		if limit < 1 {
  1236  			limit = len(dump.goroutines)
  1237  		}
  1238  
  1239  		selected := make([]*goroutine, 0)
  1240  		for _, tn := range threadName {
  1241  			g := dump.search(tn)
  1242  			if g != nil {
  1243  				selected = append(selected, g)
  1244  			}
  1245  		}
  1246  
  1247  		if IsLogLevelEnabled(logrus.DebugLevel) {
  1248  			Log("grpc").Debugf("send makePActiveThreadDumpList: %v", selected)
  1249  		}
  1250  
  1251  		for i := 0; i < limit && i < len(selected); i++ {
  1252  			aDump := makePActiveThreadDump(selected[i])
  1253  			dumpList = append(dumpList, aDump)
  1254  		}
  1255  	}
  1256  
  1257  	return dumpList
  1258  }
  1259  
  1260  func makePActiveThreadDump(g *goroutine) *pb.PActiveThreadDump {
  1261  	aDump := &pb.PActiveThreadDump{
  1262  		StartTime:    g.span.startTime.UnixNano() / int64(time.Millisecond),
  1263  		LocalTraceId: 0,
  1264  		ThreadDump: &pb.PThreadDump{
  1265  			ThreadName:         g.header,
  1266  			ThreadId:           g.id,
  1267  			BlockedTime:        0,
  1268  			BlockedCount:       0,
  1269  			WaitedTime:         0,
  1270  			WaitedCount:        0,
  1271  			LockName:           "",
  1272  			LockOwnerId:        0,
  1273  			LockOwnerName:      "",
  1274  			InNative:           false,
  1275  			Suspended:          false,
  1276  			ThreadState:        g.threadState(),
  1277  			StackTrace:         g.stackTrace(),
  1278  			LockedMonitor:      nil,
  1279  			LockedSynchronizer: nil,
  1280  		},
  1281  		Sampled:       g.span.sampled,
  1282  		TransactionId: g.span.txId,
  1283  		EntryPoint:    g.span.entryPoint,
  1284  	}
  1285  
  1286  	return aDump
  1287  }
  1288  
  1289  func (cmdGrpc *cmdGrpc) sendActiveThreadLightDump(reqId int32, limit int32, dump *goroutineDump) {
  1290  	var gRes *pb.PCmdActiveThreadLightDumpRes
  1291  
  1292  	status := int32(0)
  1293  	msg := ""
  1294  
  1295  	if dump == nil {
  1296  		status = -1
  1297  		msg = "An error occurred while dumping Goroutine"
  1298  	}
  1299  
  1300  	gRes = &pb.PCmdActiveThreadLightDumpRes{
  1301  		CommonResponse: &pb.PCmdResponse{
  1302  			ResponseId: reqId,
  1303  			Status:     status,                            //error
  1304  			Message:    &wrappers.StringValue{Value: msg}, //error message
  1305  		},
  1306  		ThreadDump: makePActiveThreadLightDumpList(dump, int(limit)),
  1307  		Type:       "Go",
  1308  		SubType:    "",
  1309  		Version:    runtime.Version(),
  1310  	}
  1311  
  1312  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1313  		Log("grpc").Debugf("send PCmdActiveThreadLightDumpRes: %s", gRes.String())
  1314  	}
  1315  
  1316  	ctx := grpcMetadataContext(cmdGrpc.agent, -1)
  1317  	_, err := cmdGrpc.cmdClient.CommandActiveThreadLightDump(ctx, gRes)
  1318  	if err != nil {
  1319  		Log("grpc").Errorf("send active thread light dump - %v", err)
  1320  	}
  1321  }
  1322  
  1323  func makePActiveThreadLightDumpList(dump *goroutineDump, limit int) []*pb.PActiveThreadLightDump {
  1324  	dumpList := make([]*pb.PActiveThreadLightDump, 0)
  1325  
  1326  	if dump != nil {
  1327  		if limit < 1 {
  1328  			limit = len(dump.goroutines)
  1329  		}
  1330  
  1331  		for i := 0; i < limit && i < len(dump.goroutines); i++ {
  1332  			aDump := makePActiveThreadLightDump(dump.goroutines[i])
  1333  			dumpList = append(dumpList, aDump)
  1334  		}
  1335  	}
  1336  
  1337  	return dumpList
  1338  }
  1339  
  1340  func makePActiveThreadLightDump(g *goroutine) *pb.PActiveThreadLightDump {
  1341  	aDump := &pb.PActiveThreadLightDump{
  1342  		StartTime:    g.span.startTime.UnixNano() / int64(time.Millisecond),
  1343  		LocalTraceId: 0,
  1344  		ThreadDump: &pb.PThreadLightDump{
  1345  			ThreadName:  g.header,
  1346  			ThreadId:    g.id,
  1347  			ThreadState: g.threadState(),
  1348  		},
  1349  		Sampled:       g.span.sampled,
  1350  		TransactionId: g.span.txId,
  1351  		EntryPoint:    g.span.entryPoint,
  1352  	}
  1353  
  1354  	return aDump
  1355  }
  1356  
  1357  func (cmdGrpc *cmdGrpc) sendEcho(reqId int32, msg string) {
  1358  	var gRes *pb.PCmdEchoResponse
  1359  
  1360  	gRes = &pb.PCmdEchoResponse{
  1361  		CommonResponse: &pb.PCmdResponse{
  1362  			ResponseId: reqId,
  1363  			Status:     0,                                //error
  1364  			Message:    &wrappers.StringValue{Value: ""}, //error message
  1365  		},
  1366  		Message: msg,
  1367  	}
  1368  
  1369  	if IsLogLevelEnabled(logrus.DebugLevel) {
  1370  		Log("grpc").Debugf("send PCmdEchoResponse: %s", gRes.String())
  1371  	}
  1372  
  1373  	ctx := grpcMetadataContext(cmdGrpc.agent, -1)
  1374  	_, err := cmdGrpc.cmdClient.CommandEcho(ctx, gRes)
  1375  	if err != nil {
  1376  		Log("grpc").Errorf("send echo response - %v", err)
  1377  	}
  1378  }