github.com/newrelic/go-agent@v3.26.0+incompatible/_integrations/nrawssdk/v2/nrawssdk_test.go (about)

     1  // Copyright 2020 New Relic Corporation. All rights reserved.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package nrawssdk
     5  
     6  import (
     7  	"bytes"
     8  	"context"
     9  	"errors"
    10  	"io/ioutil"
    11  	"net/http"
    12  	"testing"
    13  
    14  	"github.com/aws/aws-sdk-go-v2/aws"
    15  	"github.com/aws/aws-sdk-go-v2/aws/external"
    16  	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
    17  	"github.com/aws/aws-sdk-go-v2/service/lambda"
    18  	newrelic "github.com/newrelic/go-agent"
    19  	"github.com/newrelic/go-agent/internal"
    20  	"github.com/newrelic/go-agent/internal/integrationsupport"
    21  )
    22  
    23  func testApp() integrationsupport.ExpectApp {
    24  	return integrationsupport.NewTestApp(integrationsupport.SampleEverythingReplyFn, integrationsupport.DTEnabledCfgFn)
    25  }
    26  
    27  type fakeTransport struct{}
    28  
    29  func (t fakeTransport) RoundTrip(r *http.Request) (*http.Response, error) {
    30  	return &http.Response{
    31  		Status:     "200 OK",
    32  		StatusCode: 200,
    33  		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
    34  		Header: http.Header{
    35  			"X-Amzn-Requestid": []string{requestID},
    36  		},
    37  	}, nil
    38  }
    39  
    40  type fakeCredsWithoutContext struct{}
    41  
    42  func (c fakeCredsWithoutContext) Retrieve() (aws.Credentials, error) {
    43  	return aws.Credentials{}, nil
    44  }
    45  
    46  type fakeCredsWithContext struct{}
    47  
    48  func (c fakeCredsWithContext) Retrieve(ctx context.Context) (aws.Credentials, error) {
    49  	return aws.Credentials{}, nil
    50  }
    51  
    52  var fakeCreds = func() interface{} {
    53  	var c interface{} = fakeCredsWithoutContext{}
    54  	if _, ok := c.(aws.CredentialsProvider); ok {
    55  		return c
    56  	}
    57  	return fakeCredsWithContext{}
    58  }()
    59  
    60  func newConfig(instrument bool) aws.Config {
    61  	cfg, _ := external.LoadDefaultAWSConfig()
    62  	cfg.Credentials = fakeCreds.(aws.CredentialsProvider)
    63  	cfg.Region = "us-west-2"
    64  	cfg.HTTPClient = &http.Client{
    65  		Transport: &fakeTransport{},
    66  	}
    67  
    68  	if instrument {
    69  		InstrumentHandlers(&cfg.Handlers)
    70  	}
    71  	return cfg
    72  }
    73  
    74  const (
    75  	requestID = "testing request id"
    76  	txnName   = "aws-txn"
    77  )
    78  
    79  var (
    80  	genericSpan = internal.WantEvent{
    81  		Intrinsics: map[string]interface{}{
    82  			"name":          "OtherTransaction/Go/" + txnName,
    83  			"sampled":       true,
    84  			"category":      "generic",
    85  			"priority":      internal.MatchAnything,
    86  			"guid":          internal.MatchAnything,
    87  			"transactionId": internal.MatchAnything,
    88  			"nr.entryPoint": true,
    89  			"traceId":       internal.MatchAnything,
    90  		},
    91  		UserAttributes:  map[string]interface{}{},
    92  		AgentAttributes: map[string]interface{}{},
    93  	}
    94  	externalSpan = internal.WantEvent{
    95  		Intrinsics: map[string]interface{}{
    96  			"name":          "External/lambda.us-west-2.amazonaws.com/http/POST",
    97  			"sampled":       true,
    98  			"category":      "http",
    99  			"priority":      internal.MatchAnything,
   100  			"guid":          internal.MatchAnything,
   101  			"transactionId": internal.MatchAnything,
   102  			"traceId":       internal.MatchAnything,
   103  			"parentId":      internal.MatchAnything,
   104  			"component":     "http",
   105  			"span.kind":     "client",
   106  		},
   107  		UserAttributes: map[string]interface{}{},
   108  		AgentAttributes: map[string]interface{}{
   109  			"aws.operation": "Invoke",
   110  			"aws.region":    "us-west-2",
   111  			"aws.requestId": requestID,
   112  			"http.method":   "POST",
   113  			"http.url":      "https://lambda.us-west-2.amazonaws.com/2015-03-31/functions/non-existent-function/invocations",
   114  		},
   115  	}
   116  	externalSpanNoRequestID = internal.WantEvent{
   117  		Intrinsics: map[string]interface{}{
   118  			"name":          "External/lambda.us-west-2.amazonaws.com/http/POST",
   119  			"sampled":       true,
   120  			"category":      "http",
   121  			"priority":      internal.MatchAnything,
   122  			"guid":          internal.MatchAnything,
   123  			"transactionId": internal.MatchAnything,
   124  			"traceId":       internal.MatchAnything,
   125  			"parentId":      internal.MatchAnything,
   126  			"component":     "http",
   127  			"span.kind":     "client",
   128  		},
   129  		UserAttributes: map[string]interface{}{},
   130  		AgentAttributes: map[string]interface{}{
   131  			"aws.operation": "Invoke",
   132  			"aws.region":    "us-west-2",
   133  			"http.method":   "POST",
   134  			"http.url":      "https://lambda.us-west-2.amazonaws.com/2015-03-31/functions/non-existent-function/invocations",
   135  		},
   136  	}
   137  	datastoreSpan = internal.WantEvent{
   138  		Intrinsics: map[string]interface{}{
   139  			"name":          "Datastore/statement/DynamoDB/thebesttable/DescribeTable",
   140  			"sampled":       true,
   141  			"category":      "datastore",
   142  			"priority":      internal.MatchAnything,
   143  			"guid":          internal.MatchAnything,
   144  			"transactionId": internal.MatchAnything,
   145  			"traceId":       internal.MatchAnything,
   146  			"parentId":      internal.MatchAnything,
   147  			"component":     "DynamoDB",
   148  			"span.kind":     "client",
   149  		},
   150  		UserAttributes: map[string]interface{}{},
   151  		AgentAttributes: map[string]interface{}{
   152  			"aws.operation": "DescribeTable",
   153  			"aws.region":    "us-west-2",
   154  			"aws.requestId": requestID,
   155  			"db.collection": "thebesttable",
   156  			"db.statement":  "'DescribeTable' on 'thebesttable' using 'DynamoDB'",
   157  			"peer.address":  "dynamodb.us-west-2.amazonaws.com:unknown",
   158  			"peer.hostname": "dynamodb.us-west-2.amazonaws.com",
   159  		},
   160  	}
   161  
   162  	txnMetrics = []internal.WantMetric{
   163  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/all", Scope: "", Forced: false, Data: nil},
   164  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: nil},
   165  		{Name: "OtherTransaction/Go/" + txnName, Scope: "", Forced: true, Data: nil},
   166  		{Name: "OtherTransaction/all", Scope: "", Forced: true, Data: nil},
   167  		{Name: "OtherTransactionTotalTime/Go/" + txnName, Scope: "", Forced: false, Data: nil},
   168  		{Name: "OtherTransactionTotalTime", Scope: "", Forced: true, Data: nil},
   169  	}
   170  	externalMetrics = append(txnMetrics, []internal.WantMetric{
   171  		{Name: "External/all", Scope: "", Forced: true, Data: nil},
   172  		{Name: "External/allOther", Scope: "", Forced: true, Data: nil},
   173  		{Name: "External/lambda.us-west-2.amazonaws.com/all", Scope: "", Forced: false, Data: nil},
   174  		{Name: "External/lambda.us-west-2.amazonaws.com/http/POST", Scope: "OtherTransaction/Go/" + txnName, Forced: false, Data: nil},
   175  	}...)
   176  	datastoreMetrics = append(txnMetrics, []internal.WantMetric{
   177  		{Name: "Datastore/DynamoDB/all", Scope: "", Forced: true, Data: nil},
   178  		{Name: "Datastore/DynamoDB/allOther", Scope: "", Forced: true, Data: nil},
   179  		{Name: "Datastore/all", Scope: "", Forced: true, Data: nil},
   180  		{Name: "Datastore/allOther", Scope: "", Forced: true, Data: nil},
   181  		{Name: "Datastore/instance/DynamoDB/dynamodb.us-west-2.amazonaws.com/unknown", Scope: "", Forced: false, Data: nil},
   182  		{Name: "Datastore/operation/DynamoDB/DescribeTable", Scope: "", Forced: false, Data: nil},
   183  		{Name: "Datastore/statement/DynamoDB/thebesttable/DescribeTable", Scope: "", Forced: false, Data: nil},
   184  		{Name: "Datastore/statement/DynamoDB/thebesttable/DescribeTable", Scope: "OtherTransaction/Go/" + txnName, Forced: false, Data: nil},
   185  	}...)
   186  )
   187  
   188  func TestInstrumentRequestExternal(t *testing.T) {
   189  	app := testApp()
   190  	txn := app.StartTransaction(txnName, nil, nil)
   191  
   192  	client := lambda.New(newConfig(false))
   193  	input := &lambda.InvokeInput{
   194  		ClientContext:  aws.String("MyApp"),
   195  		FunctionName:   aws.String("non-existent-function"),
   196  		InvocationType: lambda.InvocationTypeEvent,
   197  		LogType:        lambda.LogTypeTail,
   198  		Payload:        []byte("{}"),
   199  	}
   200  	req := client.InvokeRequest(input)
   201  	InstrumentHandlers(&req.Handlers)
   202  	ctx := newrelic.NewContext(req.Context(), txn)
   203  
   204  	_, err := req.Send(ctx)
   205  	if nil != err {
   206  		t.Error(err)
   207  	}
   208  
   209  	txn.End()
   210  
   211  	app.ExpectMetrics(t, externalMetrics)
   212  	app.ExpectSpanEvents(t, []internal.WantEvent{
   213  		genericSpan, externalSpan})
   214  }
   215  
   216  func TestInstrumentRequestDatastore(t *testing.T) {
   217  	app := testApp()
   218  	txn := app.StartTransaction(txnName, nil, nil)
   219  
   220  	client := dynamodb.New(newConfig(false))
   221  	input := &dynamodb.DescribeTableInput{
   222  		TableName: aws.String("thebesttable"),
   223  	}
   224  
   225  	req := client.DescribeTableRequest(input)
   226  	InstrumentHandlers(&req.Handlers)
   227  	ctx := newrelic.NewContext(req.Context(), txn)
   228  
   229  	_, err := req.Send(ctx)
   230  	if nil != err {
   231  		t.Error(err)
   232  	}
   233  
   234  	txn.End()
   235  
   236  	app.ExpectMetrics(t, datastoreMetrics)
   237  	app.ExpectSpanEvents(t, []internal.WantEvent{
   238  		genericSpan, datastoreSpan})
   239  }
   240  
   241  func TestInstrumentRequestExternalNoTxn(t *testing.T) {
   242  	client := lambda.New(newConfig(false))
   243  	input := &lambda.InvokeInput{
   244  		ClientContext:  aws.String("MyApp"),
   245  		FunctionName:   aws.String("non-existent-function"),
   246  		InvocationType: lambda.InvocationTypeEvent,
   247  		LogType:        lambda.LogTypeTail,
   248  		Payload:        []byte("{}"),
   249  	}
   250  
   251  	req := client.InvokeRequest(input)
   252  	InstrumentHandlers(&req.Handlers)
   253  	ctx := req.Context()
   254  
   255  	_, err := req.Send(ctx)
   256  	if nil != err {
   257  		t.Error(err)
   258  	}
   259  }
   260  
   261  func TestInstrumentRequestDatastoreNoTxn(t *testing.T) {
   262  	client := dynamodb.New(newConfig(false))
   263  	input := &dynamodb.DescribeTableInput{
   264  		TableName: aws.String("thebesttable"),
   265  	}
   266  
   267  	req := client.DescribeTableRequest(input)
   268  	InstrumentHandlers(&req.Handlers)
   269  	ctx := req.Context()
   270  
   271  	_, err := req.Send(ctx)
   272  	if nil != err {
   273  		t.Error(err)
   274  	}
   275  }
   276  
   277  func TestInstrumentConfigExternal(t *testing.T) {
   278  	app := testApp()
   279  	txn := app.StartTransaction(txnName, nil, nil)
   280  
   281  	client := lambda.New(newConfig(true))
   282  
   283  	input := &lambda.InvokeInput{
   284  		ClientContext:  aws.String("MyApp"),
   285  		FunctionName:   aws.String("non-existent-function"),
   286  		InvocationType: lambda.InvocationTypeEvent,
   287  		LogType:        lambda.LogTypeTail,
   288  		Payload:        []byte("{}"),
   289  	}
   290  
   291  	req := client.InvokeRequest(input)
   292  	ctx := newrelic.NewContext(req.Context(), txn)
   293  
   294  	_, err := req.Send(ctx)
   295  	if nil != err {
   296  		t.Error(err)
   297  	}
   298  
   299  	txn.End()
   300  
   301  	app.ExpectMetrics(t, externalMetrics)
   302  	app.ExpectSpanEvents(t, []internal.WantEvent{
   303  		genericSpan, externalSpan})
   304  }
   305  
   306  func TestInstrumentConfigDatastore(t *testing.T) {
   307  	app := testApp()
   308  	txn := app.StartTransaction(txnName, nil, nil)
   309  
   310  	client := dynamodb.New(newConfig(true))
   311  
   312  	input := &dynamodb.DescribeTableInput{
   313  		TableName: aws.String("thebesttable"),
   314  	}
   315  
   316  	req := client.DescribeTableRequest(input)
   317  	ctx := newrelic.NewContext(req.Context(), txn)
   318  
   319  	_, err := req.Send(ctx)
   320  	if nil != err {
   321  		t.Error(err)
   322  	}
   323  
   324  	txn.End()
   325  
   326  	app.ExpectMetrics(t, datastoreMetrics)
   327  	app.ExpectSpanEvents(t, []internal.WantEvent{
   328  		genericSpan, datastoreSpan})
   329  }
   330  
   331  func TestInstrumentConfigExternalNoTxn(t *testing.T) {
   332  	client := lambda.New(newConfig(true))
   333  
   334  	input := &lambda.InvokeInput{
   335  		ClientContext:  aws.String("MyApp"),
   336  		FunctionName:   aws.String("non-existent-function"),
   337  		InvocationType: lambda.InvocationTypeEvent,
   338  		LogType:        lambda.LogTypeTail,
   339  		Payload:        []byte("{}"),
   340  	}
   341  
   342  	req := client.InvokeRequest(input)
   343  	ctx := req.Context()
   344  
   345  	_, err := req.Send(ctx)
   346  	if nil != err {
   347  		t.Error(err)
   348  	}
   349  }
   350  
   351  func TestInstrumentConfigDatastoreNoTxn(t *testing.T) {
   352  	client := dynamodb.New(newConfig(true))
   353  
   354  	input := &dynamodb.DescribeTableInput{
   355  		TableName: aws.String("thebesttable"),
   356  	}
   357  
   358  	req := client.DescribeTableRequest(input)
   359  	ctx := req.Context()
   360  
   361  	_, err := req.Send(ctx)
   362  	if nil != err {
   363  		t.Error(err)
   364  	}
   365  }
   366  
   367  func TestInstrumentConfigExternalTxnNotInCtx(t *testing.T) {
   368  	app := testApp()
   369  	txn := app.StartTransaction(txnName, nil, nil)
   370  
   371  	client := lambda.New(newConfig(true))
   372  
   373  	input := &lambda.InvokeInput{
   374  		ClientContext:  aws.String("MyApp"),
   375  		FunctionName:   aws.String("non-existent-function"),
   376  		InvocationType: lambda.InvocationTypeEvent,
   377  		LogType:        lambda.LogTypeTail,
   378  		Payload:        []byte("{}"),
   379  	}
   380  
   381  	req := client.InvokeRequest(input)
   382  	ctx := req.Context()
   383  
   384  	_, err := req.Send(ctx)
   385  	if nil != err {
   386  		t.Error(err)
   387  	}
   388  
   389  	txn.End()
   390  
   391  	app.ExpectMetrics(t, txnMetrics)
   392  }
   393  
   394  func TestInstrumentConfigDatastoreTxnNotInCtx(t *testing.T) {
   395  	app := testApp()
   396  	txn := app.StartTransaction(txnName, nil, nil)
   397  
   398  	client := dynamodb.New(newConfig(true))
   399  
   400  	input := &dynamodb.DescribeTableInput{
   401  		TableName: aws.String("thebesttable"),
   402  	}
   403  
   404  	req := client.DescribeTableRequest(input)
   405  	ctx := req.Context()
   406  
   407  	_, err := req.Send(ctx)
   408  	if nil != err {
   409  		t.Error(err)
   410  	}
   411  
   412  	txn.End()
   413  
   414  	app.ExpectMetrics(t, txnMetrics)
   415  }
   416  
   417  func TestDoublyInstrumented(t *testing.T) {
   418  	hs := &aws.Handlers{}
   419  	if found := hs.Send.Len(); 0 != found {
   420  		t.Error("unexpected number of Send handlers found:", found)
   421  	}
   422  
   423  	InstrumentHandlers(hs)
   424  	if found := hs.Send.Len(); 2 != found {
   425  		t.Error("unexpected number of Send handlers found:", found)
   426  	}
   427  
   428  	InstrumentHandlers(hs)
   429  	if found := hs.Send.Len(); 2 != found {
   430  		t.Error("unexpected number of Send handlers found:", found)
   431  	}
   432  }
   433  
   434  type firstFailingTransport struct {
   435  	failing bool
   436  }
   437  
   438  func (t *firstFailingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
   439  	if t.failing {
   440  		t.failing = false
   441  		return nil, errors.New("Oops this failed")
   442  	}
   443  	return &http.Response{
   444  		Status:     "200 OK",
   445  		StatusCode: 200,
   446  		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
   447  		Header: http.Header{
   448  			"X-Amzn-Requestid": []string{requestID},
   449  		},
   450  	}, nil
   451  }
   452  
   453  func TestRetrySend(t *testing.T) {
   454  	app := testApp()
   455  	txn := app.StartTransaction(txnName, nil, nil)
   456  
   457  	cfg := newConfig(false)
   458  	cfg.HTTPClient = &http.Client{
   459  		Transport: &firstFailingTransport{failing: true},
   460  	}
   461  
   462  	client := lambda.New(cfg)
   463  	input := &lambda.InvokeInput{
   464  		ClientContext:  aws.String("MyApp"),
   465  		FunctionName:   aws.String("non-existent-function"),
   466  		InvocationType: lambda.InvocationTypeEvent,
   467  		LogType:        lambda.LogTypeTail,
   468  		Payload:        []byte("{}"),
   469  	}
   470  	req := client.InvokeRequest(input)
   471  	InstrumentHandlers(&req.Handlers)
   472  	ctx := newrelic.NewContext(req.Context(), txn)
   473  
   474  	_, err := req.Send(ctx)
   475  	if nil != err {
   476  		t.Error(err)
   477  	}
   478  
   479  	txn.End()
   480  
   481  	app.ExpectMetrics(t, []internal.WantMetric{
   482  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/all", Scope: "", Forced: false, Data: nil},
   483  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: nil},
   484  		{Name: "External/all", Scope: "", Forced: true, Data: []float64{2}},
   485  		{Name: "External/allOther", Scope: "", Forced: true, Data: []float64{2}},
   486  		{Name: "External/lambda.us-west-2.amazonaws.com/all", Scope: "", Forced: false, Data: []float64{2}},
   487  		{Name: "External/lambda.us-west-2.amazonaws.com/http/POST", Scope: "OtherTransaction/Go/" + txnName, Forced: false, Data: []float64{2}},
   488  		{Name: "OtherTransaction/Go/" + txnName, Scope: "", Forced: true, Data: nil},
   489  		{Name: "OtherTransaction/all", Scope: "", Forced: true, Data: nil},
   490  		{Name: "OtherTransactionTotalTime/Go/" + txnName, Scope: "", Forced: false, Data: nil},
   491  		{Name: "OtherTransactionTotalTime", Scope: "", Forced: true, Data: nil},
   492  	})
   493  	app.ExpectSpanEvents(t, []internal.WantEvent{
   494  		genericSpan, externalSpanNoRequestID, externalSpan})
   495  }
   496  
   497  func TestRequestSentTwice(t *testing.T) {
   498  	app := testApp()
   499  	txn := app.StartTransaction(txnName, nil, nil)
   500  
   501  	client := lambda.New(newConfig(false))
   502  	input := &lambda.InvokeInput{
   503  		ClientContext:  aws.String("MyApp"),
   504  		FunctionName:   aws.String("non-existent-function"),
   505  		InvocationType: lambda.InvocationTypeEvent,
   506  		LogType:        lambda.LogTypeTail,
   507  		Payload:        []byte("{}"),
   508  	}
   509  	req := client.InvokeRequest(input)
   510  	InstrumentHandlers(&req.Handlers)
   511  	ctx := newrelic.NewContext(req.Context(), txn)
   512  
   513  	_, firstErr := req.Send(ctx)
   514  	if nil != firstErr {
   515  		t.Error(firstErr)
   516  	}
   517  
   518  	_, secondErr := req.Send(ctx)
   519  	if nil != secondErr {
   520  		t.Error(secondErr)
   521  	}
   522  
   523  	txn.End()
   524  
   525  	app.ExpectMetrics(t, []internal.WantMetric{
   526  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/all", Scope: "", Forced: false, Data: nil},
   527  		{Name: "DurationByCaller/Unknown/Unknown/Unknown/Unknown/allOther", Scope: "", Forced: false, Data: nil},
   528  		{Name: "External/all", Scope: "", Forced: true, Data: []float64{2}},
   529  		{Name: "External/allOther", Scope: "", Forced: true, Data: []float64{2}},
   530  		{Name: "External/lambda.us-west-2.amazonaws.com/all", Scope: "", Forced: false, Data: []float64{2}},
   531  		{Name: "External/lambda.us-west-2.amazonaws.com/http/POST", Scope: "OtherTransaction/Go/" + txnName, Forced: false, Data: []float64{2}},
   532  		{Name: "OtherTransaction/Go/" + txnName, Scope: "", Forced: true, Data: nil},
   533  		{Name: "OtherTransaction/all", Scope: "", Forced: true, Data: nil},
   534  		{Name: "OtherTransactionTotalTime/Go/" + txnName, Scope: "", Forced: false, Data: nil},
   535  		{Name: "OtherTransactionTotalTime", Scope: "", Forced: true, Data: nil},
   536  	})
   537  	app.ExpectSpanEvents(t, []internal.WantEvent{
   538  		genericSpan, externalSpan, externalSpan})
   539  }
   540  
   541  type noRequestIDTransport struct{}
   542  
   543  func (t *noRequestIDTransport) RoundTrip(r *http.Request) (*http.Response, error) {
   544  	return &http.Response{
   545  		Status:     "200 OK",
   546  		StatusCode: 200,
   547  		Body:       ioutil.NopCloser(bytes.NewReader([]byte(""))),
   548  	}, nil
   549  }
   550  
   551  func TestNoRequestIDFound(t *testing.T) {
   552  	app := testApp()
   553  	txn := app.StartTransaction(txnName, nil, nil)
   554  
   555  	cfg := newConfig(false)
   556  	cfg.HTTPClient = &http.Client{
   557  		Transport: &noRequestIDTransport{},
   558  	}
   559  
   560  	client := lambda.New(cfg)
   561  	input := &lambda.InvokeInput{
   562  		ClientContext:  aws.String("MyApp"),
   563  		FunctionName:   aws.String("non-existent-function"),
   564  		InvocationType: lambda.InvocationTypeEvent,
   565  		LogType:        lambda.LogTypeTail,
   566  		Payload:        []byte("{}"),
   567  	}
   568  	req := client.InvokeRequest(input)
   569  	InstrumentHandlers(&req.Handlers)
   570  	ctx := newrelic.NewContext(req.Context(), txn)
   571  
   572  	_, err := req.Send(ctx)
   573  	if nil != err {
   574  		t.Error(err)
   575  	}
   576  
   577  	txn.End()
   578  
   579  	app.ExpectMetrics(t, externalMetrics)
   580  	app.ExpectSpanEvents(t, []internal.WantEvent{
   581  		genericSpan, externalSpanNoRequestID})
   582  }