github.com/mweagle/Sparta@v1.15.0/docs/reference/operations/profiling/index.html (about)

     1  <!DOCTYPE html>
     2  <html lang="en" class="js csstransforms3d">
     3    <head>
     4      <meta charset="utf-8">
     5      <meta name="viewport" content="width=device-width, initial-scale=1.0">
     6      <meta name="generator" content="Hugo 0.69.2" />
     7      <meta name="description" content="Sparta - AWS Lambda Microservices">
     8  <meta name="author" content="Matt Weagle">
     9  
    10      <link rel="icon" href="/images/favicon.png" type="image/png">
    11  
    12      <title>Profiling :: Sparta - AWS Lambda Microservices</title>
    13  
    14      
    15      <link href="/css/nucleus.css?1588464711" rel="stylesheet">
    16      <link href="/css/fontawesome-all.min.css?1588464711" rel="stylesheet">
    17      <link href="/css/hybrid.css?1588464711" rel="stylesheet">
    18      <link href="/css/featherlight.min.css?1588464711" rel="stylesheet">
    19      <link href="/css/perfect-scrollbar.min.css?1588464711" rel="stylesheet">
    20      <link href="/css/auto-complete.css?1588464711" rel="stylesheet">
    21      <link href="/css/theme.css?1588464711" rel="stylesheet">
    22      <link href="/css/hugo-theme.css?1588464711" rel="stylesheet">
    23      
    24        <link href="/css/theme-sparta.css?1588464711" rel="stylesheet">
    25      
    26  
    27      <script src="/js/jquery-3.3.1.min.js?1588464711"></script>
    28      
    29      <link href="/css/vs.css?1588464711" rel="stylesheet">
    30      <script src="/js/highlight.pack.js?1588464711"></script>
    31      <script>hljs.initHighlightingOnLoad();</script>
    32      
    33  
    34      <style>
    35        :root #header + #content > #left > #rlblock_left{
    36            display:none !important;
    37        }
    38        
    39      </style>
    40      
    41  <style type="text/css">
    42  pre {
    43      padding: 2px;
    44  }
    45  </style>
    46  
    47  
    48    </head>
    49    <body class="" data-url="/reference/operations/profiling/">
    50      <nav id="sidebar" class="">
    51  
    52  
    53  
    54    <div id="header-wrapper">
    55      <div id="header">
    56        <a href="/"><img src="/images/SpartaLogoNoDomain.png" height="50%" width="50%"/></a>
    57      </div>
    58      
    59          <div class="searchbox">
    60      <label for="search-by"><i class="fas fa-search"></i></label>
    61      <input data-search-input id="search-by" type="search" placeholder="Search...">
    62      <span data-search-clear=""><i class="fas fa-times"></i></span>
    63  </div>
    64  
    65  <script type="text/javascript" src="/js/lunr.min.js?1588464711"></script>
    66  <script type="text/javascript" src="/js/auto-complete.js?1588464711"></script>
    67  <script type="text/javascript">
    68      
    69          var baseurl = "\/";
    70      
    71  </script>
    72  <script type="text/javascript" src="/js/search.js?1588464711"></script>
    73  
    74      
    75    </div>
    76  
    77      <div class="highlightable">
    78      <ul class="topics">
    79  
    80          
    81            
    82            
    83  
    84  
    85   
    86    
    87      
    88      <li data-nav-id="/getting_started/" title="Getting Started" class="dd-item 
    89          
    90          
    91          
    92          ">
    93        <a href="/getting_started/">
    94            Getting Started
    95            
    96        </a>
    97        
    98                
    99      </li>
   100    
   101   
   102  
   103            
   104            
   105  
   106  
   107   
   108    
   109      
   110      <li data-nav-id="/example_service/" title="Example Service" class="dd-item 
   111          
   112          
   113          
   114          ">
   115        <a href="/example_service/">
   116            Example Service
   117            
   118        </a>
   119        
   120        
   121          <ul>
   122            
   123            
   124            
   125            
   126          
   127            
   128              
   129              
   130  
   131  
   132   
   133    
   134      
   135        <li data-nav-id="/example_service/step1/" title="Overview" class="dd-item ">
   136          <a href="/example_service/step1/">
   137          Overview
   138          
   139          </a>
   140      </li>
   141       
   142    
   143   
   144  
   145              
   146            
   147              
   148              
   149  
   150  
   151   
   152    
   153      
   154        <li data-nav-id="/example_service/step2/" title="Details" class="dd-item ">
   155          <a href="/example_service/step2/">
   156          Details
   157          
   158          </a>
   159      </li>
   160       
   161    
   162   
   163  
   164              
   165            
   166          
   167          </ul>
   168                
   169      </li>
   170    
   171   
   172  
   173            
   174            
   175  
   176  
   177   
   178    
   179      
   180      <li data-nav-id="/concepts/" title="Concepts" class="dd-item 
   181          
   182          
   183          
   184          ">
   185        <a href="/concepts/">
   186            Concepts
   187            
   188        </a>
   189        
   190                
   191      </li>
   192    
   193   
   194  
   195            
   196            
   197  
   198  
   199   
   200    
   201      
   202      <li data-nav-id="/reference/" title="Reference" class="dd-item 
   203          parent
   204          
   205          
   206          ">
   207        <a href="/reference/">
   208            Reference
   209            
   210        </a>
   211        
   212        
   213          <ul>
   214            
   215            
   216              
   217            
   218            
   219            
   220          
   221            
   222              
   223              
   224  
   225  
   226   
   227    
   228      
   229      <li data-nav-id="/reference/apigateway/" title="" class="dd-item 
   230          
   231          
   232          
   233          ">
   234        <a href="/reference/apigateway/">
   235            <b>API Gateway</b>
   236            
   237        </a>
   238        
   239        
   240          <ul>
   241            
   242            
   243            
   244            
   245          
   246            
   247              
   248              
   249  
   250  
   251   
   252    
   253      
   254        <li data-nav-id="/reference/apigateway/echo_event/" title="Echo" class="dd-item ">
   255          <a href="/reference/apigateway/echo_event/">
   256          Echo
   257          
   258          </a>
   259      </li>
   260       
   261    
   262   
   263  
   264              
   265            
   266              
   267              
   268  
   269  
   270   
   271    
   272      
   273        <li data-nav-id="/reference/apigateway/request_params/" title="Request Parameters" class="dd-item ">
   274          <a href="/reference/apigateway/request_params/">
   275          Request Parameters
   276          
   277          </a>
   278      </li>
   279       
   280    
   281   
   282  
   283              
   284            
   285              
   286              
   287  
   288  
   289   
   290    
   291      
   292        <li data-nav-id="/reference/apigateway/context/" title="Request Context" class="dd-item ">
   293          <a href="/reference/apigateway/context/">
   294          Request Context
   295          
   296          </a>
   297      </li>
   298       
   299    
   300   
   301  
   302              
   303            
   304              
   305              
   306  
   307  
   308   
   309    
   310      
   311        <li data-nav-id="/reference/apigateway/cors/" title="CORS" class="dd-item ">
   312          <a href="/reference/apigateway/cors/">
   313          CORS
   314          
   315          </a>
   316      </li>
   317       
   318    
   319   
   320  
   321              
   322            
   323              
   324              
   325  
   326  
   327   
   328    
   329      
   330        <li data-nav-id="/reference/apigateway/slack/" title="Slack SlashCommand" class="dd-item ">
   331          <a href="/reference/apigateway/slack/">
   332          Slack SlashCommand
   333          
   334          </a>
   335      </li>
   336       
   337    
   338   
   339  
   340              
   341            
   342              
   343              
   344  
   345  
   346   
   347    
   348      
   349        <li data-nav-id="/reference/apigateway/s3site/" title="S3 Sites with CORS" class="dd-item ">
   350          <a href="/reference/apigateway/s3site/">
   351          S3 Sites with CORS
   352          
   353          </a>
   354      </li>
   355       
   356    
   357   
   358  
   359              
   360            
   361          
   362          </ul>
   363                
   364      </li>
   365    
   366   
   367  
   368              
   369            
   370              
   371              
   372  
   373  
   374   
   375    
   376      
   377      <li data-nav-id="/reference/apiv2gateway/" title="" class="dd-item 
   378          
   379          
   380          
   381          ">
   382        <a href="/reference/apiv2gateway/">
   383            <b>API V2 Gateway</b>
   384            
   385        </a>
   386        
   387                
   388      </li>
   389    
   390   
   391  
   392              
   393            
   394              
   395              
   396  
   397  
   398   
   399    
   400      
   401      <li data-nav-id="/reference/eventsources/" title="" class="dd-item 
   402          
   403          
   404          
   405          ">
   406        <a href="/reference/eventsources/">
   407            <b>Event Sources</b>
   408            
   409        </a>
   410        
   411        
   412          <ul>
   413            
   414            
   415            
   416            
   417          
   418            
   419              
   420              
   421  
   422  
   423   
   424    
   425      
   426        <li data-nav-id="/reference/eventsources/codecommit/" title="CodeCommit" class="dd-item ">
   427          <a href="/reference/eventsources/codecommit/">
   428          CodeCommit
   429          
   430          </a>
   431      </li>
   432       
   433    
   434   
   435  
   436              
   437            
   438              
   439              
   440  
   441  
   442   
   443    
   444      
   445        <li data-nav-id="/reference/eventsources/cloudformation/" title="CloudFormation" class="dd-item ">
   446          <a href="/reference/eventsources/cloudformation/">
   447          CloudFormation
   448          
   449          </a>
   450      </li>
   451       
   452    
   453   
   454  
   455              
   456            
   457              
   458              
   459  
   460  
   461   
   462    
   463      
   464        <li data-nav-id="/reference/eventsources/cloudwatchevents/" title="CloudWatch Events" class="dd-item ">
   465          <a href="/reference/eventsources/cloudwatchevents/">
   466          CloudWatch Events
   467          
   468          </a>
   469      </li>
   470       
   471    
   472   
   473  
   474              
   475            
   476              
   477              
   478  
   479  
   480   
   481    
   482      
   483        <li data-nav-id="/reference/eventsources/cloudwatchlogs/" title="CloudWatch Logs" class="dd-item ">
   484          <a href="/reference/eventsources/cloudwatchlogs/">
   485          CloudWatch Logs
   486          
   487          </a>
   488      </li>
   489       
   490    
   491   
   492  
   493              
   494            
   495              
   496              
   497  
   498  
   499   
   500    
   501      
   502        <li data-nav-id="/reference/eventsources/cognito/" title="Cognito" class="dd-item ">
   503          <a href="/reference/eventsources/cognito/">
   504          Cognito
   505          
   506          </a>
   507      </li>
   508       
   509    
   510   
   511  
   512              
   513            
   514              
   515              
   516  
   517  
   518   
   519    
   520      
   521        <li data-nav-id="/reference/eventsources/dynamodb/" title="DynamoDB" class="dd-item ">
   522          <a href="/reference/eventsources/dynamodb/">
   523          DynamoDB
   524          
   525          </a>
   526      </li>
   527       
   528    
   529   
   530  
   531              
   532            
   533              
   534              
   535  
   536  
   537   
   538    
   539      
   540        <li data-nav-id="/reference/eventsources/kinesis/" title="Kinesis" class="dd-item ">
   541          <a href="/reference/eventsources/kinesis/">
   542          Kinesis
   543          
   544          </a>
   545      </li>
   546       
   547    
   548   
   549  
   550              
   551            
   552              
   553              
   554  
   555  
   556   
   557    
   558      
   559        <li data-nav-id="/reference/eventsources/s3/" title="S3" class="dd-item ">
   560          <a href="/reference/eventsources/s3/">
   561          S3
   562          
   563          </a>
   564      </li>
   565       
   566    
   567   
   568  
   569              
   570            
   571              
   572              
   573  
   574  
   575   
   576    
   577      
   578        <li data-nav-id="/reference/eventsources/ses/" title="SES" class="dd-item ">
   579          <a href="/reference/eventsources/ses/">
   580          SES
   581          
   582          </a>
   583      </li>
   584       
   585    
   586   
   587  
   588              
   589            
   590              
   591              
   592  
   593  
   594   
   595    
   596      
   597        <li data-nav-id="/reference/eventsources/sns/" title="SNS" class="dd-item ">
   598          <a href="/reference/eventsources/sns/">
   599          SNS
   600          
   601          </a>
   602      </li>
   603       
   604    
   605   
   606  
   607              
   608            
   609              
   610              
   611  
   612  
   613   
   614    
   615      
   616        <li data-nav-id="/reference/eventsources/sqs/" title="SQS" class="dd-item ">
   617          <a href="/reference/eventsources/sqs/">
   618          SQS
   619          
   620          </a>
   621      </li>
   622       
   623    
   624   
   625  
   626              
   627            
   628          
   629          </ul>
   630                
   631      </li>
   632    
   633   
   634  
   635              
   636            
   637              
   638              
   639  
   640  
   641   
   642    
   643      
   644      <li data-nav-id="/reference/archetypes/" title="" class="dd-item 
   645          
   646          
   647          
   648          ">
   649        <a href="/reference/archetypes/">
   650            <b>Archetype Constructors</b>
   651            
   652        </a>
   653        
   654        
   655          <ul>
   656            
   657            
   658            
   659            
   660          
   661            
   662              
   663              
   664  
   665  
   666   
   667    
   668      
   669        <li data-nav-id="/reference/archetypes/event_bridge/" title="Event Bridge" class="dd-item ">
   670          <a href="/reference/archetypes/event_bridge/">
   671          Event Bridge
   672          
   673          </a>
   674      </li>
   675       
   676    
   677   
   678  
   679              
   680            
   681              
   682              
   683  
   684  
   685   
   686    
   687      
   688        <li data-nav-id="/reference/archetypes/codecommit/" title="CodeCommit" class="dd-item ">
   689          <a href="/reference/archetypes/codecommit/">
   690          CodeCommit
   691          
   692          </a>
   693      </li>
   694       
   695    
   696   
   697  
   698              
   699            
   700              
   701              
   702  
   703  
   704   
   705    
   706      
   707        <li data-nav-id="/reference/archetypes/cloudwatch/" title="CloudWatch" class="dd-item ">
   708          <a href="/reference/archetypes/cloudwatch/">
   709          CloudWatch
   710          
   711          </a>
   712      </li>
   713       
   714    
   715   
   716  
   717              
   718            
   719              
   720              
   721  
   722  
   723   
   724    
   725      
   726        <li data-nav-id="/reference/archetypes/dynamodb/" title="DynamoDB" class="dd-item ">
   727          <a href="/reference/archetypes/dynamodb/">
   728          DynamoDB
   729          
   730          </a>
   731      </li>
   732       
   733    
   734   
   735  
   736              
   737            
   738              
   739              
   740  
   741  
   742   
   743    
   744      
   745        <li data-nav-id="/reference/archetypes/kinesis/" title="Kinesis" class="dd-item ">
   746          <a href="/reference/archetypes/kinesis/">
   747          Kinesis
   748          
   749          </a>
   750      </li>
   751       
   752    
   753   
   754  
   755              
   756            
   757              
   758              
   759  
   760  
   761   
   762    
   763      
   764        <li data-nav-id="/reference/archetypes/kinesis_firehose/" title="Kinesis Firehose" class="dd-item ">
   765          <a href="/reference/archetypes/kinesis_firehose/">
   766          Kinesis Firehose
   767          
   768          </a>
   769      </li>
   770       
   771    
   772   
   773  
   774              
   775            
   776              
   777              
   778  
   779  
   780   
   781    
   782      
   783        <li data-nav-id="/reference/archetypes/rest/" title="REST Service" class="dd-item ">
   784          <a href="/reference/archetypes/rest/">
   785          REST Service
   786          
   787          </a>
   788      </li>
   789       
   790    
   791   
   792  
   793              
   794            
   795              
   796              
   797  
   798  
   799   
   800    
   801      
   802        <li data-nav-id="/reference/archetypes/s3/" title="S3" class="dd-item ">
   803          <a href="/reference/archetypes/s3/">
   804          S3
   805          
   806          </a>
   807      </li>
   808       
   809    
   810   
   811  
   812              
   813            
   814              
   815              
   816  
   817  
   818   
   819    
   820      
   821        <li data-nav-id="/reference/archetypes/sns/" title="SNS" class="dd-item ">
   822          <a href="/reference/archetypes/sns/">
   823          SNS
   824          
   825          </a>
   826      </li>
   827       
   828    
   829   
   830  
   831              
   832            
   833          
   834          </ul>
   835                
   836      </li>
   837    
   838   
   839  
   840              
   841            
   842              
   843              
   844  
   845  
   846   
   847    
   848      
   849      <li data-nav-id="/reference/decorators/" title="" class="dd-item 
   850          
   851          
   852          
   853          ">
   854        <a href="/reference/decorators/">
   855            <b>Build-Time Decorators</b>
   856            
   857        </a>
   858        
   859        
   860          <ul>
   861            
   862            
   863            
   864            
   865          
   866            
   867              
   868              
   869  
   870  
   871   
   872    
   873      
   874        <li data-nav-id="/reference/decorators/application_load_balancer/" title="Application Load Balancer" class="dd-item ">
   875          <a href="/reference/decorators/application_load_balancer/">
   876          Application Load Balancer
   877          
   878          </a>
   879      </li>
   880       
   881    
   882   
   883  
   884              
   885            
   886              
   887              
   888  
   889  
   890   
   891    
   892      
   893        <li data-nav-id="/reference/decorators/cloudmap/" title="CloudMap Service Discovery" class="dd-item ">
   894          <a href="/reference/decorators/cloudmap/">
   895          CloudMap Service Discovery
   896          
   897          </a>
   898      </li>
   899       
   900    
   901   
   902  
   903              
   904            
   905              
   906              
   907  
   908  
   909   
   910    
   911      
   912        <li data-nav-id="/reference/decorators/cloudfront_distribution/" title="CloudFront Distribution" class="dd-item ">
   913          <a href="/reference/decorators/cloudfront_distribution/">
   914          CloudFront Distribution
   915          
   916          </a>
   917      </li>
   918       
   919    
   920   
   921  
   922              
   923            
   924              
   925              
   926  
   927  
   928   
   929    
   930      
   931        <li data-nav-id="/reference/decorators/lambda_versioning/" title="Lambda Versioning Decorator" class="dd-item ">
   932          <a href="/reference/decorators/lambda_versioning/">
   933          Lambda Versioning Decorator
   934          
   935          </a>
   936      </li>
   937       
   938    
   939   
   940  
   941              
   942            
   943              
   944              
   945  
   946  
   947   
   948    
   949      
   950        <li data-nav-id="/reference/decorators/publish_outputs/" title="Publishing Outputs" class="dd-item ">
   951          <a href="/reference/decorators/publish_outputs/">
   952          Publishing Outputs
   953          
   954          </a>
   955      </li>
   956       
   957    
   958   
   959  
   960              
   961            
   962              
   963              
   964  
   965  
   966   
   967    
   968      
   969        <li data-nav-id="/reference/decorators/s3_artifact_publisher/" title="S3 Artifact Publisher" class="dd-item ">
   970          <a href="/reference/decorators/s3_artifact_publisher/">
   971          S3 Artifact Publisher
   972          
   973          </a>
   974      </li>
   975       
   976    
   977   
   978  
   979              
   980            
   981              
   982              
   983  
   984  
   985   
   986    
   987      
   988        <li data-nav-id="/reference/decorators/dynamic_infrastructure/" title="Dynamic Infrastructure" class="dd-item ">
   989          <a href="/reference/decorators/dynamic_infrastructure/">
   990          Dynamic Infrastructure
   991          
   992          </a>
   993      </li>
   994       
   995    
   996   
   997  
   998              
   999            
  1000          
  1001          </ul>
  1002                
  1003      </li>
  1004    
  1005   
  1006  
  1007              
  1008            
  1009              
  1010              
  1011  
  1012  
  1013   
  1014    
  1015      
  1016      <li data-nav-id="/reference/interceptors/" title="" class="dd-item 
  1017          
  1018          
  1019          
  1020          ">
  1021        <a href="/reference/interceptors/">
  1022            <b>Runtime Interceptors</b>
  1023            
  1024        </a>
  1025        
  1026        
  1027          <ul>
  1028            
  1029            
  1030            
  1031            
  1032          
  1033            
  1034              
  1035              
  1036  
  1037  
  1038   
  1039    
  1040      
  1041        <li data-nav-id="/reference/interceptors/xray_interceptor/" title="XRayInterceptor" class="dd-item ">
  1042          <a href="/reference/interceptors/xray_interceptor/">
  1043          XRayInterceptor
  1044          
  1045          </a>
  1046      </li>
  1047       
  1048    
  1049   
  1050  
  1051              
  1052            
  1053          
  1054          </ul>
  1055                
  1056      </li>
  1057    
  1058   
  1059  
  1060              
  1061            
  1062              
  1063              
  1064  
  1065  
  1066   
  1067    
  1068      
  1069      <li data-nav-id="/reference/cli_options/" title="" class="dd-item 
  1070          
  1071          
  1072          
  1073          ">
  1074        <a href="/reference/cli_options/">
  1075            <b>Command Line Options</b>
  1076            
  1077        </a>
  1078        
  1079                
  1080      </li>
  1081    
  1082   
  1083  
  1084              
  1085            
  1086              
  1087              
  1088  
  1089  
  1090   
  1091    
  1092      
  1093      <li data-nav-id="/reference/application/" title="" class="dd-item 
  1094          
  1095          
  1096          
  1097          ">
  1098        <a href="/reference/application/">
  1099            <b>Application Customization</b>
  1100            
  1101        </a>
  1102        
  1103        
  1104          <ul>
  1105            
  1106            
  1107            
  1108            
  1109          
  1110            
  1111              
  1112              
  1113  
  1114  
  1115   
  1116    
  1117      
  1118        <li data-nav-id="/reference/application/custom_commands/" title="Custom Commands" class="dd-item ">
  1119          <a href="/reference/application/custom_commands/">
  1120          Custom Commands
  1121          
  1122          </a>
  1123      </li>
  1124       
  1125    
  1126   
  1127  
  1128              
  1129            
  1130              
  1131              
  1132  
  1133  
  1134   
  1135    
  1136      
  1137        <li data-nav-id="/reference/application/custom_flags/" title="Custom Flags" class="dd-item ">
  1138          <a href="/reference/application/custom_flags/">
  1139          Custom Flags
  1140          
  1141          </a>
  1142      </li>
  1143       
  1144    
  1145   
  1146  
  1147              
  1148            
  1149              
  1150              
  1151  
  1152  
  1153   
  1154    
  1155      
  1156        <li data-nav-id="/reference/application/environments/" title="Managing Environments" class="dd-item ">
  1157          <a href="/reference/application/environments/">
  1158          Managing Environments
  1159          
  1160          </a>
  1161      </li>
  1162       
  1163    
  1164   
  1165  
  1166              
  1167            
  1168              
  1169              
  1170  
  1171  
  1172   
  1173    
  1174      
  1175        <li data-nav-id="/reference/application/custom_lambda_resources/" title="CloudFormation Resources" class="dd-item ">
  1176          <a href="/reference/application/custom_lambda_resources/">
  1177          CloudFormation Resources
  1178          
  1179          </a>
  1180      </li>
  1181       
  1182    
  1183   
  1184  
  1185              
  1186            
  1187              
  1188              
  1189  
  1190  
  1191   
  1192    
  1193      
  1194        <li data-nav-id="/reference/application/custom_resources/" title="Custom Resources" class="dd-item ">
  1195          <a href="/reference/application/custom_resources/">
  1196          Custom Resources
  1197          
  1198          </a>
  1199      </li>
  1200       
  1201    
  1202   
  1203  
  1204              
  1205            
  1206          
  1207          </ul>
  1208                
  1209      </li>
  1210    
  1211   
  1212  
  1213              
  1214            
  1215              
  1216              
  1217  
  1218  
  1219   
  1220    
  1221      
  1222      <li data-nav-id="/reference/step/" title="" class="dd-item 
  1223          
  1224          
  1225          
  1226          ">
  1227        <a href="/reference/step/">
  1228            <b>Step Functions</b>
  1229            
  1230        </a>
  1231        
  1232        
  1233          <ul>
  1234            
  1235            
  1236              
  1237            
  1238            
  1239            
  1240          
  1241            
  1242              
  1243              
  1244  
  1245  
  1246   
  1247    
  1248      
  1249        <li data-nav-id="/reference/step/lambda/" title="Lambda" class="dd-item ">
  1250          <a href="/reference/step/lambda/">
  1251          Lambda
  1252          
  1253          </a>
  1254      </li>
  1255       
  1256    
  1257   
  1258  
  1259              
  1260            
  1261              
  1262              
  1263  
  1264  
  1265   
  1266    
  1267      
  1268        <li data-nav-id="/reference/step/fargate/" title="Fargate" class="dd-item ">
  1269          <a href="/reference/step/fargate/">
  1270          Fargate
  1271          
  1272          </a>
  1273      </li>
  1274       
  1275    
  1276   
  1277  
  1278              
  1279            
  1280              
  1281              
  1282  
  1283  
  1284   
  1285    
  1286      
  1287      <li data-nav-id="/reference/step/services/" title="" class="dd-item 
  1288          
  1289          
  1290          
  1291          ">
  1292        <a href="/reference/step/services/">
  1293            <b>Service Integrations</b>
  1294            
  1295        </a>
  1296        
  1297        
  1298          <ul>
  1299            
  1300            
  1301            
  1302            
  1303          
  1304            
  1305              
  1306              
  1307  
  1308  
  1309   
  1310    
  1311      
  1312        <li data-nav-id="/reference/step/services/dynamodb/" title="Amazon DynamoDb" class="dd-item ">
  1313          <a href="/reference/step/services/dynamodb/">
  1314          Amazon DynamoDb
  1315          
  1316          </a>
  1317      </li>
  1318       
  1319    
  1320   
  1321  
  1322              
  1323            
  1324              
  1325              
  1326  
  1327  
  1328   
  1329    
  1330      
  1331        <li data-nav-id="/reference/step/services/sagemaker/" title="Amazon SageMaker" class="dd-item ">
  1332          <a href="/reference/step/services/sagemaker/">
  1333          Amazon SageMaker
  1334          
  1335          </a>
  1336      </li>
  1337       
  1338    
  1339   
  1340  
  1341              
  1342            
  1343              
  1344              
  1345  
  1346  
  1347   
  1348    
  1349      
  1350        <li data-nav-id="/reference/step/services/sns/" title="Amazon SNS" class="dd-item ">
  1351          <a href="/reference/step/services/sns/">
  1352          Amazon SNS
  1353          
  1354          </a>
  1355      </li>
  1356       
  1357    
  1358   
  1359  
  1360              
  1361            
  1362              
  1363              
  1364  
  1365  
  1366   
  1367    
  1368      
  1369        <li data-nav-id="/reference/step/services/sqs/" title="Amazon SQS" class="dd-item ">
  1370          <a href="/reference/step/services/sqs/">
  1371          Amazon SQS
  1372          
  1373          </a>
  1374      </li>
  1375       
  1376    
  1377   
  1378  
  1379              
  1380            
  1381              
  1382              
  1383  
  1384  
  1385   
  1386    
  1387      
  1388        <li data-nav-id="/reference/step/services/batch/" title="AWS Batch" class="dd-item ">
  1389          <a href="/reference/step/services/batch/">
  1390          AWS Batch
  1391          
  1392          </a>
  1393      </li>
  1394       
  1395    
  1396   
  1397  
  1398              
  1399            
  1400              
  1401              
  1402  
  1403  
  1404   
  1405    
  1406      
  1407        <li data-nav-id="/reference/step/services/fargate/" title="AWS Fargate" class="dd-item ">
  1408          <a href="/reference/step/services/fargate/">
  1409          AWS Fargate
  1410          
  1411          </a>
  1412      </li>
  1413       
  1414    
  1415   
  1416  
  1417              
  1418            
  1419              
  1420              
  1421  
  1422  
  1423   
  1424    
  1425      
  1426        <li data-nav-id="/reference/step/services/glue/" title="AWS Glue" class="dd-item ">
  1427          <a href="/reference/step/services/glue/">
  1428          AWS Glue
  1429          
  1430          </a>
  1431      </li>
  1432       
  1433    
  1434   
  1435  
  1436              
  1437            
  1438          
  1439          </ul>
  1440                
  1441      </li>
  1442    
  1443   
  1444  
  1445              
  1446            
  1447          
  1448          </ul>
  1449                
  1450      </li>
  1451    
  1452   
  1453  
  1454              
  1455            
  1456              
  1457              
  1458  
  1459  
  1460   
  1461    
  1462      
  1463        <li data-nav-id="/reference/discovery/" title="Discovery Service" class="dd-item ">
  1464          <a href="/reference/discovery/">
  1465          Discovery Service
  1466          
  1467          </a>
  1468      </li>
  1469       
  1470    
  1471   
  1472  
  1473              
  1474            
  1475              
  1476              
  1477  
  1478  
  1479   
  1480    
  1481      
  1482        <li data-nav-id="/reference/docker/" title="Docker" class="dd-item ">
  1483          <a href="/reference/docker/">
  1484          Docker
  1485          
  1486          </a>
  1487      </li>
  1488       
  1489    
  1490   
  1491  
  1492              
  1493            
  1494              
  1495              
  1496  
  1497  
  1498   
  1499    
  1500      
  1501        <li data-nav-id="/reference/hybrid_topologies/" title="Hybrid Topologies" class="dd-item ">
  1502          <a href="/reference/hybrid_topologies/">
  1503          Hybrid Topologies
  1504          
  1505          </a>
  1506      </li>
  1507       
  1508    
  1509   
  1510  
  1511              
  1512            
  1513              
  1514              
  1515  
  1516  
  1517   
  1518    
  1519      
  1520      <li data-nav-id="/reference/operations/" title="" class="dd-item 
  1521          parent
  1522          
  1523          
  1524          ">
  1525        <a href="/reference/operations/">
  1526            <b>Operations</b>
  1527            
  1528        </a>
  1529        
  1530        
  1531          <ul>
  1532            
  1533            
  1534            
  1535            
  1536          
  1537            
  1538              
  1539              
  1540  
  1541  
  1542   
  1543    
  1544      
  1545        <li data-nav-id="/reference/operations/magefile/" title="Magefiles" class="dd-item ">
  1546          <a href="/reference/operations/magefile/">
  1547          Magefiles
  1548          
  1549          </a>
  1550      </li>
  1551       
  1552    
  1553   
  1554  
  1555              
  1556            
  1557              
  1558              
  1559  
  1560  
  1561   
  1562    
  1563      
  1564        <li data-nav-id="/reference/operations/cloudwatch_alarms/" title="CloudWatch Alarms" class="dd-item ">
  1565          <a href="/reference/operations/cloudwatch_alarms/">
  1566          CloudWatch Alarms
  1567          
  1568          </a>
  1569      </li>
  1570       
  1571    
  1572   
  1573  
  1574              
  1575            
  1576              
  1577              
  1578  
  1579  
  1580   
  1581    
  1582      
  1583        <li data-nav-id="/reference/operations/cloudwatch_dashboard/" title="CloudWatch Dashboard" class="dd-item ">
  1584          <a href="/reference/operations/cloudwatch_dashboard/">
  1585          CloudWatch Dashboard
  1586          
  1587          </a>
  1588      </li>
  1589       
  1590    
  1591   
  1592  
  1593              
  1594            
  1595              
  1596              
  1597  
  1598  
  1599   
  1600    
  1601      
  1602        <li data-nav-id="/reference/operations/codedeploy_service_update/" title="CodeDeploy Service Update" class="dd-item ">
  1603          <a href="/reference/operations/codedeploy_service_update/">
  1604          CodeDeploy Service Update
  1605          
  1606          </a>
  1607      </li>
  1608       
  1609    
  1610   
  1611  
  1612              
  1613            
  1614              
  1615              
  1616  
  1617  
  1618   
  1619    
  1620      
  1621        <li data-nav-id="/reference/operations/cicd/" title="CI/CD" class="dd-item ">
  1622          <a href="/reference/operations/cicd/">
  1623          CI/CD
  1624          
  1625          </a>
  1626      </li>
  1627       
  1628    
  1629   
  1630  
  1631              
  1632            
  1633              
  1634              
  1635  
  1636  
  1637   
  1638    
  1639      
  1640        <li data-nav-id="/reference/operations/deployment_strategies/" title="Deployment Strategies" class="dd-item ">
  1641          <a href="/reference/operations/deployment_strategies/">
  1642          Deployment Strategies
  1643          
  1644          </a>
  1645      </li>
  1646       
  1647    
  1648   
  1649  
  1650              
  1651            
  1652              
  1653              
  1654  
  1655  
  1656   
  1657    
  1658      
  1659        <li data-nav-id="/reference/operations/metrics_publisher/" title="Metrics Publisher" class="dd-item ">
  1660          <a href="/reference/operations/metrics_publisher/">
  1661          Metrics Publisher
  1662          
  1663          </a>
  1664      </li>
  1665       
  1666    
  1667   
  1668  
  1669              
  1670            
  1671              
  1672              
  1673  
  1674  
  1675   
  1676    
  1677      
  1678        <li data-nav-id="/reference/operations/profiling/" title="Profiling" class="dd-item active">
  1679          <a href="/reference/operations/profiling/">
  1680          Profiling
  1681          
  1682          </a>
  1683      </li>
  1684       
  1685    
  1686   
  1687  
  1688              
  1689            
  1690          
  1691          </ul>
  1692                
  1693      </li>
  1694    
  1695   
  1696  
  1697              
  1698            
  1699              
  1700              
  1701  
  1702  
  1703   
  1704    
  1705      
  1706        <li data-nav-id="/reference/testing/" title="Testing" class="dd-item ">
  1707          <a href="/reference/testing/">
  1708          Testing
  1709          
  1710          </a>
  1711      </li>
  1712       
  1713    
  1714   
  1715  
  1716              
  1717            
  1718              
  1719              
  1720  
  1721  
  1722   
  1723    
  1724      
  1725        <li data-nav-id="/reference/supporting_packages/" title="Supporting Packages" class="dd-item ">
  1726          <a href="/reference/supporting_packages/">
  1727          Supporting Packages
  1728          
  1729          </a>
  1730      </li>
  1731       
  1732    
  1733   
  1734  
  1735              
  1736            
  1737              
  1738              
  1739  
  1740  
  1741   
  1742    
  1743      
  1744        <li data-nav-id="/reference/limitations/" title="Limitations" class="dd-item ">
  1745          <a href="/reference/limitations/">
  1746          Limitations
  1747          
  1748          </a>
  1749      </li>
  1750       
  1751    
  1752   
  1753  
  1754              
  1755            
  1756              
  1757              
  1758  
  1759  
  1760   
  1761    
  1762      
  1763        <li data-nav-id="/reference/faq/" title="FAQ" class="dd-item ">
  1764          <a href="/reference/faq/">
  1765          FAQ
  1766          
  1767          </a>
  1768      </li>
  1769       
  1770    
  1771   
  1772  
  1773              
  1774            
  1775          
  1776          </ul>
  1777                
  1778      </li>
  1779    
  1780   
  1781  
  1782            
  1783            
  1784  
  1785  
  1786   
  1787    
  1788      
  1789      <li data-nav-id="/presentations/" title="Presentations" class="dd-item 
  1790          
  1791          
  1792          
  1793          ">
  1794        <a href="/presentations/">
  1795            Presentations
  1796            
  1797        </a>
  1798        
  1799                
  1800      </li>
  1801    
  1802   
  1803  
  1804            
  1805            
  1806  
  1807  
  1808   
  1809    
  1810      
  1811      <li data-nav-id="/meta/" title="Meta" class="dd-item 
  1812          
  1813          
  1814          
  1815          ">
  1816        <a href="/meta/">
  1817            Meta
  1818            
  1819        </a>
  1820        
  1821                
  1822      </li>
  1823    
  1824   
  1825  
  1826            
  1827            
  1828  
  1829  
  1830   
  1831    
  1832      
  1833      <li data-nav-id="/credits/" title="Credits" class="dd-item 
  1834          
  1835          
  1836          
  1837          ">
  1838        <a href="/credits/">
  1839            Credits
  1840            
  1841        </a>
  1842        
  1843                
  1844      </li>
  1845    
  1846   
  1847  
  1848            
  1849           
  1850      </ul>
  1851  
  1852      
  1853      
  1854        <section id="shortcuts">
  1855          <h3>More</h3>
  1856          <ul>
  1857            
  1858                <li> 
  1859                    <a class="padding" href="https://github.com/mweagle/Sparta"><i class='fab fa-fw fa-github'></i> Github repo</a>
  1860                </li>
  1861            
  1862                <li> 
  1863                    <a class="padding" href="https://github.com/mweagle?utf8=%E2%9C%93&amp;tab=repositories&amp;q=Sparta&amp;type=&amp;language="><i class='fas fa-fw fa-camera'></i> Showcases</a>
  1864                </li>
  1865            
  1866                <li> 
  1867                    <a class="padding" href="https://github.com/mweagle/Sparta/blob/master/CHANGES.md"><i class='fas fa-fw fa-signal'></i> Release History</a>
  1868                </li>
  1869            
  1870          </ul>
  1871        </section>
  1872      
  1873  
  1874      
  1875      <section id="footer">
  1876        
  1877    Build: <a target="_blank" href="https://github.com/mweagle/Sparta/commit/46463572">46463572</a><p/>
  1878    Date: 2020-05-02T17:11:51-07:00
  1879  
  1880  
  1881  <p>Built with <a href="https://github.com/matcornic/hugo-theme-learn"><i class="fas fa-heart"></i></a> from <a href="http://getgrav.org">Grav</a> and <a href="http://gohugo.io/">Hugo</a></p>
  1882  
  1883      </section>
  1884    </div>
  1885  </nav>
  1886  
  1887  
  1888  
  1889  
  1890  
  1891          <section id="body">
  1892          <div id="overlay"></div>
  1893          <div class="padding highlightable">
  1894                
  1895                <div>
  1896                  <div id="top-bar">
  1897                  
  1898                    
  1899                    
  1900                    
  1901                    <div id="top-github-link">
  1902                      <a class="github-link" title='Edit this page' href="https://github.com/mweagle/Sparta/tree/docs/content/reference/operations/profiling.md" target="blank">
  1903                        <i class="fas fa-code-branch"></i>
  1904                        <span id="top-github-link-text">Edit this page</span>
  1905                      </a>
  1906                    </div>
  1907                    
  1908                  
  1909                  
  1910                  <div id="breadcrumbs" itemscope="" itemtype="http://data-vocabulary.org/Breadcrumb">
  1911                      <span id="sidebar-toggle-span">
  1912                          <a href="#" id="sidebar-toggle" data-sidebar-toggle="">
  1913                            <i class="fas fa-bars"></i>
  1914                          </a>
  1915                      </span>
  1916                    
  1917                    <span id="toc-menu"><i class="fas fa-list-alt"></i></span>
  1918                    
  1919                    <span class="links">
  1920                   
  1921                   
  1922                      
  1923            
  1924            
  1925              
  1926              
  1927            
  1928            
  1929              
  1930              
  1931            
  1932            
  1933              
  1934              
  1935            
  1936            
  1937              <a href='/'></a> > <a href='/reference/'>Reference</a> > <a href='/reference/operations/'></a> > Profiling
  1938            
  1939           
  1940            
  1941           
  1942            
  1943           
  1944            
  1945          
  1946                   
  1947                    </span>
  1948                  </div>
  1949                  
  1950                      <div class="progress">
  1951      <div class="wrapper">
  1952  <nav id="TableOfContents">
  1953    <ul>
  1954      <li><a href="#enabling-profiling">Enabling Profiling</a>
  1955        <ul>
  1956          <li><a href="#profiling-implementation">Profiling Implementation</a></li>
  1957        </ul>
  1958      </li>
  1959      <li><a href="#deploying">Deploying</a></li>
  1960      <li><a href="#generating-load">Generating Load</a></li>
  1961      <li><a href="#visualizing-profiles">Visualizing Profiles</a></li>
  1962    </ul>
  1963  </nav>
  1964      </div>
  1965  </div>
  1966  
  1967                  
  1968                </div>
  1969              </div>
  1970              
  1971          <div id="head-tags">
  1972          
  1973          </div>
  1974          
  1975          <div id="body-inner">
  1976            
  1977              <h1>
  1978                
  1979                Profiling
  1980              </h1>
  1981            
  1982  
  1983          
  1984  
  1985  
  1986  
  1987  <p>One of Lambda&rsquo;s biggest strengths, its ability to automatically scale across ephemeral containers in response to increased load, also creates one of its biggest problems: observability. The traditional set of <a href="https://jvns.ca/">tools</a> used to identify performance bottlenecks are no longer valid, as there is no host into which one can SSH and interactively interrogate. Identifying performance bottlenecks is even more significant due to the Lambda <a href="https://aws.amazon.com/lambda/pricing/">pricing model</a>, where idle time often directly translates into increased costs.</p>
  1988  <p>However, Go offers the excellent <a href="https://rakyll.org/pprof-ui/">pprof</a> tool to visualize and cost allocate program execution. Beginning with <a href="https://github.com/mweagle/Sparta/blob/master/CHANGES.md#v0204">Sparta 0.20.4</a>, it&rsquo;s possible to enable per-lambda instance snapshotting which can be locally visualized. This documentation provides an overview of how to enable profiling. The full source is available at the <a href="https://github.com/mweagle/SpartaPProf">SpartaPProf</a> repo.</p>
  1989  <p>To learn more about <code>pprof</code> itself, please visit:</p>
  1990  <ul>
  1991  <li><a href="https://twitter.com/rakyll" class="fas fas-twitter">@rakyll&rsquo;s</a> <a href="https://rakyll.org/">blog</a></li>
  1992  <li><a href="https://jvns.ca/blog/2017/09/24/profiling-go-with-pprof/">Profiling Go programs with pprof</a></li>
  1993  <li><a href="https://medium.com/@tjholowaychuk/profiling-golang-851db2d9ae24">Profiling Golang</a></li>
  1994  <li><a href="http://artem.krylysov.com/blog/2017/03/13/profiling-and-optimizing-go-web-applications/">Profiling and optimizing Go web programs</a></li>
  1995  </ul>
  1996  <h2 id="enabling-profiling">Enabling Profiling</h2>
  1997  <p>To enable profiling add a reference to <a href="https://godoc.org/github.com/mweagle/Sparta#ScheduleProfileLoop">ScheduleProfileLoop</a>
  1998  in your <code>main()</code> function as in:</p>
  1999  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-go" data-lang="go">sparta.ScheduleProfileLoop(<span style="color:#00f">nil</span>,
  2000    5*time.Second,
  2001    30*time.Second,
  2002    <span style="color:#a31515">&#34;goroutine&#34;</span>,
  2003    <span style="color:#a31515">&#34;heap&#34;</span>,
  2004    <span style="color:#a31515">&#34;threadcreate&#34;</span>,
  2005    <span style="color:#a31515">&#34;block&#34;</span>,
  2006    <span style="color:#a31515">&#34;mutex&#34;</span>)
  2007  </code></pre></div><p>This function accepts the following arguments:</p>
  2008  <ul>
  2009  <li><code>s3Bucket</code>: The S3 bucket to which profile snapshots should be written. If <code>nil</code>, the bucket used to host the original ZIP code archive is used.</li>
  2010  <li><code>snapshotInterval</code> - The interval between each snapshot.</li>
  2011  <li><code>cpuProfileDuration</code> - The duration for the <a href="https://golang.org/pkg/runtime/pprof/#StartCPUProfile">CPUProfile</a> sample.</li>
  2012  <li><code>profileNames...</code> - The profile types to snapshot. In addition to the <a href="https://golang.org/pkg/runtime/pprof/#Profile">standard profiles</a>, Sparta includes a &ldquo;cpu&rdquo; profile iff the <code>cpuProfileDuration</code> is non-zero.</li>
  2013  </ul>
  2014  <h3 id="profiling-implementation">Profiling Implementation</h3>
  2015  <p>During the <code>provision</code> step, the <code>ScheduleProfileLoop</code> adds an <a href="https://godoc.org/github.com/mweagle/Sparta#IAMRolePrivilege">IAMRolePrivilege</a> <em>Allow</em> entry (if possible) to each Lambda function&rsquo;s IAM policy. This policy extension is a minimal privilege and only enables <code>s3:PutObject</code> against the Sparta managed key prefix (see below).</p>
  2016  <p>The <code>provision</code> implementation also annotates the Lambda&rsquo;s <a href="http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-lambda-function.html#cfn-lambda-function-environment">Environment</a> map so that the publishing loop knows where to publish snapshots.</p>
  2017  <p>During the <code>execute</code> step when the Sparta binary is executing in AWS Lambda, the <code>ScheduleProfileLoop</code> installs the requested sampling and publishing steps so that profile snapshots, serialized as <a href="https://github.com/google/pprof/blob/master/proto/profile.proto">proto</a> files, are properly saved to S3. Profiles are published to a reserved location in S3 with the form:</p>
  2018  <p>s3:://{BUCKET_NAME}/sparta/pprof/{STACK_NAME}/profiles/{PROFILE_TYPE}/{SNAPSHOT_INDEX}-{PROFILE_TYPE}.λ-{INSTANCE_ID}.profile</p>
  2019  <p>To manage profile sprawl, each lambda instance uses a rolling <code>SNAPSHOT_INDEX</code> to maintain a fixed size window. The new <code>profile</code> command is responsible for aggregating them into a single local, consolidated profile that can be visualized using the existing tools.</p>
  2020  <h2 id="deploying">Deploying</h2>
  2021  <p>With profiling enabled, the next step is to deploy the <em>SpartaPProf</em> service using the <code>provision</code> command:</p>
  2022  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-plain" data-lang="plain">$ go run main.go provision --s3Bucket MY-S3-BUCKET
  2023  INFO[0000] ════════════════════════════════════════════════
  2024  INFO[0000] ╔═╗╔═╗╔═╗╦═╗╔╦╗╔═╗   Version : 1.4.0
  2025  INFO[0000] ╚═╗╠═╝╠═╣╠╦╝ ║ ╠═╣   SHA     : 8f199e1
  2026  INFO[0000] ╚═╝╩  ╩ ╩╩╚═ ╩ ╩ ╩   Go      : go1.11.1
  2027  INFO[0000] ════════════════════════════════════════════════
  2028  INFO[0000] Service: SpartaPProf-mweagle                  LinkFlags= Option=provision UTC=&#34;2018-10-11T14:59:48Z&#34;
  2029  INFO[0000] ════════════════════════════════════════════════
  2030  INFO[0000] Using `git` SHA for StampedBuildID            Command=&#34;git rev-parse HEAD&#34; SHA=c3fbe8c289c3184efec842dca56b9bf541f39d21
  2031  INFO[0000] Provisioning service                          BuildID=c3fbe8c289c3184efec842dca56b9bf541f39d21 CodePipelineTrigger= InPlaceUpdates=false NOOP=false Tags=
  2032  INFO[0000] Verifying IAM Lambda execution roles
  2033  INFO[0000] IAM roles verified                            Count=2
  2034  INFO[0000] Checking S3 versioning                        Bucket=MY-S3-BUCKET VersioningEnabled=true
  2035  INFO[0000] Checking S3 region                            Bucket=MY-S3-BUCKET Region=us-west-2
  2036  INFO[0000] Running `go generate`
  2037  INFO[0000] Compiling binary                              Name=Sparta.lambda.amd64
  2038  INFO[0002] Creating code ZIP archive for upload          TempName=./.sparta/SpartaPProf_mweagle-code.zip
  2039  INFO[0002] Lambda code archive size                      Size=&#34;17 MB&#34;
  2040  INFO[0002] Uploading local file to S3                    Bucket=MY-S3-BUCKET Key=SpartaPProf-mweagle/SpartaPProf_mweagle-code.zip Path=./.sparta/SpartaPProf_mweagle-code.zip Size=&#34;17 MB&#34;
  2041  INFO[0009] Calling WorkflowHook                          ServiceDecoratorHook= WorkflowHookContext=&#34;map[]&#34;
  2042  INFO[0009] Uploading local file to S3                    Bucket=MY-S3-BUCKET Key=SpartaPProf-mweagle/SpartaPProf_mweagle-cftemplate.json Path=./.sparta/SpartaPProf_mweagle-cftemplate.json Size=&#34;7.1 kB&#34;
  2043  INFO[0010] Issued CreateChangeSet request                StackName=SpartaPProf-mweagle
  2044  INFO[0013] Issued ExecuteChangeSet request               StackName=SpartaPProf-mweagle
  2045  INFO[0026] CloudFormation Metrics ▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬
  2046  INFO[0026]     Operation duration                        Duration=11.71s Resource=SpartaPProf-mweagle Type=&#34;AWS::CloudFormation::Stack&#34;
  2047  INFO[0026]     Operation duration                        Duration=1.60s Resource=HelloWorldLambda7d01d27fe422d278bcc652b4a989528718eb88af Type=&#34;AWS::Lambda::Function&#34;
  2048  INFO[0026]     Operation duration                        Duration=1.33s Resource=KinesisLogConsumerLambda275ace0435c45228161570811178ce06fbcb359c Type=&#34;AWS::Lambda::Function&#34;
  2049  INFO[0026] Stack Outputs ▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬▬
  2050  INFO[0026]     HelloWorldFunctionARN                     Description=&#34;Hello World Lambda ARN&#34; Value=&#34;arn:aws:lambda:us-west-2:123412341234:function:SpartaPProf-mweagle_Hello_World&#34;
  2051  INFO[0026]     KinesisLogConsumerFunctionARN             Description=&#34;KinesisLogConsumer Lambda ARN&#34; Value=&#34;arn:aws:lambda:us-west-2:123412341234:function:SpartaPProf-mweagle_KinesisLogConsumer&#34;
  2052  INFO[0026] Stack provisioned                             CreationTime=&#34;2018-10-03 23:34:21.142 +0000 UTC&#34; StackId=&#34;arn:aws:cloudformation:us-west-2:123412341234:stack/SpartaPProf-mweagle/da781540-c764-11e8-9bf1-0aceeffcea3c&#34; StackName=SpartaPProf-mweagle
  2053  INFO[0026] ════════════════════════════════════════════════
  2054  INFO[0026] SpartaPProf-mweagle Summary
  2055  INFO[0026] ════════════════════════════════════════════════
  2056  INFO[0026] Verifying IAM roles                           Duration (s)=0
  2057  INFO[0026] Verifying AWS preconditions                   Duration (s)=1
  2058  INFO[0026] Creating code bundle                          Duration (s)=2
  2059  INFO[0026] Uploading code                                Duration (s)=8
  2060  INFO[0026] Ensuring CloudFormation stack                 Duration (s)=17
  2061  INFO[0026] Total elapsed time                            Duration (s)=27</code></pre></div>
  2062  <h2 id="generating-load">Generating Load</h2>
  2063  <p>While the <strong>SpartaPProf</strong> binary does include functions that are likely to generate profiling data, we still need to issue a sufficient series of events to produce a non-empty profile data set. <strong>SpartaPProf</strong> includes a simple tool (<em>cmd/load.go</em>) that directly calls the provisioned lambda function on a regular interval. It accepts a single command line argument, the <em>ARN</em> of the lambda function listed as a <em>Stack output</em> in the log output:</p>
  2064  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-plain" data-lang="plain">INFO[0058] FunctionARN                                   Description=&#34;Lambda function ARN&#34; Value=&#34;arn:aws:lambda:us-west-2:123412341234:function:SpartaPProf_mweagle_Hello_World&#34;</code></pre></div>
  2065  <p>Run the simple load generation script with the ARN value as in:</p>
  2066  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-plain" data-lang="plain">$ cd cmd
  2067  $ go run load.go arn:aws:lambda:us-west-2:012345678910:function:SpartaPProf-mweagle-Hello_World
  2068  Lambda response (0 of 60): &#34;Hi there 🌍&#34;
  2069  Lambda response (1 of 60): &#34;Hi there 🌍&#34;
  2070  Lambda response (2 of 60): &#34;Hi there 🌍&#34;
  2071  Lambda response (3 of 60): &#34;Hi there 🌍&#34;
  2072  Lambda response (4 of 60): &#34;Hi there 🌍&#34;
  2073  Lambda response (5 of 60): &#34;Hi there 🌍&#34;
  2074  Lambda response (6 of 60): &#34;Hi there 🌍&#34;
  2075  Lambda response (7 of 60): &#34;Hi there 🌍&#34;
  2076  Lambda response (8 of 60): &#34;Hi there 🌍&#34;
  2077  Lambda response (9 of 60): &#34;Hi there 🌍&#34;
  2078  Lambda response (10 of 60): &#34;Hi there 🌍&#34;
  2079  Lambda response (11 of 60): &#34;Hi there 🌍&#34;
  2080  ...</code></pre></div>
  2081  <p>After all the requests have completed for this sample against a stack provisioned in <code>us-west-2</code>, a set of named profiles was published. Since each container&rsquo;s instance id is randomly assigned, the profile names you see will have slightly different names</p>
  2082  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-plain" data-lang="plain">---------------------------------------------------------------
  2083  S3 bucket: s3://weagle/sparta/pprof/SpartaPProf-mweagle/profiles
  2084  ---------------------------------------------------------------
  2085  2017-11-26 11:32:28   41 Bytes sparta/pprof/SpartaPProf-mweagle/profiles/block/0-block.λ-3838737145763622974.profile
  2086  2017-11-26 11:32:27    1.8 KiB sparta/pprof/SpartaPProf-mweagle/profiles/cpu/0-cpu.λ-3838737145763622974.profile
  2087  2017-11-26 11:32:28    1.8 KiB sparta/pprof/SpartaPProf-mweagle/profiles/goroutine/0-goroutine.λ-3838737145763622974.profile
  2088  2017-11-26 11:32:28    2.2 KiB sparta/pprof/SpartaPProf-mweagle/profiles/heap/0-heap.λ-3838737145763622974.profile
  2089  2017-11-26 11:32:28   54 Bytes sparta/pprof/SpartaPProf-mweagle/profiles/mutex/0-mutex.λ-3838737145763622974.profile
  2090  2017-11-26 11:32:30  200 Bytes sparta/pprof/SpartaPProf-mweagle/profiles/threadcreate/0-threadcreate.λ-3838737145763622974.profile</code></pre></div>
  2091  <h2 id="visualizing-profiles">Visualizing Profiles</h2>
  2092  <p>Sparta delegates to the pprof <a href="https://github.com/google/pprof">webui</a> to visualize profile snapshots. Ensure you have the latest version of this by running <code>go get -u -v go get github.com/google/pprof</code> first.</p>
  2093  <p>The final step is to provide the profile snapshots to <code>pprof</code>. Sparta exposes a <code>profile</code> command that accomplishes this, by fetching and consolidating all published profiles for a single type.</p>
  2094  <div class="highlight"><pre style="background-color:#fff;-moz-tab-size:2;-o-tab-size:2;tab-size:2"><code class="language-plain" data-lang="plain">$ go run main.go profile --s3Bucket weagle
  2095  INFO[0000] ════════════════════════════════════════════════
  2096  INFO[0000] ╔═╗┌─┐┌─┐┬─┐┌┬┐┌─┐   Version : 1.0.2
  2097  INFO[0000] ╚═╗├─┘├─┤├┬┘ │ ├─┤   SHA     : b37b93e
  2098  INFO[0000] ╚═╝┴  ┴ ┴┴└─ ┴ ┴ ┴   Go      : go1.9.2
  2099  INFO[0000] ════════════════════════════════════════════════
  2100  INFO[0000] Service: SpartaPProf-mweagle                  LinkFlags= Option=profile UTC=&#34;2018-01-29T15:23:18Z&#34;
  2101  INFO[0000] ════════════════════════════════════════════════
  2102  ? Which stack would you like to profile: SpartaPProf-mweagle
  2103  ? What type of profile would you like to view? heap
  2104  ? What profile snapshot(s) would you like to view? Download new snapshots from S3
  2105  ? Please select a heap profile type: alloc_space
  2106  INFO[0028] Refreshing cached profiles                    CacheRoot=.sparta/profiles/SpartaPProf-mweagle/heap ProfileRootKey=sparta/pprof/SpartaPProf-mweagle/profiles/heap S3Bucket=MY-S3-BUCKET StackName=SpartaPProf-mweagle Type=heap
  2107  INFO[0028] Aggregating profile                           Input=&#34;.sparta/profiles/SpartaPProf-mweagle/heap/0-heap.λ-8850662459689822644.profile&#34;
  2108  INFO[0028] Consolidating profiles                        ProfileCount=1
  2109  INFO[0028] Creating consolidated profile                 ConsolidatedProfile=.sparta/heap.consolidated.profile
  2110  INFO[0028] Starting pprof webserver on http://localhost:8080. Enter Ctrl+C to exit.</code></pre></div>
  2111  <p>The <code>profile</code> command downloads the published profiles and consolidates them into a single cached version in the <em>./sparta</em> directory with a name of the form:</p>
  2112  <p>./.sparta/{PROFILE_TYPE}.consolidated.profile</p>
  2113  <p>You can choose to use the cached file if it exists.</p>
  2114  <p>For this sample run, the <em>heap</em> profile output is made available to the <code>pprof</code> webserver which produces the following layout:</p>
  2115  <p><img src="/images/profiling/main_alloc_space.jpg" alt="Main Alloc Space"></p>
  2116  <p>The latest <code>pprof</code> also includes flamegraph support to help identify issues:</p>
  2117  <p><img src="/images/profiling/main_alloc_space_flamegraph.jpg" alt="Main Alloc Space Flamegraph"></p>
  2118  <p>To view another profile type, enter <code>Ctrl+C</code> to exit the blocking web ui loop and launch another <code>profile</code> session.</p>
  2119  <h1 id="conclusion">Conclusion</h1>
  2120  <p>Go includes a very powerful set of tools that can help diagnose performance bottlenecks. With the Sparta <code>profile</code> command, it&rsquo;s possible to bring that same visibility to bear to AWS Lambda, despite running on ephemeral, (typically) unaddressable hosts. Get started optimizing today! And also, don&rsquo;t forget to disable the profiling loop before pushing to production.</p>
  2121  <h1 id="notes">Notes</h1>
  2122  <ul>
  2123  <li><a href="http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html">CPU Flame Graphs</a> provides a great overview.</li>
  2124  <li>It&rsquo;s not currently possible to use <a href="https://medium.com/@cep21/creating-custom-go-profiles-with-pprof-b737dfc58e11">custom profiles</a></li>
  2125  <li>Lambda instances are limited to a window size of 3 rolling snapshots</li>
  2126  <li>The <code>explore</code> command also exposes the <code>pprof</code> <a href="https://github.com/mweagle/Sparta/blob/ead2872585dc0da81f222577a898707c6a486ab1/execute_utils.go#L35">web handlers</a> for local exploration as well.</li>
  2127  </ul>
  2128  
  2129  
  2130  <footer class="footline">
  2131  	
  2132  </footer>
  2133  
  2134  
  2135  </div> 
  2136  
  2137  
  2138  </div>
  2139  
  2140  <div id="navigation">
  2141  
  2142  
  2143  
  2144  
  2145  
  2146  
  2147      
  2148          
  2149          
  2150      
  2151      
  2152  
  2153          
  2154          
  2155              
  2156          
  2157          
  2158  
  2159          
  2160              
  2161  
  2162  
  2163      
  2164          
  2165              
  2166              
  2167          
  2168      
  2169      
  2170  
  2171          
  2172          
  2173          
  2174  
  2175          
  2176  
  2177  
  2178          
  2179              
  2180  
  2181  
  2182      
  2183          
  2184      
  2185      
  2186  
  2187          
  2188          
  2189          
  2190  
  2191          
  2192              
  2193  
  2194  
  2195      
  2196          
  2197      
  2198      
  2199  
  2200          
  2201          
  2202          
  2203  
  2204          
  2205  
  2206  
  2207          
  2208              
  2209  
  2210  
  2211      
  2212          
  2213      
  2214      
  2215  
  2216          
  2217          
  2218          
  2219  
  2220          
  2221  
  2222  
  2223          
  2224  
  2225  
  2226          
  2227              
  2228  
  2229  
  2230      
  2231          
  2232      
  2233      
  2234  
  2235          
  2236          
  2237          
  2238  
  2239          
  2240  
  2241  
  2242          
  2243              
  2244  
  2245  
  2246      
  2247          
  2248          
  2249      
  2250      
  2251  
  2252          
  2253          
  2254              
  2255          
  2256          
  2257  
  2258          
  2259              
  2260  
  2261  
  2262      
  2263          
  2264              
  2265              
  2266          
  2267      
  2268      
  2269  
  2270          
  2271          
  2272          
  2273  
  2274          
  2275              
  2276  
  2277  
  2278      
  2279          
  2280      
  2281      
  2282  
  2283          
  2284          
  2285          
  2286  
  2287          
  2288  
  2289  
  2290          
  2291              
  2292  
  2293  
  2294      
  2295          
  2296      
  2297      
  2298  
  2299          
  2300          
  2301          
  2302  
  2303          
  2304  
  2305  
  2306          
  2307              
  2308  
  2309  
  2310      
  2311          
  2312      
  2313      
  2314  
  2315          
  2316          
  2317          
  2318  
  2319          
  2320  
  2321  
  2322          
  2323              
  2324  
  2325  
  2326      
  2327          
  2328      
  2329      
  2330  
  2331          
  2332          
  2333          
  2334  
  2335          
  2336  
  2337  
  2338          
  2339              
  2340  
  2341  
  2342      
  2343          
  2344      
  2345      
  2346  
  2347          
  2348          
  2349          
  2350  
  2351          
  2352  
  2353  
  2354          
  2355              
  2356  
  2357  
  2358      
  2359          
  2360      
  2361      
  2362  
  2363          
  2364          
  2365          
  2366  
  2367          
  2368  
  2369  
  2370          
  2371  
  2372  
  2373          
  2374              
  2375  
  2376  
  2377      
  2378          
  2379      
  2380      
  2381  
  2382          
  2383          
  2384          
  2385  
  2386          
  2387  
  2388  
  2389          
  2390              
  2391  
  2392  
  2393      
  2394          
  2395      
  2396      
  2397  
  2398          
  2399          
  2400          
  2401  
  2402          
  2403              
  2404  
  2405  
  2406      
  2407          
  2408      
  2409      
  2410  
  2411          
  2412          
  2413          
  2414  
  2415          
  2416  
  2417  
  2418          
  2419              
  2420  
  2421  
  2422      
  2423          
  2424      
  2425      
  2426  
  2427          
  2428          
  2429          
  2430  
  2431          
  2432  
  2433  
  2434          
  2435              
  2436  
  2437  
  2438      
  2439          
  2440      
  2441      
  2442  
  2443          
  2444          
  2445          
  2446  
  2447          
  2448  
  2449  
  2450          
  2451              
  2452  
  2453  
  2454      
  2455          
  2456      
  2457      
  2458  
  2459          
  2460          
  2461          
  2462  
  2463          
  2464  
  2465  
  2466          
  2467              
  2468  
  2469  
  2470      
  2471          
  2472      
  2473      
  2474  
  2475          
  2476          
  2477          
  2478  
  2479          
  2480  
  2481  
  2482          
  2483              
  2484  
  2485  
  2486      
  2487          
  2488      
  2489      
  2490  
  2491          
  2492          
  2493          
  2494  
  2495          
  2496  
  2497  
  2498          
  2499              
  2500  
  2501  
  2502      
  2503          
  2504      
  2505      
  2506  
  2507          
  2508          
  2509          
  2510  
  2511          
  2512  
  2513  
  2514          
  2515              
  2516  
  2517  
  2518      
  2519          
  2520      
  2521      
  2522  
  2523          
  2524          
  2525          
  2526  
  2527          
  2528  
  2529  
  2530          
  2531              
  2532  
  2533  
  2534      
  2535          
  2536      
  2537      
  2538  
  2539          
  2540          
  2541          
  2542  
  2543          
  2544  
  2545  
  2546          
  2547              
  2548  
  2549  
  2550      
  2551          
  2552      
  2553      
  2554  
  2555          
  2556          
  2557          
  2558  
  2559          
  2560  
  2561  
  2562          
  2563              
  2564  
  2565  
  2566      
  2567          
  2568      
  2569      
  2570  
  2571          
  2572          
  2573          
  2574  
  2575          
  2576  
  2577  
  2578          
  2579  
  2580  
  2581          
  2582              
  2583  
  2584  
  2585      
  2586          
  2587      
  2588      
  2589  
  2590          
  2591          
  2592          
  2593  
  2594          
  2595              
  2596  
  2597  
  2598      
  2599          
  2600      
  2601      
  2602  
  2603          
  2604          
  2605          
  2606  
  2607          
  2608  
  2609  
  2610          
  2611              
  2612  
  2613  
  2614      
  2615          
  2616      
  2617      
  2618  
  2619          
  2620          
  2621          
  2622  
  2623          
  2624  
  2625  
  2626          
  2627              
  2628  
  2629  
  2630      
  2631          
  2632      
  2633      
  2634  
  2635          
  2636          
  2637          
  2638  
  2639          
  2640  
  2641  
  2642          
  2643              
  2644  
  2645  
  2646      
  2647          
  2648      
  2649      
  2650  
  2651          
  2652          
  2653          
  2654  
  2655          
  2656  
  2657  
  2658          
  2659              
  2660  
  2661  
  2662      
  2663          
  2664      
  2665      
  2666  
  2667          
  2668          
  2669          
  2670  
  2671          
  2672  
  2673  
  2674          
  2675              
  2676  
  2677  
  2678      
  2679          
  2680      
  2681      
  2682  
  2683          
  2684          
  2685          
  2686  
  2687          
  2688  
  2689  
  2690          
  2691              
  2692  
  2693  
  2694      
  2695          
  2696      
  2697      
  2698  
  2699          
  2700          
  2701          
  2702  
  2703          
  2704  
  2705  
  2706          
  2707              
  2708  
  2709  
  2710      
  2711          
  2712      
  2713      
  2714  
  2715          
  2716          
  2717          
  2718  
  2719          
  2720  
  2721  
  2722          
  2723              
  2724  
  2725  
  2726      
  2727          
  2728      
  2729      
  2730  
  2731          
  2732          
  2733          
  2734  
  2735          
  2736  
  2737  
  2738          
  2739  
  2740  
  2741          
  2742              
  2743  
  2744  
  2745      
  2746          
  2747      
  2748      
  2749  
  2750          
  2751          
  2752          
  2753  
  2754          
  2755              
  2756  
  2757  
  2758      
  2759          
  2760      
  2761      
  2762  
  2763          
  2764          
  2765          
  2766  
  2767          
  2768  
  2769  
  2770          
  2771              
  2772  
  2773  
  2774      
  2775          
  2776      
  2777      
  2778  
  2779          
  2780          
  2781          
  2782  
  2783          
  2784  
  2785  
  2786          
  2787              
  2788  
  2789  
  2790      
  2791          
  2792      
  2793      
  2794  
  2795          
  2796          
  2797          
  2798  
  2799          
  2800  
  2801  
  2802          
  2803              
  2804  
  2805  
  2806      
  2807          
  2808      
  2809      
  2810  
  2811          
  2812          
  2813          
  2814  
  2815          
  2816  
  2817  
  2818          
  2819              
  2820  
  2821  
  2822      
  2823          
  2824      
  2825      
  2826  
  2827          
  2828          
  2829          
  2830  
  2831          
  2832  
  2833  
  2834          
  2835              
  2836  
  2837  
  2838      
  2839          
  2840      
  2841      
  2842  
  2843          
  2844          
  2845          
  2846  
  2847          
  2848  
  2849  
  2850          
  2851              
  2852  
  2853  
  2854      
  2855          
  2856      
  2857      
  2858  
  2859          
  2860          
  2861          
  2862  
  2863          
  2864  
  2865  
  2866          
  2867  
  2868  
  2869          
  2870              
  2871  
  2872  
  2873      
  2874          
  2875      
  2876      
  2877  
  2878          
  2879          
  2880          
  2881  
  2882          
  2883              
  2884  
  2885  
  2886      
  2887          
  2888      
  2889      
  2890  
  2891          
  2892          
  2893          
  2894  
  2895          
  2896  
  2897  
  2898          
  2899  
  2900  
  2901          
  2902              
  2903  
  2904  
  2905      
  2906          
  2907      
  2908      
  2909  
  2910          
  2911          
  2912          
  2913  
  2914          
  2915  
  2916  
  2917          
  2918              
  2919  
  2920  
  2921      
  2922          
  2923      
  2924      
  2925  
  2926          
  2927          
  2928          
  2929  
  2930          
  2931              
  2932  
  2933  
  2934      
  2935          
  2936      
  2937      
  2938  
  2939          
  2940          
  2941          
  2942  
  2943          
  2944  
  2945  
  2946          
  2947              
  2948  
  2949  
  2950      
  2951          
  2952      
  2953      
  2954  
  2955          
  2956          
  2957          
  2958  
  2959          
  2960  
  2961  
  2962          
  2963              
  2964  
  2965  
  2966      
  2967          
  2968      
  2969      
  2970  
  2971          
  2972          
  2973          
  2974  
  2975          
  2976  
  2977  
  2978          
  2979              
  2980  
  2981  
  2982      
  2983          
  2984      
  2985      
  2986  
  2987          
  2988          
  2989          
  2990  
  2991          
  2992  
  2993  
  2994          
  2995              
  2996  
  2997  
  2998      
  2999          
  3000      
  3001      
  3002  
  3003          
  3004          
  3005          
  3006  
  3007          
  3008  
  3009  
  3010          
  3011  
  3012  
  3013          
  3014              
  3015  
  3016  
  3017      
  3018          
  3019      
  3020      
  3021  
  3022          
  3023          
  3024              
  3025          
  3026          
  3027  
  3028          
  3029              
  3030  
  3031  
  3032      
  3033          
  3034      
  3035      
  3036  
  3037          
  3038          
  3039          
  3040  
  3041          
  3042  
  3043  
  3044          
  3045              
  3046  
  3047  
  3048      
  3049          
  3050      
  3051      
  3052  
  3053          
  3054          
  3055          
  3056  
  3057          
  3058  
  3059  
  3060          
  3061              
  3062  
  3063  
  3064      
  3065          
  3066      
  3067      
  3068  
  3069          
  3070          
  3071          
  3072  
  3073          
  3074              
  3075  
  3076  
  3077      
  3078          
  3079      
  3080      
  3081  
  3082          
  3083          
  3084          
  3085  
  3086          
  3087  
  3088  
  3089          
  3090              
  3091  
  3092  
  3093      
  3094          
  3095      
  3096      
  3097  
  3098          
  3099          
  3100          
  3101  
  3102          
  3103  
  3104  
  3105          
  3106              
  3107  
  3108  
  3109      
  3110          
  3111      
  3112      
  3113  
  3114          
  3115          
  3116          
  3117  
  3118          
  3119  
  3120  
  3121          
  3122              
  3123  
  3124  
  3125      
  3126          
  3127      
  3128      
  3129  
  3130          
  3131          
  3132          
  3133  
  3134          
  3135  
  3136  
  3137          
  3138              
  3139  
  3140  
  3141      
  3142          
  3143      
  3144      
  3145  
  3146          
  3147          
  3148          
  3149  
  3150          
  3151  
  3152  
  3153          
  3154              
  3155  
  3156  
  3157      
  3158          
  3159      
  3160      
  3161  
  3162          
  3163          
  3164          
  3165  
  3166          
  3167  
  3168  
  3169          
  3170              
  3171  
  3172  
  3173      
  3174          
  3175      
  3176      
  3177  
  3178          
  3179          
  3180          
  3181  
  3182          
  3183  
  3184  
  3185          
  3186  
  3187  
  3188          
  3189  
  3190  
  3191          
  3192              
  3193  
  3194  
  3195      
  3196          
  3197      
  3198      
  3199  
  3200          
  3201          
  3202          
  3203  
  3204          
  3205  
  3206  
  3207          
  3208              
  3209  
  3210  
  3211      
  3212          
  3213      
  3214      
  3215  
  3216          
  3217          
  3218          
  3219  
  3220          
  3221  
  3222  
  3223          
  3224              
  3225  
  3226  
  3227      
  3228          
  3229      
  3230      
  3231  
  3232          
  3233          
  3234          
  3235  
  3236          
  3237  
  3238  
  3239          
  3240              
  3241  
  3242  
  3243      
  3244          
  3245          
  3246      
  3247      
  3248  
  3249          
  3250          
  3251          
  3252  
  3253          
  3254              
  3255  
  3256  
  3257      
  3258          
  3259              
  3260              
  3261          
  3262      
  3263      
  3264  
  3265          
  3266          
  3267          
  3268  
  3269          
  3270  
  3271  
  3272          
  3273              
  3274  
  3275  
  3276      
  3277          
  3278      
  3279      
  3280  
  3281          
  3282          
  3283          
  3284  
  3285          
  3286  
  3287  
  3288          
  3289              
  3290  
  3291  
  3292      
  3293          
  3294      
  3295      
  3296  
  3297          
  3298          
  3299          
  3300  
  3301          
  3302  
  3303  
  3304          
  3305              
  3306  
  3307  
  3308      
  3309          
  3310      
  3311      
  3312  
  3313          
  3314          
  3315          
  3316  
  3317          
  3318  
  3319  
  3320          
  3321              
  3322  
  3323  
  3324      
  3325          
  3326      
  3327      
  3328  
  3329          
  3330          
  3331          
  3332  
  3333          
  3334  
  3335  
  3336          
  3337              
  3338  
  3339  
  3340      
  3341          
  3342      
  3343      
  3344  
  3345          
  3346          
  3347          
  3348  
  3349          
  3350  
  3351  
  3352          
  3353              
  3354  
  3355  
  3356      
  3357          
  3358      
  3359      
  3360  
  3361          
  3362          
  3363          
  3364  
  3365          
  3366  
  3367  
  3368          
  3369              
  3370  
  3371  
  3372      
  3373          
  3374          
  3375      
  3376      
  3377  
  3378          
  3379          
  3380          
  3381  
  3382          
  3383  
  3384  
  3385          
  3386  
  3387  
  3388          
  3389              
  3390  
  3391  
  3392      
  3393          
  3394              
  3395              
  3396          
  3397      
  3398      
  3399  
  3400          
  3401          
  3402          
  3403  
  3404          
  3405  
  3406  
  3407          
  3408              
  3409  
  3410  
  3411      
  3412          
  3413      
  3414      
  3415  
  3416          
  3417          
  3418          
  3419  
  3420          
  3421  
  3422  
  3423          
  3424              
  3425  
  3426  
  3427      
  3428          
  3429      
  3430      
  3431  
  3432          
  3433          
  3434          
  3435  
  3436          
  3437  
  3438  
  3439          
  3440              
  3441  
  3442  
  3443      
  3444          
  3445      
  3446      
  3447  
  3448          
  3449          
  3450          
  3451  
  3452          
  3453  
  3454  
  3455          
  3456  
  3457  
  3458          
  3459              
  3460  
  3461  
  3462      
  3463          
  3464      
  3465      
  3466  
  3467          
  3468          
  3469          
  3470  
  3471          
  3472  
  3473  
  3474          
  3475              
  3476  
  3477  
  3478      
  3479          
  3480      
  3481      
  3482  
  3483          
  3484          
  3485          
  3486  
  3487          
  3488  
  3489  
  3490          
  3491              
  3492  
  3493  
  3494      
  3495          
  3496      
  3497      
  3498  
  3499          
  3500          
  3501          
  3502  
  3503          
  3504  
  3505  
  3506          
  3507  
  3508  
  3509  
  3510  
  3511  
  3512  
  3513  
  3514  
  3515  
  3516  <a class="nav nav-prev" href="/reference/operations/metrics_publisher/" title="Metrics Publisher"> <i class="fa fa-chevron-left"></i></a>
  3517  
  3518  
  3519  <a class="nav nav-next" href="/reference/testing/" title="Testing" style="margin-right: 0px;"><i class="fa fa-chevron-right"></i></a>
  3520  
  3521  
  3522  </div>
  3523  
  3524  </section>
  3525  
  3526  <div style="left: -1000px; overflow: scroll; position: absolute; top: -1000px; border: none; box-sizing: content-box; height: 200px; margin: 0px; padding: 0px; width: 200px;">
  3527  <div style="border: none; box-sizing: content-box; height: 200px; margin: 0px; padding: 0px; width: 200px;"></div>
  3528  </div>
  3529  <script src="/js/clipboard.min.js?1588464711"></script>
  3530  <script src="/js/perfect-scrollbar.min.js?1588464711"></script>
  3531  <script src="/js/perfect-scrollbar.jquery.min.js?1588464711"></script>
  3532  <script src="/js/jquery.sticky.js?1588464711"></script>
  3533  <script src="/js/featherlight.min.js?1588464711"></script>
  3534  <script src="/js/html5shiv-printshiv.min.js?1588464711"></script>
  3535  <script src="/js/modernizr.custom-3.6.0.js?1588464711"></script>
  3536  <script src="/js/learn.js?1588464711"></script>
  3537  <script src="/js/hugo-learn.js?1588464711"></script>
  3538  
  3539  <link href="/mermaid/mermaid.css?1588464711" rel="stylesheet" />
  3540  <script src="/mermaid/mermaid.js?1588464711"></script>
  3541  <script>
  3542  mermaid.initialize({ startOnLoad: true });
  3543  </script>
  3544  
  3545  <script async src="https://www.googletagmanager.com/gtag/js?id=UA-70794971-2"></script>
  3546  <script>
  3547    window.dataLayer = window.dataLayer || [];
  3548    function gtag(){dataLayer.push(arguments);}
  3549    gtag('js', new Date());
  3550  
  3551    gtag('config', 'UA-70794971-2');
  3552  </script>
  3553  </body>
  3554  </html>
  3555