github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/module/metrics/execution.go (about) 1 package metrics 2 3 import ( 4 "time" 5 6 "github.com/prometheus/client_golang/prometheus" 7 "github.com/prometheus/client_golang/prometheus/promauto" 8 9 "github.com/onflow/flow-go/model/flow" 10 "github.com/onflow/flow-go/module" 11 "github.com/onflow/flow-go/module/counters" 12 ) 13 14 type ExecutionCollector struct { 15 tracer module.Tracer 16 totalExecutedBlocksCounter prometheus.Counter 17 totalExecutedCollectionsCounter prometheus.Counter 18 totalExecutedTransactionsCounter prometheus.Counter 19 totalExecutedScriptsCounter prometheus.Counter 20 totalFailedTransactionsCounter prometheus.Counter 21 lastExecutedBlockHeightGauge prometheus.Gauge 22 lastFinalizedExecutedBlockHeightGauge prometheus.Gauge 23 stateStorageDiskTotal prometheus.Gauge 24 storageStateCommitment prometheus.Gauge 25 checkpointSize prometheus.Gauge 26 forestApproxMemorySize prometheus.Gauge 27 forestNumberOfTrees prometheus.Gauge 28 latestTrieRegCount prometheus.Gauge 29 latestTrieRegCountDiff prometheus.Gauge 30 latestTrieRegSize prometheus.Gauge 31 latestTrieRegSizeDiff prometheus.Gauge 32 latestTrieMaxDepthTouched prometheus.Gauge 33 updated prometheus.Counter 34 proofSize prometheus.Gauge 35 updatedValuesNumber prometheus.Counter 36 updatedValuesSize prometheus.Gauge 37 updatedDuration prometheus.Histogram 38 updatedDurationPerValue prometheus.Histogram 39 readValuesNumber prometheus.Counter 40 readValuesSize prometheus.Gauge 41 readDuration prometheus.Histogram 42 readDurationPerValue prometheus.Histogram 43 blockComputationUsed prometheus.Histogram 44 blockComputationVector *prometheus.GaugeVec 45 blockCachedPrograms prometheus.Gauge 46 blockMemoryUsed prometheus.Histogram 47 blockEventCounts prometheus.Histogram 48 blockEventSize prometheus.Histogram 49 blockExecutionTime prometheus.Histogram 50 blockTransactionCounts prometheus.Histogram 51 blockCollectionCounts prometheus.Histogram 52 collectionComputationUsed prometheus.Histogram 53 collectionMemoryUsed prometheus.Histogram 54 collectionEventSize prometheus.Histogram 55 collectionEventCounts prometheus.Histogram 56 collectionNumberOfRegistersTouched prometheus.Histogram 57 collectionTotalBytesWrittenToRegisters prometheus.Histogram 58 collectionExecutionTime prometheus.Histogram 59 collectionTransactionCounts prometheus.Histogram 60 collectionRequestSent prometheus.Counter 61 collectionRequestRetried prometheus.Counter 62 transactionParseTime prometheus.Histogram 63 transactionCheckTime prometheus.Histogram 64 transactionInterpretTime prometheus.Histogram 65 transactionExecutionTime prometheus.Histogram 66 transactionConflictRetries prometheus.Histogram 67 transactionMemoryEstimate prometheus.Histogram 68 transactionComputationUsed prometheus.Histogram 69 transactionNormalizedTimePerComputation prometheus.Histogram 70 transactionEmittedEvents prometheus.Histogram 71 transactionEventSize prometheus.Histogram 72 scriptExecutionTime prometheus.Histogram 73 scriptComputationUsed prometheus.Histogram 74 scriptMemoryUsage prometheus.Histogram 75 scriptMemoryEstimate prometheus.Histogram 76 scriptMemoryDifference prometheus.Histogram 77 numberOfAccounts prometheus.Gauge 78 programsCacheMiss prometheus.Counter 79 programsCacheHit prometheus.Counter 80 chunkDataPackRequestProcessedTotal prometheus.Counter 81 chunkDataPackProofSize prometheus.Histogram 82 chunkDataPackCollectionSize prometheus.Histogram 83 stateSyncActive prometheus.Gauge 84 blockDataUploadsInProgress prometheus.Gauge 85 blockDataUploadsDuration prometheus.Histogram 86 maxCollectionHeightData counters.StrictMonotonousCounter 87 maxCollectionHeight prometheus.Gauge 88 computationResultUploadedCount prometheus.Counter 89 computationResultUploadRetriedCount prometheus.Counter 90 } 91 92 func NewExecutionCollector(tracer module.Tracer) *ExecutionCollector { 93 94 forestApproxMemorySize := promauto.NewGauge(prometheus.GaugeOpts{ 95 Namespace: namespaceExecution, 96 Subsystem: subsystemMTrie, 97 Name: "forest_approx_memory_size", 98 Help: "an approximate size of in-memory forest in bytes", 99 }) 100 101 forestNumberOfTrees := promauto.NewGauge(prometheus.GaugeOpts{ 102 Namespace: namespaceExecution, 103 Subsystem: subsystemMTrie, 104 Name: "forest_number_of_trees", 105 Help: "the number of trees in memory", 106 }) 107 108 latestTrieRegCount := promauto.NewGauge(prometheus.GaugeOpts{ 109 Namespace: namespaceExecution, 110 Subsystem: subsystemMTrie, 111 Name: "latest_trie_reg_count", 112 Help: "the number of allocated registers (latest created trie)", 113 }) 114 115 latestTrieRegCountDiff := promauto.NewGauge(prometheus.GaugeOpts{ 116 Namespace: namespaceExecution, 117 Subsystem: subsystemMTrie, 118 Name: "latest_trie_reg_count_diff", 119 Help: "the difference between number of unique register allocated of the latest created trie and parent trie", 120 }) 121 122 latestTrieRegSize := promauto.NewGauge(prometheus.GaugeOpts{ 123 Namespace: namespaceExecution, 124 Subsystem: subsystemMTrie, 125 Name: "latest_trie_reg_size", 126 Help: "the size of allocated registers (latest created trie)", 127 }) 128 129 latestTrieRegSizeDiff := promauto.NewGauge(prometheus.GaugeOpts{ 130 Namespace: namespaceExecution, 131 Subsystem: subsystemMTrie, 132 Name: "latest_trie_reg_size_diff", 133 Help: "the difference between size of unique register allocated of the latest created trie and parent trie", 134 }) 135 136 latestTrieMaxDepthTouched := promauto.NewGauge(prometheus.GaugeOpts{ 137 Namespace: namespaceExecution, 138 Subsystem: subsystemMTrie, 139 Name: "latest_trie_max_depth_touched", 140 Help: "the maximum depth touched of the latest created trie", 141 }) 142 143 updatedCount := promauto.NewCounter(prometheus.CounterOpts{ 144 Namespace: namespaceExecution, 145 Subsystem: subsystemMTrie, 146 Name: "updates_counted", 147 Help: "the number of updates", 148 }) 149 150 proofSize := promauto.NewGauge(prometheus.GaugeOpts{ 151 Namespace: namespaceExecution, 152 Subsystem: subsystemMTrie, 153 Name: "average_proof_size", 154 Help: "the average size of a single generated proof in bytes", 155 }) 156 157 updatedValuesNumber := promauto.NewCounter(prometheus.CounterOpts{ 158 Namespace: namespaceExecution, 159 Subsystem: subsystemMTrie, 160 Name: "update_values_number", 161 Help: "the total number of values updated", 162 }) 163 164 updatedValuesSize := promauto.NewGauge(prometheus.GaugeOpts{ 165 Namespace: namespaceExecution, 166 Subsystem: subsystemMTrie, 167 Name: "update_values_size", 168 Help: "the total size of values for single update in bytes", 169 }) 170 171 updatedDuration := promauto.NewHistogram(prometheus.HistogramOpts{ 172 Namespace: namespaceExecution, 173 Subsystem: subsystemMTrie, 174 Name: "update_duration", 175 Help: "the duration of update operation", 176 Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5}, 177 }) 178 179 updatedDurationPerValue := promauto.NewHistogram(prometheus.HistogramOpts{ 180 Namespace: namespaceExecution, 181 Subsystem: subsystemMTrie, 182 Name: "update_duration_per_value", 183 Help: "the duration of update operation per value", 184 Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5}, 185 }) 186 187 readValuesNumber := promauto.NewCounter(prometheus.CounterOpts{ 188 Namespace: namespaceExecution, 189 Subsystem: subsystemMTrie, 190 Name: "read_values_number", 191 Help: "the total number of values read", 192 }) 193 194 readValuesSize := promauto.NewGauge(prometheus.GaugeOpts{ 195 Namespace: namespaceExecution, 196 Subsystem: subsystemMTrie, 197 Name: "read_values_size", 198 Help: "the total size of values for single read in bytes", 199 }) 200 201 readDuration := promauto.NewHistogram(prometheus.HistogramOpts{ 202 Namespace: namespaceExecution, 203 Subsystem: subsystemMTrie, 204 Name: "read_duration", 205 Help: "the duration of read operation", 206 Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5}, 207 }) 208 209 readDurationPerValue := promauto.NewHistogram(prometheus.HistogramOpts{ 210 Namespace: namespaceExecution, 211 Subsystem: subsystemMTrie, 212 Name: "read_duration_per_value", 213 Help: "the duration of read operation per value", 214 Buckets: []float64{0.05, 0.2, 0.5, 1, 2, 5}, 215 }) 216 217 blockExecutionTime := promauto.NewHistogram(prometheus.HistogramOpts{ 218 Namespace: namespaceExecution, 219 Subsystem: subsystemRuntime, 220 Name: "block_execution_time_milliseconds", 221 Help: "the total time spent on block execution in milliseconds", 222 Buckets: []float64{50, 100, 200, 300, 400, 1000, 2000, 6000}, 223 }) 224 225 blockComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 226 Namespace: namespaceExecution, 227 Subsystem: subsystemRuntime, 228 Name: "block_computation_used", 229 Help: "the total amount of computation used by a block", 230 Buckets: []float64{1000, 10000, 100000, 500000, 1000000, 5000000, 10000000}, 231 }) 232 233 blockMemoryUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 234 Namespace: namespaceExecution, 235 Subsystem: subsystemRuntime, 236 Name: "block_memory_used", 237 Help: "the total amount of memory (cadence estimate) used by a block", 238 Buckets: []float64{100_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000, 50_000_000_000, 100_000_000_000, 500_000_000_000, 1_000_000_000_000, 5_000_000_000_000, 10_000_000_000_000}, 239 }) 240 241 blockEventCounts := promauto.NewHistogram(prometheus.HistogramOpts{ 242 Namespace: namespaceExecution, 243 Subsystem: subsystemRuntime, 244 Name: "block_event_counts", 245 Help: "the total number of events emitted during a block execution", 246 Buckets: []float64{10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000}, 247 }) 248 249 blockEventSize := promauto.NewHistogram(prometheus.HistogramOpts{ 250 Namespace: namespaceExecution, 251 Subsystem: subsystemRuntime, 252 Name: "block_event_size", 253 Help: "the total number of bytes used by events emitted during a block execution", 254 Buckets: []float64{1_000, 10_000, 100_000, 500_000, 1_000_000, 5_000_000, 10_000_000, 50_000_000, 100_000_000, 500_000_000}, 255 }) 256 257 blockComputationVector := promauto.NewGaugeVec(prometheus.GaugeOpts{ 258 Namespace: namespaceExecution, 259 Subsystem: subsystemRuntime, 260 Name: "block_execution_effort_vector", 261 Help: "execution effort vector of the last executed block by computation kind", 262 }, []string{LabelComputationKind}) 263 264 blockCachedPrograms := promauto.NewGauge(prometheus.GaugeOpts{ 265 Namespace: namespaceExecution, 266 Subsystem: subsystemRuntime, 267 Name: "block_execution_cached_programs", 268 Help: "Number of cached programs at the end of block execution", 269 }) 270 271 blockTransactionCounts := promauto.NewHistogram(prometheus.HistogramOpts{ 272 Namespace: namespaceExecution, 273 Subsystem: subsystemRuntime, 274 Name: "block_transaction_counts", 275 Help: "the total number of transactions per block", 276 Buckets: prometheus.ExponentialBuckets(4, 2, 10), 277 }) 278 279 blockCollectionCounts := promauto.NewHistogram(prometheus.HistogramOpts{ 280 Namespace: namespaceExecution, 281 Subsystem: subsystemRuntime, 282 Name: "block_collection_counts", 283 Help: "the total number of collections per block", 284 Buckets: prometheus.ExponentialBuckets(1, 2, 8), 285 }) 286 287 collectionExecutionTime := promauto.NewHistogram(prometheus.HistogramOpts{ 288 Namespace: namespaceExecution, 289 Subsystem: subsystemRuntime, 290 Name: "collection_execution_time_milliseconds", 291 Help: "the total time spent on collection execution in milliseconds", 292 Buckets: []float64{100, 200, 500, 1000, 1500, 2000}, 293 }) 294 295 collectionComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 296 Namespace: namespaceExecution, 297 Subsystem: subsystemRuntime, 298 Name: "collection_computation_used", 299 Help: "the total amount of computation used by a collection", 300 Buckets: []float64{1000, 10000, 50000, 100000, 500000, 1000000}, 301 }) 302 303 collectionMemoryUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 304 Namespace: namespaceExecution, 305 Subsystem: subsystemRuntime, 306 Name: "collection_memory_used", 307 Help: "the total amount of memory used (cadence estimate) by a collection", 308 Buckets: []float64{10_000_000, 100_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000, 50_000_000_000, 100_000_000_000, 500_000_000_000, 1_000_000_000_000, 5_000_000_000_000}, 309 }) 310 311 collectionEventSize := promauto.NewHistogram(prometheus.HistogramOpts{ 312 Namespace: namespaceExecution, 313 Subsystem: subsystemRuntime, 314 Name: "collection_event_size", 315 Help: "the total byte size used by all events generated during a collection execution", 316 Buckets: []float64{100, 1000, 10000, 100000, 10000000, 100000000, 1000000000}, 317 }) 318 319 collectionEventCounts := promauto.NewHistogram(prometheus.HistogramOpts{ 320 Namespace: namespaceExecution, 321 Subsystem: subsystemRuntime, 322 Name: "collection_event_counts", 323 Help: "the total number of events emitted per collection", 324 Buckets: prometheus.ExponentialBuckets(4, 2, 8), 325 }) 326 327 collectionNumberOfRegistersTouched := promauto.NewHistogram(prometheus.HistogramOpts{ 328 Namespace: namespaceExecution, 329 Subsystem: subsystemRuntime, 330 Name: "collection_number_of_registers_touched", 331 Help: "the total number of registers touched during collection execution", 332 Buckets: prometheus.ExponentialBuckets(10, 2, 12), 333 }) 334 335 collectionTotalBytesWrittenToRegisters := promauto.NewHistogram(prometheus.HistogramOpts{ 336 Namespace: namespaceExecution, 337 Subsystem: subsystemRuntime, 338 Name: "collection_total_number_of_bytes_written_to_registers", 339 Help: "the total number of bytes written to registers during collection execution", 340 Buckets: prometheus.ExponentialBuckets(1000, 2, 16), 341 }) 342 343 collectionTransactionCounts := promauto.NewHistogram(prometheus.HistogramOpts{ 344 Namespace: namespaceExecution, 345 Subsystem: subsystemRuntime, 346 Name: "collection_transaction_counts", 347 Help: "the total number of transactions per collection", 348 Buckets: prometheus.ExponentialBuckets(4, 2, 8), 349 }) 350 351 collectionRequestsSent := promauto.NewCounter(prometheus.CounterOpts{ 352 Namespace: namespaceExecution, 353 Subsystem: subsystemIngestion, 354 Name: "collection_requests_sent", 355 Help: "the number of collection requests sent", 356 }) 357 358 collectionRequestsRetries := promauto.NewCounter(prometheus.CounterOpts{ 359 Namespace: namespaceExecution, 360 Subsystem: subsystemIngestion, 361 Name: "collection_requests_retries", 362 Help: "the number of collection requests retried", 363 }) 364 365 transactionParseTime := promauto.NewHistogram(prometheus.HistogramOpts{ 366 Namespace: namespaceExecution, 367 Subsystem: subsystemRuntime, 368 Name: "transaction_parse_time_nanoseconds", 369 Help: "the parse time for a transaction in nanoseconds", 370 Buckets: prometheus.ExponentialBuckets(10, 10, 8), 371 }) 372 373 transactionCheckTime := promauto.NewHistogram(prometheus.HistogramOpts{ 374 Namespace: namespaceExecution, 375 Subsystem: subsystemRuntime, 376 Name: "transaction_check_time_nanoseconds", 377 Help: "the checking time for a transaction in nanoseconds", 378 Buckets: prometheus.ExponentialBuckets(10, 10, 8), 379 }) 380 381 transactionInterpretTime := promauto.NewHistogram(prometheus.HistogramOpts{ 382 Namespace: namespaceExecution, 383 Subsystem: subsystemRuntime, 384 Name: "transaction_interpret_time_nanoseconds", 385 Help: "the interpretation time for a transaction in nanoseconds", 386 Buckets: prometheus.ExponentialBuckets(10, 10, 8), 387 }) 388 389 transactionExecutionTime := promauto.NewHistogram(prometheus.HistogramOpts{ 390 Namespace: namespaceExecution, 391 Subsystem: subsystemRuntime, 392 Name: "transaction_execution_time_milliseconds", 393 Help: "the total time spent on transaction execution in milliseconds", 394 Buckets: prometheus.ExponentialBuckets(2, 2, 10), 395 }) 396 397 transactionConflictRetries := promauto.NewHistogram(prometheus.HistogramOpts{ 398 Namespace: namespaceExecution, 399 Subsystem: subsystemRuntime, 400 Name: "transaction_conflict_retries", 401 Help: "the number of conflict retries needed to successfully commit a transaction. If retry count is high, consider reducing concurrency", 402 Buckets: []float64{0, 1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 100}, 403 }) 404 405 transactionComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 406 Namespace: namespaceExecution, 407 Subsystem: subsystemRuntime, 408 Name: "transaction_computation_used", 409 Help: "the total amount of computation used by a transaction", 410 Buckets: []float64{50, 100, 500, 1000, 5000, 10000}, 411 }) 412 413 transactionNormalizedTimePerComputation := promauto.NewHistogram(prometheus.HistogramOpts{ 414 Namespace: namespaceExecution, 415 Subsystem: subsystemRuntime, 416 Name: "transaction_ms_per_computation", 417 Help: "The normalized ratio of millisecond of execution time per computation used. Value below 1 means the transaction was executed faster than estimated (is using less resources then estimated)", 418 Buckets: []float64{0.015625, 0.03125, 0.0625, 0.125, 0.25, 0.5, 1, 2, 4, 8, 16, 32, 64}, 419 }) 420 421 transactionMemoryEstimate := promauto.NewHistogram(prometheus.HistogramOpts{ 422 Namespace: namespaceExecution, 423 Subsystem: subsystemRuntime, 424 Name: "transaction_memory_estimate", 425 Help: "the estimated memory used by a transaction", 426 Buckets: []float64{1_000_000, 10_000_000, 100_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000, 50_000_000_000, 100_000_000_000}, 427 }) 428 429 transactionEmittedEvents := promauto.NewHistogram(prometheus.HistogramOpts{ 430 Namespace: namespaceExecution, 431 Subsystem: subsystemRuntime, 432 Name: "transaction_emitted_events", 433 Help: "the total number of events emitted by a transaction", 434 Buckets: prometheus.ExponentialBuckets(2, 2, 10), 435 }) 436 437 transactionEventSize := promauto.NewHistogram(prometheus.HistogramOpts{ 438 Namespace: namespaceExecution, 439 Subsystem: subsystemRuntime, 440 Name: "transaction_event_size", 441 Help: "the total number bytes used of events emitted during a transaction execution", 442 Buckets: prometheus.ExponentialBuckets(100, 2, 12), 443 }) 444 445 scriptExecutionTime := promauto.NewHistogram(prometheus.HistogramOpts{ 446 Namespace: namespaceExecution, 447 Subsystem: subsystemRuntime, 448 Name: "script_execution_time_milliseconds", 449 Help: "the total time spent on script execution in milliseconds", 450 Buckets: []float64{2, 4, 8, 16, 32, 64, 100, 250, 500}, 451 }) 452 453 scriptComputationUsed := promauto.NewHistogram(prometheus.HistogramOpts{ 454 Namespace: namespaceExecution, 455 Subsystem: subsystemRuntime, 456 Name: "script_computation_used", 457 Help: "the total amount of computation used by an script", 458 Buckets: []float64{50, 100, 500, 1000, 5000, 10000}, 459 }) 460 461 scriptMemoryUsage := promauto.NewHistogram(prometheus.HistogramOpts{ 462 Namespace: namespaceExecution, 463 Subsystem: subsystemRuntime, 464 Name: "script_memory_usage", 465 Help: "the total amount of memory allocated by a script", 466 Buckets: []float64{100_000, 1_000_000, 10_000_000, 50_000_000, 100_000_000, 500_000_000, 1_000_000_000}, 467 }) 468 469 scriptMemoryEstimate := promauto.NewHistogram(prometheus.HistogramOpts{ 470 Namespace: namespaceExecution, 471 Subsystem: subsystemRuntime, 472 Name: "script_memory_estimate", 473 Help: "the estimated memory used by a script", 474 Buckets: []float64{1_000_000, 10_000_000, 100_000_000, 1_000_000_000, 5_000_000_000, 10_000_000_000, 50_000_000_000, 100_000_000_000}, 475 }) 476 477 scriptMemoryDifference := promauto.NewHistogram(prometheus.HistogramOpts{ 478 Namespace: namespaceExecution, 479 Subsystem: subsystemRuntime, 480 Name: "script_memory_difference", 481 Help: "the difference in actual memory usage and estimate for a script", 482 Buckets: []float64{-1, 0, 10_000_000, 100_000_000, 1_000_000_000}, 483 }) 484 485 chunkDataPackRequestProcessedTotal := promauto.NewCounter(prometheus.CounterOpts{ 486 Namespace: namespaceExecution, 487 Subsystem: subsystemProvider, 488 Name: "chunk_data_packs_requested_total", 489 Help: "the total number of chunk data pack requests processed by provider engine", 490 }) 491 492 chunkDataPackProofSize := promauto.NewHistogram(prometheus.HistogramOpts{ 493 Namespace: namespaceExecution, 494 Subsystem: subsystemIngestion, 495 Name: "chunk_data_pack_proof_size", 496 Help: "the total number bytes used for storing proof part of chunk data pack", 497 Buckets: prometheus.ExponentialBuckets(1000, 2, 16), 498 }) 499 500 chunkDataPackCollectionSize := promauto.NewHistogram(prometheus.HistogramOpts{ 501 Namespace: namespaceExecution, 502 Subsystem: subsystemIngestion, 503 Name: "chunk_data_pack_collection_size", 504 Help: "the total number transactions in the collection", 505 Buckets: prometheus.ExponentialBuckets(1, 2, 10), 506 }) 507 508 blockDataUploadsInProgress := promauto.NewGauge(prometheus.GaugeOpts{ 509 Namespace: namespaceExecution, 510 Subsystem: subsystemBlockDataUploader, 511 Name: "block_data_upload_in_progress", 512 Help: "number of concurrently running Block Data upload operations", 513 }) 514 515 blockDataUploadsDuration := promauto.NewHistogram(prometheus.HistogramOpts{ 516 Namespace: namespaceExecution, 517 Subsystem: subsystemBlockDataUploader, 518 Name: "block_data_upload_duration_ms", 519 Help: "the duration of update upload operation", 520 Buckets: []float64{1, 100, 500, 1000, 2000}, 521 }) 522 523 computationResultUploadedCount := promauto.NewCounter(prometheus.CounterOpts{ 524 Namespace: namespaceExecution, 525 Subsystem: subsystemProvider, 526 Name: "computation_result_uploaded_count", 527 Help: "the total count of computation result uploaded", 528 }) 529 530 computationResultUploadRetriedCount := promauto.NewCounter(prometheus.CounterOpts{ 531 Namespace: namespaceExecution, 532 Subsystem: subsystemProvider, 533 Name: "computation_result_upload_retried_count", 534 Help: "the total count of computation result upload retried", 535 }) 536 537 ec := &ExecutionCollector{ 538 tracer: tracer, 539 540 forestApproxMemorySize: forestApproxMemorySize, 541 forestNumberOfTrees: forestNumberOfTrees, 542 latestTrieRegCount: latestTrieRegCount, 543 latestTrieRegCountDiff: latestTrieRegCountDiff, 544 latestTrieRegSize: latestTrieRegSize, 545 latestTrieRegSizeDiff: latestTrieRegSizeDiff, 546 latestTrieMaxDepthTouched: latestTrieMaxDepthTouched, 547 updated: updatedCount, 548 proofSize: proofSize, 549 updatedValuesNumber: updatedValuesNumber, 550 updatedValuesSize: updatedValuesSize, 551 updatedDuration: updatedDuration, 552 updatedDurationPerValue: updatedDurationPerValue, 553 readValuesNumber: readValuesNumber, 554 readValuesSize: readValuesSize, 555 readDuration: readDuration, 556 readDurationPerValue: readDurationPerValue, 557 blockExecutionTime: blockExecutionTime, 558 blockComputationUsed: blockComputationUsed, 559 blockComputationVector: blockComputationVector, 560 blockCachedPrograms: blockCachedPrograms, 561 blockMemoryUsed: blockMemoryUsed, 562 blockEventCounts: blockEventCounts, 563 blockEventSize: blockEventSize, 564 blockTransactionCounts: blockTransactionCounts, 565 blockCollectionCounts: blockCollectionCounts, 566 collectionExecutionTime: collectionExecutionTime, 567 collectionComputationUsed: collectionComputationUsed, 568 collectionMemoryUsed: collectionMemoryUsed, 569 collectionEventSize: collectionEventSize, 570 collectionEventCounts: collectionEventCounts, 571 collectionNumberOfRegistersTouched: collectionNumberOfRegistersTouched, 572 collectionTotalBytesWrittenToRegisters: collectionTotalBytesWrittenToRegisters, 573 collectionTransactionCounts: collectionTransactionCounts, 574 collectionRequestSent: collectionRequestsSent, 575 collectionRequestRetried: collectionRequestsRetries, 576 transactionParseTime: transactionParseTime, 577 transactionCheckTime: transactionCheckTime, 578 transactionInterpretTime: transactionInterpretTime, 579 transactionExecutionTime: transactionExecutionTime, 580 transactionConflictRetries: transactionConflictRetries, 581 transactionComputationUsed: transactionComputationUsed, 582 transactionNormalizedTimePerComputation: transactionNormalizedTimePerComputation, 583 transactionMemoryEstimate: transactionMemoryEstimate, 584 transactionEmittedEvents: transactionEmittedEvents, 585 transactionEventSize: transactionEventSize, 586 scriptExecutionTime: scriptExecutionTime, 587 scriptComputationUsed: scriptComputationUsed, 588 scriptMemoryUsage: scriptMemoryUsage, 589 scriptMemoryEstimate: scriptMemoryEstimate, 590 scriptMemoryDifference: scriptMemoryDifference, 591 chunkDataPackRequestProcessedTotal: chunkDataPackRequestProcessedTotal, 592 chunkDataPackProofSize: chunkDataPackProofSize, 593 chunkDataPackCollectionSize: chunkDataPackCollectionSize, 594 blockDataUploadsInProgress: blockDataUploadsInProgress, 595 blockDataUploadsDuration: blockDataUploadsDuration, 596 computationResultUploadedCount: computationResultUploadedCount, 597 computationResultUploadRetriedCount: computationResultUploadRetriedCount, 598 totalExecutedBlocksCounter: promauto.NewCounter(prometheus.CounterOpts{ 599 Namespace: namespaceExecution, 600 Subsystem: subsystemRuntime, 601 Name: "total_executed_blocks", 602 Help: "the total number of blocks that have been executed", 603 }), 604 605 totalExecutedCollectionsCounter: promauto.NewCounter(prometheus.CounterOpts{ 606 Namespace: namespaceExecution, 607 Subsystem: subsystemRuntime, 608 Name: "total_executed_collections", 609 Help: "the total number of collections that have been executed", 610 }), 611 612 totalExecutedTransactionsCounter: promauto.NewCounter(prometheus.CounterOpts{ 613 Namespace: namespaceExecution, 614 Subsystem: subsystemRuntime, 615 Name: "total_executed_transactions", 616 Help: "the total number of transactions that have been executed", 617 }), 618 619 totalFailedTransactionsCounter: promauto.NewCounter(prometheus.CounterOpts{ 620 Namespace: namespaceExecution, 621 Subsystem: subsystemRuntime, 622 Name: "total_failed_transactions", 623 Help: "the total number of transactions that has failed when executed", 624 }), 625 626 totalExecutedScriptsCounter: promauto.NewCounter(prometheus.CounterOpts{ 627 Namespace: namespaceExecution, 628 Subsystem: subsystemRuntime, 629 Name: "total_executed_scripts", 630 Help: "the total number of scripts that have been executed", 631 }), 632 633 lastExecutedBlockHeightGauge: promauto.NewGauge(prometheus.GaugeOpts{ 634 Namespace: namespaceExecution, 635 Subsystem: subsystemRuntime, 636 Name: "last_executed_block_height", 637 Help: "the last height that was executed", 638 }), 639 640 lastFinalizedExecutedBlockHeightGauge: promauto.NewGauge(prometheus.GaugeOpts{ 641 Namespace: namespaceExecution, 642 Subsystem: subsystemRuntime, 643 Name: "last_finalized_executed_block_height", 644 Help: "the last height that was finalized and executed", 645 }), 646 647 stateStorageDiskTotal: promauto.NewGauge(prometheus.GaugeOpts{ 648 Namespace: namespaceExecution, 649 Subsystem: subsystemStateStorage, 650 Name: "data_size_bytes", 651 Help: "the execution state size on disk in bytes", 652 }), 653 654 // TODO: remove 655 storageStateCommitment: promauto.NewGauge(prometheus.GaugeOpts{ 656 Namespace: namespaceExecution, 657 Subsystem: subsystemStateStorage, 658 Name: "commitment_size_bytes", 659 Help: "the storage size of a state commitment in bytes", 660 }), 661 662 checkpointSize: promauto.NewGauge(prometheus.GaugeOpts{ 663 Namespace: namespaceExecution, 664 Subsystem: subsystemStateStorage, 665 Name: "checkpoint_size_bytes", 666 Help: "the size of a checkpoint in bytes", 667 }), 668 669 stateSyncActive: promauto.NewGauge(prometheus.GaugeOpts{ 670 Namespace: namespaceExecution, 671 Subsystem: subsystemIngestion, 672 Name: "state_sync_active", 673 Help: "indicates if the state sync is active", 674 }), 675 676 numberOfAccounts: promauto.NewGauge(prometheus.GaugeOpts{ 677 Namespace: namespaceExecution, 678 Subsystem: subsystemRuntime, 679 Name: "number_of_accounts", 680 Help: "the number of existing accounts on the network", 681 }), 682 683 programsCacheMiss: promauto.NewCounter(prometheus.CounterOpts{ 684 Namespace: namespaceExecution, 685 Subsystem: subsystemRuntime, 686 Name: "programs_cache_miss", 687 Help: "the number of times a program was not found in the cache and had to be loaded", 688 }), 689 690 programsCacheHit: promauto.NewCounter(prometheus.CounterOpts{ 691 Namespace: namespaceExecution, 692 Subsystem: subsystemRuntime, 693 Name: "programs_cache_hit", 694 Help: "the number of times a program was found in the cache", 695 }), 696 697 maxCollectionHeightData: counters.NewMonotonousCounter(0), 698 maxCollectionHeight: prometheus.NewGauge(prometheus.GaugeOpts{ 699 Name: "max_collection_height", 700 Namespace: namespaceExecution, 701 Subsystem: subsystemIngestion, 702 Help: "gauge to track the maximum block height of collections received", 703 }), 704 } 705 706 return ec 707 } 708 709 // StartBlockReceivedToExecuted starts a span to trace the duration of a block 710 // from being received for execution to execution being finished 711 func (ec *ExecutionCollector) StartBlockReceivedToExecuted(blockID flow.Identifier) { 712 } 713 714 // FinishBlockReceivedToExecuted finishes a span to trace the duration of a block 715 // from being received for execution to execution being finished 716 func (ec *ExecutionCollector) FinishBlockReceivedToExecuted(blockID flow.Identifier) { 717 } 718 719 // ExecutionBlockExecuted reports execution meta data after executing a block 720 func (ec *ExecutionCollector) ExecutionBlockExecuted( 721 dur time.Duration, 722 stats module.ExecutionResultStats, 723 ) { 724 ec.totalExecutedBlocksCounter.Inc() 725 ec.blockExecutionTime.Observe(float64(dur.Milliseconds())) 726 ec.blockComputationUsed.Observe(float64(stats.ComputationUsed)) 727 ec.blockMemoryUsed.Observe(float64(stats.MemoryUsed)) 728 ec.blockEventCounts.Observe(float64(stats.EventCounts)) 729 ec.blockEventSize.Observe(float64(stats.EventSize)) 730 ec.blockTransactionCounts.Observe(float64(stats.NumberOfTransactions)) 731 ec.blockCollectionCounts.Observe(float64(stats.NumberOfCollections)) 732 } 733 734 // ExecutionCollectionExecuted reports stats for executing a collection 735 func (ec *ExecutionCollector) ExecutionCollectionExecuted( 736 dur time.Duration, 737 stats module.ExecutionResultStats, 738 ) { 739 ec.totalExecutedCollectionsCounter.Inc() 740 ec.collectionExecutionTime.Observe(float64(dur.Milliseconds())) 741 ec.collectionComputationUsed.Observe(float64(stats.ComputationUsed)) 742 ec.collectionMemoryUsed.Observe(float64(stats.MemoryUsed)) 743 ec.collectionEventCounts.Observe(float64(stats.EventCounts)) 744 ec.collectionEventSize.Observe(float64(stats.EventSize)) 745 ec.collectionNumberOfRegistersTouched.Observe(float64(stats.NumberOfRegistersTouched)) 746 ec.collectionTotalBytesWrittenToRegisters.Observe(float64(stats.NumberOfBytesWrittenToRegisters)) 747 ec.collectionTransactionCounts.Observe(float64(stats.NumberOfTransactions)) 748 } 749 750 func (ec *ExecutionCollector) ExecutionBlockExecutionEffortVectorComponent(compKind string, value uint) { 751 ec.blockComputationVector.With(prometheus.Labels{LabelComputationKind: compKind}).Set(float64(value)) 752 } 753 754 func (ec *ExecutionCollector) ExecutionBlockCachedPrograms(programs int) { 755 ec.blockCachedPrograms.Set(float64(programs)) 756 } 757 758 // ExecutionTransactionExecuted reports stats for executing a transaction 759 func (ec *ExecutionCollector) ExecutionTransactionExecuted( 760 dur time.Duration, 761 numConflictRetries int, 762 compUsed uint64, 763 memoryUsed uint64, 764 eventCounts int, 765 eventSize int, 766 failed bool, 767 ) { 768 ec.totalExecutedTransactionsCounter.Inc() 769 ec.transactionExecutionTime.Observe(float64(dur.Milliseconds())) 770 ec.transactionConflictRetries.Observe(float64(numConflictRetries)) 771 ec.transactionComputationUsed.Observe(float64(compUsed)) 772 ec.transactionNormalizedTimePerComputation.Observe( 773 flow.NormalizedExecutionTimePerComputationUnit(dur, compUsed)) 774 ec.transactionMemoryEstimate.Observe(float64(memoryUsed)) 775 ec.transactionEmittedEvents.Observe(float64(eventCounts)) 776 ec.transactionEventSize.Observe(float64(eventSize)) 777 if failed { 778 ec.totalFailedTransactionsCounter.Inc() 779 } 780 } 781 782 // ExecutionChunkDataPackGenerated reports stats on chunk data pack generation 783 func (ec *ExecutionCollector) ExecutionChunkDataPackGenerated(proofSize, numberOfTransactions int) { 784 ec.chunkDataPackProofSize.Observe(float64(proofSize)) 785 ec.chunkDataPackCollectionSize.Observe(float64(numberOfTransactions)) 786 } 787 788 // ScriptExecuted reports the time spent executing a single script 789 func (ec *ExecutionCollector) ExecutionScriptExecuted(dur time.Duration, compUsed, memoryUsed, memoryEstimated uint64) { 790 ec.totalExecutedScriptsCounter.Inc() 791 ec.scriptExecutionTime.Observe(float64(dur.Milliseconds())) 792 ec.scriptComputationUsed.Observe(float64(compUsed)) 793 ec.scriptMemoryUsage.Observe(float64(memoryUsed)) 794 ec.scriptMemoryEstimate.Observe(float64(memoryEstimated)) 795 ec.scriptMemoryDifference.Observe(float64(memoryEstimated) - float64(memoryUsed)) 796 } 797 798 // ExecutionStateStorageDiskTotal reports the total storage size of the execution state on disk in bytes 799 func (ec *ExecutionCollector) ExecutionStateStorageDiskTotal(bytes int64) { 800 ec.stateStorageDiskTotal.Set(float64(bytes)) 801 } 802 803 // ExecutionStorageStateCommitment reports the storage size of a state commitment 804 func (ec *ExecutionCollector) ExecutionStorageStateCommitment(bytes int64) { 805 ec.storageStateCommitment.Set(float64(bytes)) 806 } 807 808 // ExecutionCheckpointSize reports the size of a checkpoint in bytes 809 func (ec *ExecutionCollector) ExecutionCheckpointSize(bytes uint64) { 810 ec.checkpointSize.Set(float64(bytes)) 811 } 812 813 // ExecutionLastExecutedBlockHeight reports last executed block height 814 func (ec *ExecutionCollector) ExecutionLastExecutedBlockHeight(height uint64) { 815 ec.lastExecutedBlockHeightGauge.Set(float64(height)) 816 } 817 818 // ExecutionLastFinalizedExecutedBlockHeight reports last finalized executed block height 819 func (ec *ExecutionCollector) ExecutionLastFinalizedExecutedBlockHeight(height uint64) { 820 ec.lastFinalizedExecutedBlockHeightGauge.Set(float64(height)) 821 } 822 823 // ForestApproxMemorySize records approximate memory usage of forest (all in-memory trees) 824 func (ec *ExecutionCollector) ForestApproxMemorySize(bytes uint64) { 825 ec.forestApproxMemorySize.Set(float64(bytes)) 826 } 827 828 // ForestNumberOfTrees current number of trees in a forest (in memory) 829 func (ec *ExecutionCollector) ForestNumberOfTrees(number uint64) { 830 ec.forestNumberOfTrees.Set(float64(number)) 831 } 832 833 // LatestTrieRegCount records the number of unique register allocated (the lastest created trie) 834 func (ec *ExecutionCollector) LatestTrieRegCount(number uint64) { 835 ec.latestTrieRegCount.Set(float64(number)) 836 } 837 838 // LatestTrieRegCountDiff records the difference between the number of unique register allocated of the latest created trie and parent trie 839 func (ec *ExecutionCollector) LatestTrieRegCountDiff(number int64) { 840 ec.latestTrieRegCountDiff.Set(float64(number)) 841 } 842 843 // LatestTrieRegSize records the size of unique register allocated (the lastest created trie) 844 func (ec *ExecutionCollector) LatestTrieRegSize(size uint64) { 845 ec.latestTrieRegSize.Set(float64(size)) 846 } 847 848 // LatestTrieRegSizeDiff records the difference between the size of unique register allocated of the latest created trie and parent trie 849 func (ec *ExecutionCollector) LatestTrieRegSizeDiff(size int64) { 850 ec.latestTrieRegSizeDiff.Set(float64(size)) 851 } 852 853 // LatestTrieMaxDepthTouched records the maximum depth touched of the last created trie 854 func (ec *ExecutionCollector) LatestTrieMaxDepthTouched(maxDepth uint16) { 855 ec.latestTrieMaxDepthTouched.Set(float64(maxDepth)) 856 } 857 858 // UpdateCount increase a counter of performed updates 859 func (ec *ExecutionCollector) UpdateCount() { 860 ec.updated.Inc() 861 } 862 863 // ProofSize records a proof size 864 func (ec *ExecutionCollector) ProofSize(bytes uint32) { 865 ec.proofSize.Set(float64(bytes)) 866 } 867 868 // UpdateValuesNumber accumulates number of updated values 869 func (ec *ExecutionCollector) UpdateValuesNumber(number uint64) { 870 ec.updatedValuesNumber.Add(float64(number)) 871 } 872 873 // UpdateValuesSize total size (in bytes) of updates values 874 func (ec *ExecutionCollector) UpdateValuesSize(bytes uint64) { 875 ec.updatedValuesSize.Set(float64(bytes)) 876 } 877 878 // UpdateDuration records absolute time for the update of a trie 879 func (ec *ExecutionCollector) UpdateDuration(duration time.Duration) { 880 ec.updatedDuration.Observe(duration.Seconds()) 881 } 882 883 // UpdateDurationPerItem records update time for single value (total duration / number of updated values) 884 func (ec *ExecutionCollector) UpdateDurationPerItem(duration time.Duration) { 885 ec.updatedDurationPerValue.Observe(duration.Seconds()) 886 } 887 888 // ReadValuesNumber accumulates number of read values 889 func (ec *ExecutionCollector) ReadValuesNumber(number uint64) { 890 ec.readValuesNumber.Add(float64(number)) 891 } 892 893 // ReadValuesSize total size (in bytes) of read values 894 func (ec *ExecutionCollector) ReadValuesSize(bytes uint64) { 895 ec.readValuesSize.Set(float64(bytes)) 896 } 897 898 // ReadDuration records absolute time for the read from a trie 899 func (ec *ExecutionCollector) ReadDuration(duration time.Duration) { 900 ec.readDuration.Observe(duration.Seconds()) 901 } 902 903 // ReadDurationPerItem records read time for single value (total duration / number of read values) 904 func (ec *ExecutionCollector) ReadDurationPerItem(duration time.Duration) { 905 ec.readDurationPerValue.Observe(duration.Seconds()) 906 } 907 908 func (ec *ExecutionCollector) ExecutionCollectionRequestSent() { 909 ec.collectionRequestSent.Inc() 910 } 911 912 func (ec *ExecutionCollector) ExecutionCollectionRequestRetried() { 913 ec.collectionRequestRetried.Inc() 914 } 915 916 func (ec *ExecutionCollector) ExecutionBlockDataUploadStarted() { 917 ec.blockDataUploadsInProgress.Inc() 918 } 919 920 func (ec *ExecutionCollector) ExecutionBlockDataUploadFinished(dur time.Duration) { 921 ec.blockDataUploadsInProgress.Dec() 922 ec.blockDataUploadsDuration.Observe(float64(dur.Milliseconds())) 923 } 924 925 // TransactionParsed reports the time spent parsing a single transaction 926 func (ec *ExecutionCollector) RuntimeTransactionParsed(dur time.Duration) { 927 ec.transactionParseTime.Observe(float64(dur)) 928 } 929 930 // TransactionChecked reports the time spent checking a single transaction 931 func (ec *ExecutionCollector) RuntimeTransactionChecked(dur time.Duration) { 932 ec.transactionCheckTime.Observe(float64(dur)) 933 } 934 935 // TransactionInterpreted reports the time spent interpreting a single transaction 936 func (ec *ExecutionCollector) RuntimeTransactionInterpreted(dur time.Duration) { 937 ec.transactionInterpretTime.Observe(float64(dur)) 938 } 939 940 // ChunkDataPackRequestProcessed is executed every time a chunk data pack request is picked up for processing at execution node. 941 // It increases the request processed counter by one. 942 func (ec *ExecutionCollector) ChunkDataPackRequestProcessed() { 943 ec.chunkDataPackRequestProcessedTotal.Inc() 944 } 945 946 func (ec *ExecutionCollector) ExecutionSync(syncing bool) { 947 if syncing { 948 ec.stateSyncActive.Set(float64(1)) 949 return 950 } 951 ec.stateSyncActive.Set(float64(0)) 952 } 953 954 func (ec *ExecutionCollector) RuntimeSetNumberOfAccounts(count uint64) { 955 ec.numberOfAccounts.Set(float64(count)) 956 } 957 958 func (ec *ExecutionCollector) RuntimeTransactionProgramsCacheMiss() { 959 ec.programsCacheMiss.Inc() 960 } 961 962 func (ec *ExecutionCollector) RuntimeTransactionProgramsCacheHit() { 963 ec.programsCacheHit.Inc() 964 } 965 966 func (ec *ExecutionCollector) UpdateCollectionMaxHeight(height uint64) { 967 updated := ec.maxCollectionHeightData.Set(height) 968 if updated { 969 ec.maxCollectionHeight.Set(float64(height)) 970 } 971 } 972 973 func (ec *ExecutionCollector) ExecutionComputationResultUploaded() { 974 ec.computationResultUploadedCount.Inc() 975 } 976 977 func (ec *ExecutionCollector) ExecutionComputationResultUploadRetried() { 978 ec.computationResultUploadRetriedCount.Inc() 979 }