github.com/caos/orbos@v1.5.14-0.20221103111702-e6cd0cea7ad4/internal/operator/boom/application/applications/metricspersisting/helm/rules.go (about) 1 package helm 2 3 import ( 4 "gopkg.in/yaml.v3" 5 ) 6 7 const ( 8 cockroachStatefulsetName = "cockroachdb" 9 cockroachServiceName = "cockroachdb-public" 10 cockroachNamespace = "caos-zitadel" 11 caosToolingNamespace = "caos-system" 12 nodeExporterService = "node-exporter" 13 kubeStateMetricsService = "kube-state-metrics" 14 ) 15 16 func GetDefaultRules(labels map[string]string) (*AdditionalPrometheusRules, error) { 17 rulesStr := `name: node-exporter.rules 18 groups: 19 - name: node-exporter.rules 20 rules: 21 - expr: |- 22 count without (cpu) ( 23 count without (mode) ( 24 node_cpu_seconds_total{job="` + nodeExporterService + `"} 25 ) 26 ) 27 record: instance:node_num_cpu:sum 28 - expr: |- 29 1 - avg without (cpu, mode) ( 30 rate(node_cpu_seconds_total{job="` + nodeExporterService + `", mode="idle"}[1m]) 31 ) 32 record: instance:node_cpu_utilisation:rate1m 33 - expr: |- 34 ( 35 node_load1{job="` + nodeExporterService + `"} 36 / 37 instance:node_num_cpu:sum{job="` + nodeExporterService + `"} 38 ) 39 record: instance:node_load1_per_cpu:ratio 40 - expr: |- 41 1 - ( 42 node_memory_MemAvailable_bytes{job="` + nodeExporterService + `"} 43 / 44 node_memory_MemTotal_bytes{job="` + nodeExporterService + `"} 45 ) 46 record: instance:node_memory_utilisation:ratio 47 - expr: rate(node_vmstat_pgmajfault{job="` + nodeExporterService + `"}[1m]) 48 record: instance:node_vmstat_pgmajfault:rate1m 49 - expr: rate(node_disk_io_time_seconds_total{job="` + nodeExporterService + `", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) 50 record: instance_device:node_disk_io_time_seconds:rate1m 51 - expr: rate(node_disk_io_time_weighted_seconds_total{job="` + nodeExporterService + `", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+"}[1m]) 52 record: instance_device:node_disk_io_time_weighted_seconds:rate1m 53 - expr: |- 54 sum without (device) ( 55 rate(node_network_receive_bytes_total{job="` + nodeExporterService + `", device!="lo"}[1m]) 56 ) 57 record: instance:node_network_receive_bytes_excluding_lo:rate1m 58 - expr: |- 59 sum without (device) ( 60 rate(node_network_transmit_bytes_total{job="` + nodeExporterService + `", device!="lo"}[1m]) 61 ) 62 record: instance:node_network_transmit_bytes_excluding_lo:rate1m 63 - expr: |- 64 sum without (device) ( 65 rate(node_network_receive_drop_total{job="` + nodeExporterService + `", device!="lo"}[1m]) 66 ) 67 record: instance:node_network_receive_drop_excluding_lo:rate1m 68 - expr: |- 69 sum without (device) ( 70 rate(node_network_transmit_drop_total{job="` + nodeExporterService + `", device!="lo"}[1m]) 71 ) 72 record: instance:node_network_transmit_drop_excluding_lo:rate1m 73 - name: node.rules 74 rules: 75 - expr: sum(min(kube_pod_info) by (node)) 76 record: ':kube_pod_info_node_count:' 77 - expr: max(label_replace(kube_pod_info{job="` + kubeStateMetricsService + `"}, "pod", "$1", "pod", "(.*)")) by (node, namespace, pod) 78 record: 'node_namespace_pod:kube_pod_info:' 79 - expr: |- 80 count by (node) (sum by (node, cpu) ( 81 node_cpu_seconds_total{job="` + nodeExporterService + `"} 82 * on (namespace, pod) group_left(node) 83 node_namespace_pod:kube_pod_info: 84 )) 85 record: node:node_num_cpu:sum 86 - expr: |- 87 sum( 88 node_memory_MemAvailable_bytes{job="` + nodeExporterService + `"} or 89 ( 90 node_memory_Buffers_bytes{job="` + nodeExporterService + `"} + 91 node_memory_Cached_bytes{job="` + nodeExporterService + `"} + 92 node_memory_MemFree_bytes{job="` + nodeExporterService + `"} + 93 node_memory_Slab_bytes{job="` + nodeExporterService + `"} 94 ) 95 ) 96 record: :node_memory_MemAvailable_bytes:sum 97 - name: caos.rules 98 rules: 99 - expr: dist_node_boot_time_seconds 100 record: caos_node_boot_time_seconds 101 - expr: floor(avg_over_time(dist_systemd_unit_active[5m])+0.2) 102 record: caos_systemd_unit_active 103 - expr: min(min_over_time(caos_systemd_unit_active[5m])) by (instance) 104 record: caos_systemd_ryg 105 - expr: avg(max_over_time(caos_probe{type="Upstream",name!="httpingress"}[1m])) by (name) 106 record: caos_upstream_probe_ryg 107 - expr: max_over_time(caos_probe{type="VIP"}[1m]) 108 record: caos_vip_probe_ryg 109 - expr: sum(1 - avg(rate(dist_node_cpu_seconds_total[5m]))) 110 record: caos_cluster_cpu_utilisation_5m 111 - expr: 100 - (avg by (instance) (irate(dist_node_cpu_seconds_total[5m])) * 100) 112 record: caos_node_cpu_utilisation_5m 113 - expr: (clamp_max(clamp_min(100-caos_node_cpu_utilisation_5m, 10),20)-10)/10 114 record: caos_node_cpu_ryg 115 - expr: |- 116 sum by (instance) (100 - 117 ( 118 dist_node_memory_MemAvailable_bytes 119 / 120 dist_node_memory_MemTotal_bytes 121 * 100 122 )) 123 record: caos_node_memory_utilisation 124 - expr: (clamp_max(clamp_min(100-caos_node_memory_utilisation, 10),20)-10)/10 125 record: caos_node_memory_ryg 126 - expr: |- 127 100 - ( 128 min by (instance) (dist_node_filesystem_avail_bytes) 129 / min by (instance) (dist_node_filesystem_size_bytes) 130 * 100) 131 record: caos_node_disk_utilisation 132 - expr: dist_kube_node_status_condition 133 record: caos_node_ready 134 - expr: min_over_time(caos_node_ready[5m]) 135 record: caos_k8s_node_ryg 136 - expr: dist_etcd_server_has_leader or on(instance) up{job="caos_remote_etcd"} 137 record: caos_etcd_server_has_leader_and_is_up 138 - expr: min_over_time(caos_etcd_server_has_leader_and_is_up[5m]) 139 record: caos_etcd_ryg 140 - expr: |- 141 clamp_max( 142 clamp_min( 143 ( 144 max_over_time(dist_kube_deployment_status_replicas_available{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 145 dist_kube_deployment_spec_replicas{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} or 146 max_over_time(dist_kube_statefulset_status_replicas_ready{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 147 dist_kube_statefulset_replicas{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} or 148 max_over_time(dist_kube_daemonset_status_number_available{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 149 dist_kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} 150 ) + 151 1, 152 0 153 ), 154 1 155 ) 156 record: caos_ready_pods_ryg 157 - expr: |- 158 clamp_max( 159 clamp_min( 160 ( 161 max_over_time(dist_kube_deployment_status_replicas{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 162 dist_kube_deployment_spec_replicas{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} or 163 max_over_time(dist_kube_statefulset_status_replicas_current{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 164 dist_kube_statefulset_replicas{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} or 165 max_over_time(dist_kube_daemonset_status_current_number_scheduled{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}[5m]) - 166 dist_kube_daemonset_status_desired_number_scheduled{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"} 167 ) + 168 1, 169 0 170 ), 171 1 172 ) 173 record: caos_scheduled_pods_ryg 174 - expr: |- 175 sum(dist_kube_deployment_spec_replicas) + sum(dist_kube_statefulset_replicas) + sum(dist_kube_daemonset_status_desired_number_scheduled) 176 record: caos_desired_pods 177 - expr: |- 178 sum(dist_kube_deployment_status_replicas) + sum(dist_kube_statefulset_status_replicas_current) + sum(dist_kube_daemonset_status_current_number_scheduled) 179 record: caos_scheduled_pods 180 - expr: |- 181 sum(dist_kube_deployment_status_replicas_available) + sum(dist_kube_statefulset_status_replicas_ready) + sum(dist_kube_daemonset_status_number_available) 182 record: caos_ready_pods 183 184 # Cockroachdb aggregation rules 185 - record: node:capacity 186 expr: sum without(store) (capacity{job="` + cockroachServiceName + `"}) 187 - record: cluster:capacity 188 expr: sum without(instance) (node:capacity{job="` + cockroachServiceName + `"}) 189 - record: node:capacity_available 190 expr: sum without(store) (capacity_available{job="` + cockroachServiceName + `"}) 191 - record: cluster:capacity_available 192 expr: sum without(instance) (node:capacity_available{job="` + cockroachServiceName + `"}) 193 - record: capacity_available:ratio 194 expr: capacity_available{job="` + cockroachServiceName + `"} / capacity{job="` + cockroachServiceName + `"} 195 - record: node:capacity_available:ratio 196 expr: node:capacity_available{job="` + cockroachServiceName + `"} / node:capacity{job="` + cockroachServiceName + `"} 197 - record: cluster:capacity_available:ratio 198 expr: cluster:capacity_available{job="` + cockroachServiceName + `"} / cluster:capacity{job="` + cockroachServiceName + `"} 199 # Histogram rules: these are fairly expensive to compute live, so we precompute a few percetiles. 200 - record: txn_durations_bucket:rate1m 201 expr: rate(txn_durations_bucket{job="` + cockroachServiceName + `"}[1m]) 202 - record: txn_durations:rate1m:quantile_50 203 expr: histogram_quantile(0.5, txn_durations_bucket:rate1m) 204 - record: txn_durations:rate1m:quantile_75 205 expr: histogram_quantile(0.75, txn_durations_bucket:rate1m) 206 - record: txn_durations:rate1m:quantile_90 207 expr: histogram_quantile(0.9, txn_durations_bucket:rate1m) 208 - record: txn_durations:rate1m:quantile_95 209 expr: histogram_quantile(0.95, txn_durations_bucket:rate1m) 210 - record: txn_durations:rate1m:quantile_99 211 expr: histogram_quantile(0.99, txn_durations_bucket:rate1m) 212 - record: exec_latency_bucket:rate1m 213 expr: rate(exec_latency_bucket{job="` + cockroachServiceName + `"}[1m]) 214 - record: exec_latency:rate1m:quantile_50 215 expr: histogram_quantile(0.5, exec_latency_bucket:rate1m) 216 - record: exec_latency:rate1m:quantile_75 217 expr: histogram_quantile(0.75, exec_latency_bucket:rate1m) 218 - record: exec_latency:rate1m:quantile_90 219 expr: histogram_quantile(0.9, exec_latency_bucket:rate1m) 220 - record: exec_latency:rate1m:quantile_95 221 expr: histogram_quantile(0.95, exec_latency_bucket:rate1m) 222 - record: exec_latency:rate1m:quantile_99 223 expr: histogram_quantile(0.99, exec_latency_bucket:rate1m) 224 - record: round_trip_latency_bucket:rate1m 225 expr: rate(round_trip_latency_bucket{job="` + cockroachServiceName + `"}[1m]) 226 - record: round_trip_latency:rate1m:quantile_50 227 expr: histogram_quantile(0.5, round_trip_latency_bucket:rate1m) 228 - record: round_trip_latency:rate1m:quantile_75 229 expr: histogram_quantile(0.75, round_trip_latency_bucket:rate1m) 230 - record: round_trip_latency:rate1m:quantile_90 231 expr: histogram_quantile(0.9, round_trip_latency_bucket:rate1m) 232 - record: round_trip_latency:rate1m:quantile_95 233 expr: histogram_quantile(0.95, round_trip_latency_bucket:rate1m) 234 - record: round_trip_latency:rate1m:quantile_99 235 expr: histogram_quantile(0.99, round_trip_latency_bucket:rate1m) 236 - record: sql_exec_latency_bucket:rate1m 237 expr: rate(sql_exec_latency_bucket{job="` + cockroachServiceName + `"}[1m]) 238 - record: sql_exec_latency:rate1m:quantile_50 239 expr: histogram_quantile(0.5, sql_exec_latency_bucket:rate1m) 240 - record: sql_exec_latency:rate1m:quantile_75 241 expr: histogram_quantile(0.75, sql_exec_latency_bucket:rate1m) 242 - record: sql_exec_latency:rate1m:quantile_90 243 expr: histogram_quantile(0.9, sql_exec_latency_bucket:rate1m) 244 - record: sql_exec_latency:rate1m:quantile_95 245 expr: histogram_quantile(0.95, sql_exec_latency_bucket:rate1m) 246 - record: sql_exec_latency:rate1m:quantile_99 247 expr: histogram_quantile(0.99, sql_exec_latency_bucket:rate1m) 248 - record: raft_process_logcommit_latency_bucket:rate1m 249 expr: rate(raft_process_logcommit_latency_bucket{job="` + cockroachServiceName + `"}[1m]) 250 - record: raft_process_logcommit_latency:rate1m:quantile_50 251 expr: histogram_quantile(0.5, raft_process_logcommit_latency_bucket:rate1m) 252 - record: raft_process_logcommit_latency:rate1m:quantile_75 253 expr: histogram_quantile(0.75, raft_process_logcommit_latency_bucket:rate1m) 254 - record: raft_process_logcommit_latency:rate1m:quantile_90 255 expr: histogram_quantile(0.9, raft_process_logcommit_latency_bucket:rate1m) 256 - record: raft_process_logcommit_latency:rate1m:quantile_95 257 expr: histogram_quantile(0.95, raft_process_logcommit_latency_bucket:rate1m) 258 - record: raft_process_logcommit_latency:rate1m:quantile_99 259 expr: histogram_quantile(0.99, raft_process_logcommit_latency_bucket:rate1m) 260 - record: raft_process_commandcommit_latency_bucket:rate1m 261 expr: rate(raft_process_commandcommit_latency_bucket{job="cockroachdb"}[1m]) 262 - record: raft_process_commandcommit_latency:rate1m:quantile_50 263 expr: histogram_quantile(0.5, raft_process_commandcommit_latency_bucket:rate1m) 264 - record: raft_process_commandcommit_latency:rate1m:quantile_75 265 expr: histogram_quantile(0.75, raft_process_commandcommit_latency_bucket:rate1m) 266 - record: raft_process_commandcommit_latency:rate1m:quantile_90 267 expr: histogram_quantile(0.9, raft_process_commandcommit_latency_bucket:rate1m) 268 - record: raft_process_commandcommit_latency:rate1m:quantile_95 269 expr: histogram_quantile(0.95, raft_process_commandcommit_latency_bucket:rate1m) 270 - record: raft_process_commandcommit_latency:rate1m:quantile_99 271 expr: histogram_quantile(0.99, raft_process_commandcommit_latency_bucket:rate1m) 272 273 # ZITADEL CockroachDB Runtime 274 - record: cr_runtime_pod_flapping 275 expr: resets(sys_uptime{job="` + cockroachServiceName + `"}[10m]) 276 - record: cr_runtime_pod_flapping_ryg 277 expr: clamp_min(1 - cr_runtime_pod_flapping, 0) 278 - record: cr_runtime_high_open_fd_count 279 expr: sys_fd_open{job="` + cockroachServiceName + `"} / sys_fd_softlimit{job="` + cockroachServiceName + `"} 280 - record: cr_runtime_high_open_fd_count_ryg 281 expr: (1 - (clamp_max(clamp_min(floor(cr_runtime_high_open_fd_count * 100), 20), 30) - 20) / 10) 282 - record: caos_cr_runtime_pods_ryg 283 expr: |- 284 cr_runtime_pod_flapping_ryg 285 * cr_runtime_high_open_fd_count_ryg 286 287 - record: cr_runtime_version_mismatches 288 expr: count by(cluster) (count_values by(tag, cluster) ("version", build_timestamp{job="` + cockroachServiceName + `"})) 289 - record: cr_runtime_version_mismatches_ryg 290 expr: clamp_min(2 - cr_runtime_version_mismatches, 0) 291 - record: caos_cr_runtime_cluster_ryg 292 expr: |- 293 min(caos_ready_pods_ryg{controller="` + cockroachStatefulsetName + `",namespace="` + cockroachNamespace + `"}) 294 * min(caos_scheduled_pods_ryg{controller="` + cockroachStatefulsetName + `",namespace="` + cockroachNamespace + `"}) 295 * cr_runtime_version_mismatches 296 - record: caos_security_certificate_expiration_ca 297 expr: (security_certificate_expiration_ca{job="` + cockroachServiceName + `"} - time()) / 86400 298 - record: caos_security_certificate_expiration_node 299 expr: (security_certificate_expiration_node{job="` + cockroachServiceName + `"} - time()) / 86400 300 301 302 # ZITADEL CockroachDB Capacity 303 - record: node:capacity 304 expr: sum without(store) (capacity{job="` + cockroachServiceName + `"}) 305 - record: cluster:capacity 306 expr: sum without(instance) (node:capacity{job="` + cockroachServiceName + `"}) 307 - record: node:capacity_available 308 expr: sum without(store) (capacity_available{job="` + cockroachServiceName + `"}) 309 - record: cluster:capacity_available 310 expr: sum without(instance) (node:capacity_available{job="` + cockroachServiceName + `"}) 311 - record: capacity_available:ratio 312 expr: capacity_available{job="` + cockroachServiceName + `"} / capacity{job="` + cockroachServiceName + `"} 313 - record: node:capacity_available:ratio 314 expr: node:capacity_available{job="` + cockroachServiceName + `"} / node:capacity{job="` + cockroachServiceName + `"} 315 - record: cluster:capacity_available:ratio 316 expr: cluster:capacity_available{job="` + cockroachServiceName + `"} / cluster:capacity{job="` + cockroachServiceName + `"} 317 318 - record: caos_cr_capacity_store_ryg 319 expr: (clamp_max(clamp_min(100 * capacity_available:ratio{job="` + cockroachServiceName + `"}, 15), 30)-15) /15 320 - record: caos_cr_capacity_cluster_ryg 321 expr: (clamp_max(clamp_min(100 * cluster:capacity_available:ratio{job="` + cockroachServiceName + `"}, 20), 30) - 20) /10 322 323 # ZITADEL CockroachDB Replicas 324 - record: cr_replicas_clock_offset_near_max_ryg 325 expr: 1 - (clamp_max(clamp_min(abs(clock_offset_meannanos), 200 * 1000 * 1000), 300 * 1000 * 1000) - 200 * 1000 * 1000) / (100 * 1000 * 1000) 326 - record: cr_ca_certificate_expires_soon_ryg 327 expr: (clamp_max(clamp_min(security_certificate_expiration_ca{job="` + cockroachServiceName + `"}- time(), 86400 * 180), 86400 * 366) - 86400 * 180) / (86400 * 186) 328 # - record: cr_client_ca_certificate_expires_soon_ryg 329 # expr: (clamp_max(clamp_min(security_certificate_expiration_client_ca{job="` + cockroachServiceName + `"}- time(), 86400 * 180), 86400 * 366) - 86400 * 180) / (86400 * 186) 330 - record: cr_node_certificate_expires_soon_ryg 331 expr: (clamp_max(clamp_min(security_certificate_expiration_node{job="` + cockroachServiceName + `"}- time(), 86400 * 180), 86400 * 366) - 86400 * 180) / (86400 * 186) 332 # - record: cr_node_client_certificate_expires_soon_ryg 333 # expr: (clamp_max(clamp_min(cr_node_client_certificate_expires_soon{job="` + cockroachServiceName + `"}- time(), 86400 * 180), 86400 * 366) - 86400 * 180) / (86400 * 186) 334 - record: caos_cr_replicas_nodes_ryg 335 expr: |- 336 cr_replicas_clock_offset_near_max_ryg 337 * cr_ca_certificate_expires_soon_ryg 338 * cr_node_certificate_expires_soon_ryg 339 # * cr_client_ca_certificate_expires_soon_ryg 340 # * cr_node_client_certificate_expires_soon_ryg 341 - record: caos_cr_replicas_stores_ryg 342 expr: |- 343 clamp_min(1 - requests_slow_latch{job="` + cockroachServiceName + `"}, 0) 344 * clamp_min(1 - requests_slow_lease{job="` + cockroachServiceName + `"}, 0) 345 * clamp_min(1 - requests_slow_raft{job="` + cockroachServiceName + `"}, 0) 346 347 # Cluster RYG 348 - record: caos_orb_ryg 349 expr: |- 350 # Platform Monitoring 351 min(caos_node_cpu_ryg) 352 * min(caos_systemd_ryg) 353 * min(caos_vip_probe_ryg) 354 * min(caos_upstream_probe_ryg) 355 * min(caos_node_memory_ryg) 356 * min(caos_k8s_node_ryg) 357 * avg(caos_etcd_ryg) 358 * min(caos_ready_pods_ryg{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}) 359 * min(caos_scheduled_pods_ryg{namespace=~"(kube-system|` + caosToolingNamespace + `|` + cockroachNamespace + `)"}) 360 # ZITADEL Monitoring 361 * min(caos_cr_runtime_cluster_ryg) or on() 1 - avg(up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 362 * min(caos_cr_runtime_pods_ryg) or on() 1 - (up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 363 * min(caos_cr_replicas_nodes_ryg) or on() 1 - (up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 364 * min(caos_cr_replicas_stores_ryg) or on() 1 - (up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 365 * min(caos_cr_capacity_cluster_ryg) or on() 1 - (up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 366 * min(caos_cr_capacity_store_ryg) or on() 1 - (up{endpoint="http",job="` + cockroachServiceName + `",namespace="` + cockroachNamespace + `",pod="` + cockroachStatefulsetName + `-0",service="` + cockroachServiceName + `"} or on() vector(0)) 367 ` 368 369 struc := &AdditionalPrometheusRules{ 370 AdditionalLabels: labels, 371 } 372 rulesByte := []byte(rulesStr) 373 if err := yaml.Unmarshal(rulesByte, struc); err != nil { 374 return nil, err 375 } 376 return struc, nil 377 }