{ "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "test-cluster", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "4.6.3" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "${DS_TEST-CLUSTER}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "hideControls": false, "id": null, "links": [ { "icon": "doc", "includeVars": true, "keepTime": true, "tags": [], "targetBlank": true, "title": "Report", "tooltip": "Open a pdf report for the current dashboard", "type": "link", "url": "http://172.16.10.49:8686/api/report/test-cluster-tikv?apitoken=eyJrIjoiQkU2bmFQdjZMeTRmWVJ3UFo0bmV3Ym5zRG04Y2hmMHYiLCJuIjoiZ3JhZmFuYV9hcGlrZXkiLCJpZCI6MX0=" } ], "refresh": "1m", "rows": [ { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 5, "grid": {}, "id": 56, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_size_bytes) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "store size", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 5, "grid": {}, "id": 1706, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(tikv_store_size_bytes{type=\"available\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "available size", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 5, "grid": {}, "id": 1707, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(tikv_store_size_bytes{type=\"capacity\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "capacity size", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1708, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total[1m])) by (instance,job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "cpu", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1709, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(process_resident_memory_bytes{job=~\"tikv.*\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "memory", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1710, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(node_disk_io_time_ms[1m]) / 1000", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}} - {{device}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "io utilization", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1711, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=\"wal_file_bytes\"}[1m])) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}-write", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[1m])) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}-read", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MBps", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1713, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{type!=\"kv_gc\"}[1m])) by (job,type)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}} - {{type}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1712, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_fail_total{type!=\"kv_gc\"}[1m])) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}-grpc-msg-fail", "refId": "A", "step": 10 }, { "expr": "sum(delta(tikv_pd_heartbeat_message_total{type=\"noop\"}[1m])) by (job) < 1", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-pd-heartbeat", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errps", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1715, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "total", "lines": false } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"leader\"}) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 10 }, { "expr": "delta(tikv_pd_heartbeat_tick_total{type=\"leader\"}[30s]) < -10", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "leader", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1714, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"region\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "region", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Cluster", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1584, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_too_busy_total[1m])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "scheduler-{{job}}", "metric": "", "refId": "A", "step": 4 }, { "expr": "sum(rate(tikv_channel_full_total[1m])) by (job, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "channelfull-{{job}}-{{type}}", "metric": "", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_coprocessor_request_error{type='full'}[1m])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "coprocessor-{{job}}", "metric": "", "refId": "C", "step": 4 }, { "expr": "avg(tikv_engine_write_stall{type=\"write_stall_percentile99\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "stall-{{job}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "server is busy", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 0 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "10s", "now" ] }, "reducer": { "params": [], "type": "max" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "10s", "handler": 1, "message": "TiKV server report failures", "name": "server report failures alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 18, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_server_report_failure_msg_total[1m])) by (type,instance,job,store_id)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} - {{type}} - to - {{store_id}}", "metric": "tikv_server_raft_store_msg_total", "refId": "A", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0 } ], "timeFrom": null, "timeShift": null, "title": "server report failures", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1718, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_storage_engine_async_request_total{status!~\"success|all\"}[1m])) by (job, status)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{status}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raftstore error", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1719, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_stage_total{stage=~\"snapshot_err|prepare_write_err\"}[1m])) by (job, stage)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{stage}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler error", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1720, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_request_error[1m])) by (job, reason)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{reason}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor error", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1721, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_fail_total[1m])) by (job, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}-{{type}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor error", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1722, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "total", "lines": false } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(delta(tikv_pd_heartbeat_tick_total{type=\"leader\"}[1m])) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "leader drop", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1723, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "total", "lines": false } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_raftstore_leader_missing) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "leader missing", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Errors", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 34, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "total", "lines": false } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"leader\"}) by (job)", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 10 }, { "expr": "delta(tikv_pd_heartbeat_tick_total{type=\"leader\"}[30s]) < -10", "format": "time_series", "hide": true, "intervalFactor": 2, "legendFormat": "", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "leader", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 37, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_pd_heartbeat_tick_total{type=\"region\"}) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "region", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 3, "grid": {}, "id": 33, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_size_bytes) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "cf size", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 5, "grid": {}, "id": 1705, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_size_bytes) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "store size", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 0 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "datasourceId": 1, "model": { "expr": "sum(rate(tikv_channel_full_total[1m])) by (job, type)", "intervalFactor": 2, "legendFormat": "{{job}} - {{type}}", "metric": "", "refId": "A", "step": 10 }, "params": [ "A", "10s", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "10s", "handler": 1, "message": "TiKV channel full", "name": "TiKV channel full alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 3, "grid": {}, "id": 22, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_channel_full_total[1m])) by (job, type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} - {{type}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0 } ], "timeFrom": null, "timeShift": null, "title": "channel full", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1717, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_server_report_failure_msg_total[1m])) by (type,instance,job,store_id)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}} - {{type}} - to - {{store_id}}", "metric": "tikv_server_raft_store_msg_total", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "server report failures", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 57, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_region_written_keys_sum[1m])) by (job) / sum(rate(tikv_region_written_keys_count[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_region_written_keys_bucket", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "region average written keys", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 58, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_region_written_bytes_sum[1m])) by (job) / sum(rate(tikv_region_written_bytes_count[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_regi", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "region average written bytes", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 75, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_region_written_keys_count[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_region_written_keys_bucket", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "active written leaders", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 1073741824 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "1m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "approximate region size alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1481, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "B", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_region_size_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_raftstore_region_size_sum[1m])) / sum(rate(tikv_raftstore_region_size_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "approximate region size", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Server", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 31, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " 99%", "metric": "", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "apply log duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 32, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": " {{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "apply log duration per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " 99%", "metric": "", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_raftstore_append_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count[1m]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "append log duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 40, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}} ", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "append log duration per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Raft IO", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 5, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_raft_ready_handled_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_raftstore_raft_ready_handled_total", "refId": "A", "step": 4 }, { "expr": "sum(rate(tikv_raftstore_raft_process_duration_secs_count{type=\"ready\"}[1m]))", "intervalFactor": 2, "legendFormat": "count", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "ready handled", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 118, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "C", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1 } ], "timeFrom": null, "timeShift": null, "title": "process ready duration per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1165, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "C", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1 } ], "timeFrom": null, "timeShift": null, "title": "process tick duration per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Raft Process", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1615, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_raft_sent_message_total[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "sent messages per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1616, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_server_raft_message_flush_total[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_server_raft_message_flush_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "flush messages per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 106, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_server_raft_message_recv_total[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "recv messages per server", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 11, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_raft_sent_message_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "messages", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 25, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_raft_sent_message_total{type=\"vote\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "vote", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1309, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_raft_dropped_message_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft dropped messages", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Raft Message", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 108, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_proposal_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft proposals per ready", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|normal|read_index\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_raftstore_proposal_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft read/write proposals", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 119, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"local_read|read_index\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft read proposals per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 120, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_proposal_total{type=\"normal\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_raftstore_proposal_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft write proposals per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_raftstore_request_wait_time_duration_secs_sum[1m])) / sum(rate(tikv_raftstore_request_wait_time_duration_secs_count[1m]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "propose wait duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 42, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "propose wait duration per server", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "id": 1975, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(rate(tikv_raftstore_propose_log_size_sum[1m])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft log speed", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "bytes/s", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Raft Propose", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 76, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_proposal_total{type=~\"conf_change|transfer_leader\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_raftstore_proposal_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "admin proposals", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 77, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_admin_cmd_total{status=\"success\", type!=\"compact\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_raftstore_admin_cmd_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "admin apply", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 70, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_raftstore_check_split_total{type!=\"ignore\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_raftstore_check_split_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "check split", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 71, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.9999, sum(rate(tikv_raftstore_check_split_duration_seconds_bucket[1m])) by (le, job))", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_raftstore_check_split_duration_seconds_bucket", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% check split duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Raft Admin", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_storage_command_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "storage command total", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_storage_engine_async_request_total{status!~\"all|success\"}[1m])) by (status)", "intervalFactor": 2, "legendFormat": "{{status}}", "metric": "tikv_raftstore_raft_process_duration_secs_bucket", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "storage async request error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 15, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"snapshot\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"snapshot\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"snapshot\"}[1m]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "storage async snapshot duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 109, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type=\"write\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{type=\"write\"}[1m]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "storage async write duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1310, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "raft-95%", "yaxis": 2 } ], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_batch_commands_total_bucket[30s])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_batch_commands_total_bucket[30s])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_storage_batch_commands_total_sum[30s])) / sum(rate(tikv_storage_batch_commands_total_count[30s]))", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_batch_snapshot_commands_total_bucket[30s])) by (le))", "intervalFactor": 2, "legendFormat": "raft-95%", "refId": "D", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "storage async batch snapshot", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Storage Batch Size", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": "Raftstore Batch Size", "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Storage", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "height": "400", "id": 167, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_too_busy_total[1m])) by (stage)", "intervalFactor": 2, "legendFormat": "busy", "refId": "A", "step": 20 }, { "expr": "sum(rate(tikv_scheduler_stage_total[1m])) by (stage)", "intervalFactor": 2, "legendFormat": "{{stage}}", "refId": "B", "step": 20 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler stage total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "height": "", "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_commands_pri_total[1m])) by (priority)", "intervalFactor": 2, "legendFormat": "{{priority}}", "metric": "", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler priority commands", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "transparent": false, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 300 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "120s", "handler": 1, "message": "TiKV scheduler context total", "name": "scheduler pending commands alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "height": "", "id": 193, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_scheduler_contex_total) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 40 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 300 } ], "timeFrom": null, "timeShift": null, "title": "scheduler pending commands", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "transparent": false, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Scheduler", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "height": "400", "id": 168, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 12, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 12, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_too_busy_total{type=\"$command\"}[1m]))", "intervalFactor": 2, "legendFormat": "busy", "refId": "A", "step": 4 }, { "expr": "sum(rate(tikv_scheduler_stage_total{type=\"$command\"}[1m])) by (stage)", "intervalFactor": 2, "legendFormat": "{{stage}}", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler stage total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 3, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_command_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_scheduler_command_duration_seconds_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_command_duration_seconds_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler command duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 194, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_scheduler_latch_wait_duration_seconds_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_latch_wait_duration_seconds_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler latch wait duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 195, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_read_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "kv_command_key", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_read_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_scheduler_kv_command_key_read_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_read_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler keys read", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 373, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_kv_command_key_write_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "kv_command_key", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_kv_command_key_write_bucket{type=\"$command\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_scheduler_kv_command_key_write_sum{type=\"$command\"}[1m])) / sum(rate(tikv_scheduler_kv_command_key_write_count{type=\"$command\"}[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler keys written", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 560, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\"}[1m])) by (tag)", "intervalFactor": 2, "legendFormat": "{{tag}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler scan details", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 675, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"lock\"}[1m])) by (tag)", "intervalFactor": 2, "legendFormat": "{{tag}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler scan details [lock]", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 829, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"write\"}[1m])) by (tag)", "intervalFactor": 2, "legendFormat": "{{tag}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler scan details [write]", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 830, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_scheduler_kv_scan_details{req=\"$command\", cf=\"default\"}[1m])) by (tag)", "intervalFactor": 2, "legendFormat": "{{tag}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler scan details [default]", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": "command", "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Scheduler - $command", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 16, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-95%", "refId": "B", "step": 4 }, { "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"select\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "select-avg", "refId": "C", "step": 4 }, { "expr": " sum(rate(tikv_coprocessor_request_duration_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_duration_seconds_count{req=\"index\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "index-avg", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor request duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 111, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-95%", "refId": "B", "step": 4 }, { "expr": " sum(rate(tikv_coprocessor_request_wait_seconds_sum[1m])) / sum(rate(tikv_coprocessor_request_wait_seconds_count[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor wait duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 113, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le,req))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{req}}-95%", "refId": "B", "step": 4 }, { "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"select\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"select\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "select-avg", "refId": "C", "step": 4 }, { "expr": " sum(rate(tikv_coprocessor_request_handle_seconds_sum{req=\"index\"}[1m])) / sum(rate(tikv_coprocessor_request_handle_seconds_count{req=\"index\"}[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "index-avg", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor handle duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 115, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_duration_seconds_bucket[1m])) by (le, job,req))", "intervalFactor": 2, "legendFormat": "{{job}}-{{req}}", "metric": "tikv_coprocessor_request_duration_seconds_bucket", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "95% coprocessor request duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 116, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m])) by (le, job,req))", "intervalFactor": 2, "legendFormat": "{{job}}-{{req}}", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "95% coprocessor wait duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 5, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 117, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tikv_coprocessor_request_handle_seconds_bucket[1m])) by (le, job,req))", "intervalFactor": 2, "legendFormat": "{{job}}-{{req}}", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "95% coprocessor handle duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 52, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, avg(rate(tikv_coprocessor_scan_keys_bucket[1m])) by (le, req)) ", "interval": "", "intervalFactor": 2, "legendFormat": "{{req}}-99%", "metric": "tikv_coprocessor_scan_keys_bucket", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, avg(rate(tikv_coprocessor_scan_keys_bucket[1m])) by (le, req)) ", "intervalFactor": 2, "legendFormat": "{{req}}-95%", "metric": "tikv_coprocessor_scan_keys_bucket", "refId": "B", "step": 10 }, { "expr": "histogram_quantile(0.90, avg(rate(tikv_coprocessor_scan_keys_bucket[1m])) by (le, req)) ", "intervalFactor": 2, "legendFormat": "{{req}}-90%", "metric": "tikv_coprocessor_scan_keys_bucket", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor scan keys", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 551, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_executor_count[1m])) by (type)", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor executor count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 74, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_request_error[1m])) by (reason)", "interval": "", "intervalFactor": 2, "legendFormat": "{{reason}}", "metric": "tikv_coprocessor_request_error", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor request errors", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 552, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_scan_details[1m])) by (tag,req)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{req}}-{{tag}}", "metric": "scan_details", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor scan details", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 122, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_scan_details{req=\"select\"}[1m])) by (tag,cf)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{cf}}-{{tag}}", "metric": "scan_details", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor table scan details -- cf", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 554, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": "cf", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_scan_details{req=\"index\"}[1m])) by (tag,cf)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{cf}}-{{tag}}", "metric": "scan_details", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "coprocessor index scan details - cf", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Coprocessor", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 26, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " max", "metric": "", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "B", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " 95%", "metric": "", "refId": "C", "step": 4 }, { "expr": "sum(rate(tikv_storage_mvcc_versions_sum[1m])) / sum(rate(tikv_storage_mvcc_versions_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "D", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MVCC Versions", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 559, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1.0, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " max", "metric": "", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "B", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_mvcc_gc_delete_versions_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": " 95%", "metric": "", "refId": "C", "step": 4 }, { "expr": "sum(rate(tikv_storage_mvcc_gc_delete_versions_sum[1m])) / sum(rate(tikv_storage_mvcc_gc_delete_versions_count[1m])) ", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "D", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MVCC Delete Versions", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 121, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_storage_command_total{type=\"gc\"}[1m]))", "intervalFactor": 2, "legendFormat": "total", "metric": "tikv_storage_command_total", "refId": "A", "step": 4 }, { "expr": "sum(rate(tikv_storage_gc_skipped_counter[1m]))", "intervalFactor": 2, "legendFormat": "skipped", "metric": "tikv_storage_gc_skipped_counter", "refId": "B", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Commands", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 967, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tidb_tikvclient_gc_action_result_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Actions Result", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 966, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tidb_tikvclient_gc_worker_actions_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Worker Actions", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 969, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1.0, sum(rate(tidb_tikvclient_gc_seconds_bucket[1m])) by (instance, le))", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gc Seconds", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 968, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tidb_tikvclient_gc_failure_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gc failure", "tooltip": { "msResolution": false, "shared": true, "sort": 2, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "editable": true, "error": false, "format": "s", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "id": 27, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "null", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 3, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "tableColumn": "", "targets": [ { "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_life_time\"})", "interval": "", "intervalFactor": 2, "refId": "A", "step": 60 } ], "thresholds": "", "title": "GC LifeTime", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" }, { "cacheTimeout": null, "colorBackground": false, "colorValue": false, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "editable": true, "error": false, "format": "s", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "id": 28, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": 100, "nullPointMode": "null", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "span": 3, "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": false }, "targets": [ { "expr": "max(tidb_tikvclient_gc_config{type=\"tikv_gc_run_interval\"})", "intervalFactor": 2, "refId": "A", "step": 60 } ], "thresholds": "", "title": "GC interval", "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current" } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "GC", "titleSize": "h6" }, { "collapse": true, "height": "250px", "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(delta(tikv_raftstore_raft_sent_message_total{type=\"snapshot\"}[1m]))", "intervalFactor": 2, "legendFormat": " ", "refId": "A", "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "rate snapshot message", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 4, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_server_send_snapshot_duration_seconds_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "send", "refId": "A", "step": 60 }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"apply\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "apply", "refId": "B", "step": 60 }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_snapshot_duration_seconds_bucket{type=\"generate\"}[1m])) by (le))", "intervalFactor": 2, "legendFormat": "generate", "refId": "C", "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% handle snapshot duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 38, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 4, "stack": false, "steppedLine": true, "targets": [ { "expr": "sum(tikv_raftstore_snapshot_traffic_total) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 60 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "snapshot state count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_size_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "size", "metric": "tikv_snapshot_size_bucket", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% snapshot size", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 1, "grid": {}, "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.9999, sum(rate(tikv_snapshot_kv_count_bucket[1m])) by (le))", "intervalFactor": 2, "legendFormat": "count", "metric": "tikv_snapshot_kv_count_bucket", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% snapshot kv count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Snapshot", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_worker_handled_task_total[1m])) by (name)", "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker Handled Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1395, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_worker_pending_task_total[1m])) by (name)", "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker Pending Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1876, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_futurepool_handled_task_total[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool Handled Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 1877, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 400, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_futurepool_pending_task_total[1m])) by (name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}}", "metric": "tikv_pd_heartbeat_tick_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool Pending Tasks", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "alert": { "conditions": [ { "evaluator": { "params": [ 2 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "B", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "10s", "handler": 1, "message": "coprocessor pending requests", "name": "coprocessor pending requests alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 550, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_pending_request[1m])) by (req, priority)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{ req }} - {{priority}}", "metric": "tikv_coprocessor_request_error", "refId": "A", "step": 4 }, { "expr": "sum(rate(tikv_coprocessor_pending_request[1m])) BY (type, instance)", "format": "time_series", "hide": true, "intervalFactor": 2, "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 2 } ], "timeFrom": null, "timeShift": null, "title": "coprocessor pending requests", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Task", "titleSize": "h6" }, { "collapse": true, "height": 250, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.8 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "datasourceId": 1, "model": { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 20 }, "params": [ "A", "1m", "now" ] }, "reducer": { "params": [], "type": "max" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "message": "TiKV raftstore thread CPU usage is high", "name": "TiKV raft store CPU alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 61, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"raftstore_.*\"}[1m])) by (job, name)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8 } ], "timeFrom": null, "timeShift": null, "title": "raft store CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 1, "grid": {}, "id": 79, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=\"apply_worker\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "async apply CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "editable": true, "error": false, "fill": 0, "grid": {}, "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"storage_schedul.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "scheduler CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 64, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"sched_.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler Worker CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 1908, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"store_read.*\"}[1m])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage ReadPool CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 78, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"cop_.*\"}[1m])) by (job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"snapshot_worker.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "snapshot worker CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 68, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"split_check.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "split check CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "editable": true, "error": false, "fill": 0, "grid": {}, "id": 69, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(rate(tikv_thread_cpu_seconds_total{name=~\"rocksdb.*\"}[1m])) by (job)", "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "warning", "fill": true, "line": true, "op": "gt", "value": 1 }, { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 4 } ], "timeFrom": null, "timeShift": null, "title": "rocksdb CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 105, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{name=~\"grpc.*\"}[1m])) by (job)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job}}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "grpc poll CPU", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Thread CPU", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 138, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m]))", "intervalFactor": 2, "legendFormat": "memtable", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=~\"block_cache_data_hit|block_cache_filter_hit\"}[1m]))", "intervalFactor": 2, "legendFormat": "block_cache", "metric": "", "refId": "E", "step": 10 }, { "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l0\"}[1m]))", "intervalFactor": 2, "legendFormat": "l0", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l1\"}[1m]))", "intervalFactor": 2, "legendFormat": "l1", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_get_served{db=\"$db\", type=\"get_hit_l2_and_up\"}[1m]))", "intervalFactor": 2, "legendFormat": "l2_and_up", "refId": "F", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 82, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_get_micro_seconds{db=\"$db\",type=\"get_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 129, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_seek\"}[1m]))", "intervalFactor": 2, "legendFormat": "seek", "metric": "", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_seek_found\"}[1m]))", "intervalFactor": 2, "legendFormat": "seek_found", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_next\"}[1m]))", "intervalFactor": 2, "legendFormat": "next", "metric": "", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_next_found\"}[1m]))", "intervalFactor": 2, "legendFormat": "next_found", "metric": "", "refId": "D", "step": 10 }, { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_prev\"}[1m]))", "intervalFactor": 2, "legendFormat": "prev", "metric": "", "refId": "E", "step": 10 }, { "expr": "sum(rate(tikv_engine_locate{db=\"$db\", type=\"number_db_prev_found\"}[1m]))", "intervalFactor": 2, "legendFormat": "prev_found", "metric": "", "refId": "F", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 125, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_seek_micro_seconds{db=\"$db\",type=\"seek_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 139, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=~\"write_done_by_self|write_done_by_other\"}[1m]))", "intervalFactor": 2, "legendFormat": "done", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=\"write_timeout\"}[1m]))", "intervalFactor": 2, "legendFormat": "timeout", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_write_served{db=\"$db\", type=\"write_with_wal\"}[1m]))", "intervalFactor": 2, "legendFormat": "with_wal", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 126, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_write_micro_seconds{db=\"$db\",type=\"write_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 137, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_wal_file_synced{db=\"$db\"}[1m]))", "intervalFactor": 2, "legendFormat": "sync", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL Sync Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 135, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{db=\"$db\",type=\"wal_file_sync_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL Sync Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 128, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_event_total{db=\"$db\"}[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_engine_event_total", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 136, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": null, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_max\"})", "intervalFactor": 2, "legendFormat": "max", "metric": "", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_compaction_time{db=\"$db\",type=\"compaction_time_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 140, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_max\"})", "intervalFactor": 2, "legendFormat": "max", "metric": "", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_sst_read_micros{db=\"$db\", type=\"sst_read_micros_average\"})", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST Read Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 87, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_max\"})", "intervalFactor": 2, "legendFormat": "max", "metric": "", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "metric": "", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "metric": "", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_write_stall{db=\"$db\", type=\"write_stall_average\"})", "intervalFactor": 2, "legendFormat": "avg", "metric": "", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Stall Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 103, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_memory_bytes{db=\"$db\", type=\"mem-tables\"}) by (cf)", "intervalFactor": 2, "legendFormat": "{{cf}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 88, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": null, "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) / (sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_hit\"}[1m])) + sum(rate(tikv_engine_memtable_efficiency{db=\"$db\", type=\"memtable_miss\"}[1m])))", "intervalFactor": 2, "legendFormat": "hit", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable Hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 102, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_block_cache_size_bytes{db=\"$db\"}) by(cf)", "intervalFactor": 2, "legendFormat": "{{cf}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 80, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 6, "nullPointMode": "connected", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_miss\"}[1m])))", "intervalFactor": 2, "legendFormat": "all", "metric": "", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_miss\"}[1m])))", "intervalFactor": 2, "legendFormat": "data", "metric": "", "refId": "D", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_miss\"}[1m])))", "intervalFactor": 2, "legendFormat": "filter", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_miss\"}[1m])))", "intervalFactor": 2, "legendFormat": "index", "metric": "", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_useful\"}[1m])) / sum(rate(tikv_engine_bloom_efficiency{db=\"$db\", type=\"bloom_prefix_checked\"}[1m]))", "intervalFactor": 2, "legendFormat": "bloom prefix", "metric": "", "refId": "E", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache Hit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "height": "", "id": 467, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"block_cache_byte_read\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "total_read", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"block_cache_byte_write\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "total_written", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_bytes_insert\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "data_insert", "metric": "", "refId": "D", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_bytes_insert\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "filter_insert", "metric": "", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_bytes_evict\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "filter_evict", "metric": "", "refId": "E", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_bytes_insert\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "index_insert", "metric": "", "refId": "F", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_bytes_evict\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "index_evict", "metric": "", "refId": "G", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 468, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_add\"}[1m]))", "intervalFactor": 2, "legendFormat": "total_add", "metric": "", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_data_add\"}[1m]))", "intervalFactor": 2, "legendFormat": "data_add", "metric": "", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_filter_add\"}[1m]))", "intervalFactor": 2, "legendFormat": "filter_add", "metric": "", "refId": "D", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_index_add\"}[1m]))", "intervalFactor": 2, "legendFormat": "index_add", "metric": "", "refId": "E", "step": 10 }, { "expr": "sum(rate(tikv_engine_cache_efficiency{db=\"$db\", type=\"block_cache_add_failures\"}[1m]))", "intervalFactor": 2, "legendFormat": "add_failures", "metric": "", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache Operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "height": "", "id": 132, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"keys_read\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "read", "refId": "B", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"keys_written\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "written", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_compaction_num_corrupt_keys{db=\"$db\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "corrupt", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 131, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_estimate_num_keys{db=\"$db\"}) by (cf)", "hide": false, "intervalFactor": 2, "legendFormat": "{{cf}}", "metric": "tikv_engine_estimate_num_keys", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Keys", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "height": "", "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"bytes_read\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "get", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"iter_bytes_read\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "scan", "refId": "C", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 133, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_read{db=\"$db\",type=\"bytes_per_read_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Read", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "height": "", "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"wal_file_bytes\"}[1m]))", "hide": false, "intervalFactor": 2, "legendFormat": "wal", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"bytes_written\"}[1m]))", "hide": false, "intervalFactor": 2, "legendFormat": "write", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 0, "id": 134, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "minSpan": 6, "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_max\"})", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_percentile99\"})", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_percentile95\"})", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "avg(tikv_engine_bytes_per_write{db=\"$db\",type=\"bytes_per_write_average\"})", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Write", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 10, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 90, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"$db\", type=\"bytes_read\"}[1m]))", "hide": false, "intervalFactor": 2, "legendFormat": "read", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{db=\"$db\", type=\"bytes_written\"}[1m]))", "hide": false, "intervalFactor": 2, "legendFormat": "written", "refId": "C", "step": 10 }, { "expr": "sum(rate(tikv_engine_flow_bytes{db=\"$db\", type=\"flush_write_bytes\"}[1m]))", "hide": false, "intervalFactor": 2, "legendFormat": "flushed", "refId": "B", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 127, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_pending_compaction_bytes{db=\"$db\"}[1m])) by (cf)", "hide": false, "intervalFactor": 2, "legendFormat": "{{cf}}", "metric": "tikv_engine_pending_compaction_bytes", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Pending Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 518, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_total_read_bytes\"}[1m])) by (job) / sum(rate(tikv_engine_read_amp_flow_bytes{db=\"$db\", type=\"read_amp_estimate_useful_bytes\"}[1m])) by (job)", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Amplication", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 863, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_compression_ratio{db=\"$db\"}) by (level)", "hide": false, "intervalFactor": 2, "legendFormat": "level - {{level}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compression Ratio", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 516, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "tikv_engine_num_snapshots{db=\"$db\"}", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Snapshots", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 517, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "tikv_engine_oldest_snapshot_duration{db=\"$db\"}", "hide": false, "intervalFactor": 2, "legendFormat": "{{job}}", "metric": "tikv_engine_oldest_snapshot_duration", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Oldest Snapshots Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "id": 2002, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_num_files_at_level{db=\"$db\"}) by (cf, level)", "format": "time_series", "intervalFactor": 2, "legendFormat": "cf-{{cf}}, level-{{level}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number files at each level", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fill": 1, "id": 2003, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_snapshot_ingest_sst_duration_seconds_bucket[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "A" }, { "expr": "sum(rate(tikv_snapshot_ingest_sst_duration_seconds_sum[1m])) / sum(rate(tikv_snapshot_ingest_sst_duration_seconds_count[1m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "average", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ingest SST duration seconds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": "db", "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Rocksdb - $db", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 95, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{type!=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "grpc message count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 107, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_fail_total{type!=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tikv_grpc_msg_fail_total", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "grpc message failed", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 98, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type!=\"kv_gc\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% grpc messge duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1844, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{type=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "gc message cnt", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_grpc_msg_fail_total{type=\"kv_gc\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "gc failed cnt", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "grpc gc message count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1845, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type=\"kv_gc\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% grpc kv_gc messge duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "Grpc", "titleSize": "h6" }, { "collapse": true, "height": "300", "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1069, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_pd_request_duration_seconds_count[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{ type }}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD requests", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1070, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_pd_request_duration_seconds_sum[1m])) by (type) / sum(rate(tikv_pd_request_duration_seconds_count[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{ type }}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD request duration (average)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1215, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_pd_heartbeat_message_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{ type }}", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD heartbeats", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "fill": 1, "id": 1396, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "span": 6, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_pd_validate_peer_total[1m])) by (type)", "intervalFactor": 2, "legendFormat": "{{ type }}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD validate peers", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": true, "title": "PD", "titleSize": "h6" } ], "schemaVersion": 14, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "db", "multi": true, "name": "db", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes, db)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "command", "multi": true, "name": "command", "options": [], "query": "label_values(tikv_storage_command_total, type)", "refresh": 1, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-TiKV", "version": 2 }