{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "gnetId": null, "graphTooltip": 0, "id": 228, "iteration": 1765278517529, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 123157, "panels": [], "title": "Runtime", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total cpu usage of all milvus components.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 123202, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(process_cpu_seconds_total{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}[1m]))", "interval": "", "intervalFactor": 1, "legendFormat": "cpu usage", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Usage Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:8411", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:8412", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, "hiddenSeries": false, "id": 123204, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "sum(process_resident_memory_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"})", "interval": "", "intervalFactor": 2, "legendFormat": "memory", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1340", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1341", "format": "bytes", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of goroutines that currently exist.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, "hiddenSeries": false, "id": 123206, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "sum(go_goroutines{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"})", "interval": "", "intervalFactor": 2, "legendFormat": "goroutines", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Goroutines Total", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1281", "format": "short", "label": "", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1282", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The CPU util of milvus components. ", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 123471, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "cpu_util{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Util", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:8411", "format": "percent", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:8412", "format": "percent", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, "hiddenSeries": false, "id": 123472, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "mem_util{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Util", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1340", "format": "percent", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1341", "format": "bytes", "logBase": 1, "show": false } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of goroutines that currently exist.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, "hiddenSeries": false, "id": 123125, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_goroutines{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Goroutines", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1539", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1540", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "process cpu usage", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 123135, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "rate(process_cpu_seconds_total{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}[1m])", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:544", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:545", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Process memory of milvus pod_name.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 }, "hiddenSeries": false, "id": 123133, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": false, "expr": "process_resident_memory_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:598", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:599", "format": "bytes", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of OS threads created.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 17 }, "hiddenSeries": false, "id": 123127, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_threads{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "OS Threads", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1952", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1953", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of bytes allocated and still in use.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 123137, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_alloc_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Allocated Memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1699", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1700", "format": "bytes", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "GC Max duration seconds.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 25 }, "hiddenSeries": false, "id": 123131, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_gc_duration_seconds{quantile=\"1\", _app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GC Max duration seconds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1871", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:1872", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The fraction of this program's available CPU time used by the GC since the program started.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 25 }, "hiddenSeries": false, "id": 123212, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_gc_cpu_fraction{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "GC CPU time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2515", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2516", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Next GC Bytes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 33 }, "hiddenSeries": false, "id": 123129, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_next_gc_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Next GC Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2191", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2192", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of process opened fds.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 33 }, "hiddenSeries": false, "id": 123211, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "process_open_fds{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Process Opened Fds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2355", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2356", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of allocated heap objects.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 33 }, "hiddenSeries": false, "id": 123216, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_heap_objects{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Allocated Heap Objects", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2837", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2838", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Rate of memory frees.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 41 }, "hiddenSeries": false, "id": 123213, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "rate(go_memstats_frees_total{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Free Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2675", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2676", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Rate of go memory mallocs.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 41 }, "hiddenSeries": false, "id": 123215, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "rate(go_memstats_mallocs_total{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}[2m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Memory mallocs rate.", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2756", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:2757", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of heap bytes waiting to be used.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 41 }, "hiddenSeries": false, "id": 123219, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_heap_idle_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Heap idle", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3473", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3474", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Rate of bytes allocated, even if freed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 49 }, "hiddenSeries": false, "id": 123217, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "rate(go_memstats_alloc_bytes_total{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}[5m])", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Allocated Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:8539", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:8540", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of heap bytes released to OS.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 49 }, "hiddenSeries": false, "id": 123218, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "cffe750a-c62f-4676-939f-6c82b3374b1f" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_heap_released_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Heap released", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3313", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3314", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of heap bytes that be used.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 49 }, "hiddenSeries": false, "id": 123375, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "go_memstats_heap_inuse_bytes{_app_name_=\"$clusterId\", pod_name=~\".*($pod_name)-.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Go Heap in Use", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3800", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3801", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "collapsed": true, "datasource": "${datasource}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 57 }, "id": 123248, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of searching vectors.", "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 123250, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${datasource}", "exemplar": true, "expr": "sum(increase(milvus_proxy_search_vectors_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": "${datasource}", "editorMode": "code", "expr": "sum(rate(milvus_proxy_search_vectors_count{_app_name_=\"$clusterId\"}[1m]))", "hide": false, "legendFormat": "total", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Vector Count Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of inserting vectors.", "fieldConfig": { "defaults": { "unit": "cps" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 2 }, "hiddenSeries": false, "id": 123315, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_insert_vectors_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Insert Vector Count Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 2 }, "hiddenSeries": false, "id": 123264, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_proxy_sq_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type) / sum(increase(milvus_proxy_sq_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 }, "hiddenSeries": false, "id": 123377, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_proxy_collection_sq_latency_bucket{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{query_type}}-{{pod_name}}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_sq_latency_sum{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])) by (pod_name, query_type) / sum(increase(milvus_proxy_collection_sq_latency_count{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])) by (pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{query_type}}-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Collection Search Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of mutation request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 }, "hiddenSeries": false, "id": 123320, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.95, sum by (le, msg_type, pod_name) (rate(milvus_proxy_mutation_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{msg_type}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Mutation Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of mutation request for a specified collection over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, "hiddenSeries": false, "id": 123378, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod_name) (rate(milvus_proxy_collection_mutation_latency_bucket{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_collection_mutation_latency_sum{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])) by (pod_name, msg_type) / sum(increase(milvus_proxy_collection_mutation_latency_count{_app_name_=\"$clusterId\", collection_name=~\"$collection_name\"}[2m])) by (pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{msg_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Collection Mutation Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average search latency in step", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, "hiddenSeries": false, "id": 123348, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_proxy_sq_wait_result_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}-wait-result", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_wait_result_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type) / sum(increase(milvus_proxy_sq_wait_result_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}-wait-result", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_proxy_sq_reduce_result_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "legendFormat": "p99-{{pod_name}}-{{query_type}}-reduce", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_proxy_sq_reduce_result_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type) / sum(increase(milvus_proxy_sq_reduce_result_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type)", "hide": false, "legendFormat": "avg-{{pod_name}}-{{query_type}}-reduce", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_proxy_sq_decode_result_latency_bucket{_app_name_=\"$clusterId\"}[2m])))\n", "hide": false, "legendFormat": "p99-{{pod_name}}-{{query_type}}-decode", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_proxy_sq_decode_result_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type) / sum(increase(milvus_proxy_sq_decode_resultlatency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, query_type)", "hide": false, "legendFormat": "avg-{{pod_name}}-{{query_type}}-decode", "range": true, "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Latency In Step", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of send mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 }, "hiddenSeries": false, "id": 123321, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, msg_type, pod_name) (rate(milvus_proxy_mutation_send_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_send_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type) / sum(increase(milvus_proxy_mutation_send_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{msg_type}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Mutation Send Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of mq mutation over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 18 }, "hiddenSeries": false, "id": 123448, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_msgstream_request_latency_bucket{_app_name_=\"$clusterId\", message_op_type=\"produce\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_msgstream_request_latency_sum{_app_name_=\"$clusterId\", message_op_type=\"produce\"}[2m])) by (pod_name) / sum(increase(milvus_msgstream_request_latency_count{_app_name_=\"$clusterId\", message_op_type=\"produce\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MQ Send Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of apply timestamp over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, "hiddenSeries": false, "id": 123327, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_proxy_apply_timestamp_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_apply_timestamp_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_apply_timestamp_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Apply Timestamp Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "ms", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile of latency for applying PK over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 }, "hiddenSeries": false, "id": 123326, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_proxy_apply_pk_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_apply_pk_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_apply_pk_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Apply PK Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of rpc in queue over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 26 }, "hiddenSeries": false, "id": 123470, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, function_name) (rate(milvus_proxy_req_in_queue_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{function_name}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_proxy_req_in_queue_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, function_name) / sum(increase(milvus_proxy_req_in_queue_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, function_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{function_name}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "In Queue Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the number of msgstream objects created on all physical topics.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 34 }, "hiddenSeries": false, "id": 123319, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_proxy_msgstream_obj_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_proxy_msgstream_obj_num{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_proxy_msgstream_obj_num{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Msg Stream Object Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 34 }, "hiddenSeries": false, "id": 123325, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_proxy_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_proxy_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_proxy_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "", "hide": false, "interval": "", "legendFormat": "", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Produced Timetick Lag Behind Now", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile of latency for assigning segment id over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 34 }, "hiddenSeries": false, "id": 123449, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_proxy_assign_segmentID_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_assign_segmentID_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_assign_segmentID_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Assign Segment ID Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 42 }, "hiddenSeries": false, "id": 123381, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, function_name) (rate(milvus_proxy_req_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{function_name}}-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, function_name) / sum(increase(milvus_proxy_req_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, function_name)", "hide": false, "interval": "", "legendFormat": "avg-{{function_name}}-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of success requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 42 }, "hiddenSeries": false, "id": 123329, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{_app_name_=\"$clusterId\", status=\"success\"}[2m])/120) by(function_name, pod_name)", "interval": "", "legendFormat": "{{function_name}}-{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Success Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of update cache over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 42 }, "hiddenSeries": false, "id": 123323, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_proxy_cache_update_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_cache_update_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_cache_update_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cache Update Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3538", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3539", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of bytes sent back to the client by the Proxy in response to a Search or Query request.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 50 }, "hiddenSeries": false, "id": 123369, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(milvus_proxy_send_bytes_count{_app_name_=\"$clusterId\"}[30s])) by(pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Send Byte Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "binBps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of bytes received in proxy", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 50 }, "hiddenSeries": false, "id": 123368, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(milvus_proxy_receive_bytes_count{_app_name_=\"$clusterId\"}[1m])) by(pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Received Byte Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "binBps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of faild requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 50 }, "hiddenSeries": false, "id": 123379, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{_app_name_=\"$clusterId\", status=\"fail\"}[2m])/120) by(function_name, pod_name)", "interval": "", "legendFormat": "{{function_name}}-{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Faild Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Failed Request QPS caused by Rate Limit", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 58 }, "hiddenSeries": false, "id": 123473, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(rate(milvus_proxy_rate_limit_req_count{_app_name_=\"$clusterId\", status=\"fail\"}[1m])) by (msg_type,pod_name,status)", "interval": "", "legendFormat": "{{pod_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Rate Limit Request QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Proxy", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 58 }, "id": 123155, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of proxy nodes which has register with etcd", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 123141, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_proxy_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Proxy Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 3 }, "hiddenSeries": false, "id": 123384, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_rootcoord_produce_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_rootcoord_produce_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_rootcoord_produce_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Produced Timetick Lag Behind Now", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency for rootcoord to finish synchronizing timestamp messages to all pchanels.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 3 }, "hiddenSeries": false, "id": 123338, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le) (rate(milvus_rootcoord_sync_timetick_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-latency", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_sync_timetick_latency_sum{_app_name_=\"$clusterId\"}[2m])) / sum(increase(milvus_rootcoord_sync_timetick_latency_count{_app_name_=\"$clusterId\"}[2m]))", "hide": false, "interval": "", "legendFormat": "avg-latency", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Produced Timetick Time Taken", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of DDL request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 123337, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, function_name) (rate(milvus_rootcoord_ddl_req_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{function_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (function_name) / sum(increase(milvus_rootcoord_ddl_req_latency_count{_app_name_=\"$clusterId\"}[2m])) by (function_name)", "hide": false, "interval": "", "legendFormat": "avg-{{function_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DDL Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "RootCoord stores pre-assigned timestamps in the metastore", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 11 }, "hiddenSeries": false, "id": 123340, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_rootcoord_timestamp_saved{_app_name_=\"$clusterId\"})", "interval": "", "intervalFactor": 1, "legendFormat": "timestamp", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Timestamp Saved", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of DDL request", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 11 }, "hiddenSeries": false, "id": 123347, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_ddl_req_count{_app_name_=\"$clusterId\"}[2m])/120) by (status, function_name)", "interval": "", "legendFormat": "{{function_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DDL Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1456", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "RoootCoord current latest timestamp", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 123339, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_rootcoord_timestamp{_app_name_=\"$clusterId\"})", "interval": "", "intervalFactor": 1, "legendFormat": "timestamp", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Timestamp", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of DML channels", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 19 }, "hiddenSeries": false, "id": 123345, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_dml_channel_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DML Channel Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of IDs assigned by RootCoord", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 19 }, "hiddenSeries": false, "id": 123221, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_rootcoord_id_alloc_count{_app_name_=\"$clusterId\"}[2m])/120)", "interval": "", "legendFormat": "total", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "ID Alloc Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1456", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:1457", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of partitions.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 123344, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_partition_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Partition Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The max time tick delay of flow graphs. unit ms", "fieldConfig": { "defaults": { "unit": "ms" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 27 }, "hiddenSeries": false, "id": 123383, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_time_tick_delay{_app_name_=\"$clusterId\"}) by (role_name, _node_name_)", "interval": "", "intervalFactor": 2, "legendFormat": "{{role_name}}-{{_node_name_}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Time Tick Delay", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of collections.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 27 }, "hiddenSeries": false, "id": 123342, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_collection_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Collection Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of credential", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 123343, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_credential_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Credential Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of Msgstream objects.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, "hiddenSeries": false, "id": 123346, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_rootcoord_msgstream_obj_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Msgstream Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:4353", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:4354", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Root Coordinator", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, "id": 123246, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of loaded collections.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 4 }, "hiddenSeries": false, "id": 123288, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querycoord_collection_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of loaded collections.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 4 }, "hiddenSeries": false, "id": 123403, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_querycoord_task_num{_app_name_=\"$clusterId\"}) by (querycoord_task_type)", "interval": "", "intervalFactor": 2, "legendFormat": "{{querycoord_task_type}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Task Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of load requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 4 }, "hiddenSeries": false, "id": 123291, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querycoord_load_req_count{_app_name_=\"$clusterId\"}[2m])/120) by (status)", "interval": "", "legendFormat": "{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Load Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of Query nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 123297, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querycoord_querynode_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "num", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Segment Num On Stopping Node", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 12 }, "hiddenSeries": false, "id": 123439, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_querynode_stopping_balance_segment_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segment Num On Stopping Node", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Channel Num On Stopping Node", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 12 }, "hiddenSeries": false, "id": 123444, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_querynode_stopping_balance_channel_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Channel Num On Stopping Node", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Query Coordinator", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 60 }, "id": 123244, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of the loaded collections in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 5 }, "hiddenSeries": false, "id": 123299, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_collection_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Collection Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of the loaded partitions in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 5 }, "hiddenSeries": false, "id": 123303, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_partition_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Partition Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of the loaded segment in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 5 }, "hiddenSeries": false, "id": 123305, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_segment_num{_app_name_=\"$clusterId\"}) by (pod_name, segment_state)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segment Loaded Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick behind.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 123385, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_querynode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"insert\"}) by (pod_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_querynode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"insert\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_querynode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"insert\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TimeTick Lag Behind Now (Consumed Insert)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of dml virtual channels for QueryNode watch", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 13 }, "hiddenSeries": false, "id": 123304, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_dml_vchannel_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DML Virtual Channel", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 13 }, "hiddenSeries": false, "id": 123387, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_consume_msg_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_consume_msg_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-all", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of delegator process insert/delete over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 123438, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, msg_type) (rate(milvus_querynode_process_insert_or_delete_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_querynode_process_insert_or_delete_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, msg_type) / sum(increase(milvus_querynode_process_insert_or_delete_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{msg_type}}", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Processing Insert/Delete Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of the queryable entities in QueryNode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 21 }, "hiddenSeries": false, "id": 123365, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(milvus_querynode_entity_num{_app_name_=\"$clusterId\"}) by (pod_name, collection_id, segment_state)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-{{collection_id}}-{{segment_state}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(milvus_querynode_entity_num{_app_name_=\"$clusterId\"})", "hide": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queryable Entity Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of searching requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 21 }, "hiddenSeries": false, "id": 123350, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_req_count{_app_name_=\"$clusterId\"}[2m])/120) by (query_type, status, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{query_type}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "search latency by phase", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 123466, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{query_type}}-InQueue", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{_app_name_=\"$clusterId\"}[2m])) by (query_type) ", "hide": false, "legendFormat": "avg-{{query_type}}-InQueue", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "legendFormat": "p99-{{query_type}}-{{segment_state}}-Segment", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{_app_name_=\"$clusterId\"}[2m])) by (query_type, segment_state)", "hide": false, "legendFormat": "avg-{{query_type}}-{{segment_state}}-Segment", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "legendFormat": "p99-{{query_type}}-Reduce", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "(sum(increase(milvus_querynode_sq_reduce_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{_app_name_=\"$clusterId\"}[2m])) by (query_type)) ", "hide": false, "legendFormat": "avg-{{query_type}}-Reduce", "range": true, "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Latency By Phase", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search segment over the last 2 minutes.", "fieldConfig": { "defaults": { "unit": "ms" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 29 }, "hiddenSeries": false, "id": 123374, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": false, "expr": "histogram_quantile(0.99, sum by (le, pod_name, query_type, segment_state) (rate(milvus_querynode_sq_segment_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}-{{segment_state}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_segment_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type, segment_state) / sum(increase(milvus_querynode_sq_segment_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type, segment_state)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}_{{segment_state}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Segment Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search and query request over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 29 }, "hiddenSeries": false, "id": 123366, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, query_type) (rate(milvus_querynode_sq_req_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_req_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type) / sum(increase(milvus_querynode_sq_req_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search or query in queue over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 123372, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, query_type) (rate(milvus_querynode_sq_queue_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_queue_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type) / sum(increase(milvus_querynode_sq_queue_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search in Queue Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of load segment over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 37 }, "hiddenSeries": false, "id": 123311, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_load_segment_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_load_segment_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_load_segment_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Load Segment Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 37 }, "hiddenSeries": false, "id": 123310, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, query_type, pod_name) (rate(milvus_querynode_sq_core_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_core_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type) / sum(increase(milvus_querynode_sq_core_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segcore Request Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search or query reduce over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 45 }, "hiddenSeries": false, "id": 123367, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, query_type) (rate(milvus_querynode_sq_reduce_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_reduce_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type) / sum(increase(milvus_querynode_sq_reduce_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Reduce Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The length of the task queue of read requests to be executed", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 45 }, "hiddenSeries": false, "id": 123356, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_ready_len{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Ready Read Task Length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of search at the segcore step over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 45 }, "hiddenSeries": false, "id": 123435, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name, query_type) (rate(milvus_querynode_sq_wait_tsafe_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}-{{query_type}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_sq_wait_tsafe_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type) / sum(increase(milvus_querynode_sq_wait_tsafe_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name, query_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{query_type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait tSafe Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The length of the task queue for unsolved read requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 53 }, "hiddenSeries": false, "id": 123351, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_unsolved_len{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Unsolved Read Task Length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of queries for search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 53 }, "hiddenSeries": false, "id": 123361, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_search_nq_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_nq_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_search_nq_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search NQ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 53 }, "hiddenSeries": false, "id": 123312, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_flowgraph_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flowgraph Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of queries for the merged search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 61 }, "hiddenSeries": false, "id": 123360, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_search_group_nq_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_nq_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_search_group_nq_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-milvus_querynode_search_group_nq", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Group NQ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of original tasks contained in the merged search task", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 61 }, "hiddenSeries": false, "id": 123352, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_search_group_size_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_size_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_search_group_size_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Group Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of read requests currently being executed in parallel", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 61 }, "hiddenSeries": false, "id": 123357, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_querynode_read_task_concurrency{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Parallel Read Task Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of evicted read requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 69 }, "hiddenSeries": false, "id": 123364, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_read_evicted_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "interval": "", "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Evicted Read Requests Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Top_K for search requests", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 69 }, "hiddenSeries": false, "id": 123359, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_search_topk_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_topk_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_search_topk_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Top_K", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Top_K for the merged search requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 69 }, "hiddenSeries": false, "id": 123362, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_querynode_search_group_topk_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_querynode_search_group_topk_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_querynode_search_group_topk_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Search Group Top_K", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Query Node", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 61 }, "id": 123419, "panels": [ { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 5 }, "id": 123420, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name, module) (rate(search_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(search_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module) / sum(increase(search_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module)", "hide": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "search latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 5 }, "id": 123421, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name, module) (rate(range_search_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(range_search_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module) / sum(increase(range_search_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module)", "hide": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "range search latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 5 }, "id": 123428, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name, module) (rate(build_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(build_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module) / sum(increase(knowhere_build_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module)", "hide": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "build index latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 13 }, "id": 123425, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(search_latency_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name, module)", "legendFormat": "{{module}}-{{pod_name}}", "range": true, "refId": "A" } ], "title": "search rate", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 13 }, "id": 123423, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(range_search_latencycount{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name, module)", "legendFormat": "{{module}}-{{pod_name}}", "range": true, "refId": "A" } ], "title": "range search rate", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 13 }, "id": 123417, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(build_latency_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name, module)", "legendFormat": "{{module}}-{{pod_name}}", "range": true, "refId": "A" } ], "title": "build index rate", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 21 }, "id": 123443, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, module, pod_name) (rate(load_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "instant": false, "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(load_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (module, pod_name) / sum(increase(load_latency_count{_app_name_=\"$clusterId\"}[2m])) by (module, pod_name)", "hide": false, "instant": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "Load Latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 21 }, "id": 123427, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name, module) (rate(ann_iterator_init_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(ann_iterator_init_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module) / sum(increase(ann_iterator_init_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module)", "hide": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "ann iterator init latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 21 }, "id": 123429, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name) (rate(knowhere_index_version_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(knowhere_index_version_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name) / sum(increase(knowhere_index_version_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name)", "hide": false, "legendFormat": "avg-{{pod_name}}", "range": true, "refId": "B" } ], "title": "knowhere create version index latency", "type": "timeseries" }, { "datasource": "${datasource}", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 29 }, "id": 123433, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" }, "tooltipOptions": { "mode": "single" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le,pod_name, module) (rate(search_topk_bucket{_app_name_=\"$clusterId\"}[2m])))", "legendFormat": "p99-{{module}}-{{pod_name}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(increase(search_topk_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module) / sum(increase(search_topk_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, module)", "hide": false, "legendFormat": "avg-{{module}}-{{pod_name}}", "range": true, "refId": "B" } ], "title": "search topk ", "type": "timeseries" } ], "title": "Knowhere", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 62 }, "id": 123172, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of data nodes which has register with etcd.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, "hiddenSeries": false, "id": 123207, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_datanode_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "total", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Data Node Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, "hiddenSeries": false, "id": 123390, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_datacoord_consume_datanode_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_datacoord_consume_datanode_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_datacoord_consume_datanode_tt_lag_ms{_app_name_=\"$clusterId\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TimeTick Lag Behind Now From DataNode", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The number of rows of valid data accumulated in DataCoord that flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }, "hiddenSeries": false, "id": 123269, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_rows_num{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "row", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Stored Rows", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of data flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }, "hiddenSeries": false, "id": 123371, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_datacoord_stored_rows_num{_app_name_=\"$clusterId\"}[2m])/120) by (segment_state)", "interval": "", "legendFormat": "{{segment_state}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Stored Rows Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Number of segments with different states in the Meta of DataCoord", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }, "hiddenSeries": false, "id": 123267, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_segment_num{_app_name_=\"$clusterId\"}) by (segment_state)", "interval": "", "intervalFactor": 2, "legendFormat": "{{segment_state}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segment Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "binlog size of all collections/segments, unit byte", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }, "hiddenSeries": false, "id": 123382, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datacoord_stored_binlog_size{_app_name_=\"$clusterId\"}) ", "interval": "", "intervalFactor": 2, "legendFormat": "size", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Stored Binlog Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "bytes", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now from datanode.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 123437, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "topK(10, timestamp(milvus_datacoord_channel_checkpoint_unix_seconds{_app_name_=\"$clusterId\"}) - milvus_datacoord_channel_checkpoint_unix_seconds{_app_name_=\"$clusterId\"}) by (pod_name, channel_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "channel cp lag", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "s", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of building index requests recived by IndexCoord.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 22 }, "hiddenSeries": false, "id": 123225, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(milvus_datacoord_index_req_count{_app_name_=\"$clusterId\"}[1m])) by (status)", "interval": "", "legendFormat": "{{status}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:372", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:373", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of building index requests recived by IndexCoord.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 22 }, "hiddenSeries": false, "id": 123451, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(milvus_datacoord_index_task_count{_app_name_=\"$clusterId\"}[1m])) by (index_task_status)", "interval": "", "legendFormat": "{{index_task_status}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Task Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:372", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:373", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Data Coordinator", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 63 }, "id": 123242, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Total number of flowgraph", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 123272, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(milvus_datanode_flowgraph_num{_app_name_=\"$clusterId\"}) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flowgraph Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of consuming message", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 7 }, "hiddenSeries": false, "id": 123389, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_consume_msg_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name, msg_type)", "interval": "", "legendFormat": "{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Consumed Message Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3414", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:3415", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "Average, maximum and minimum values of the timestamps for time tick lag behind now.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 7 }, "hiddenSeries": false, "id": 123393, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "avg(milvus_datanode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"all\"}) by (pod_name)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}-avg", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "max(milvus_datanode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"all\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-max", "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "min(milvus_datanode_consume_tt_lag_ms{_app_name_=\"$clusterId\", msg_type=\"all\"}) by (pod_name)", "hide": false, "interval": "", "legendFormat": "{{pod_name}}-min", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TimeTick Lag Behind Now (Consumed All)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:536", "decimals": 0, "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:537", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of flush operete.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 123284, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_buffer_op_count{_app_name_=\"$clusterId\"}[2m])/120) by (status, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flush Operate Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of encode the data in the buffer over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 15 }, "hiddenSeries": false, "id": 123282, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_datanode_encode_buffer_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_encode_buffer_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_datanode_encode_buffer_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Encode Buffer Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of messages consumed for insert and delete operation.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, "hiddenSeries": false, "id": 123274, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_msg_rows_count{_app_name_=\"$clusterId\"}[2m])/120) by (msg_type, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Msg Rows Consumed Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "cps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of compaction over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 23 }, "hiddenSeries": false, "id": 123314, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_datanode_compaction_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_compaction_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_datanode_compaction_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Compaction Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of auto flush operate.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 23 }, "hiddenSeries": false, "id": 123285, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_autoflush_buffer_op_count{_app_name_=\"$clusterId\"}[2m])/120) by (status, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Autoflush Operate Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of each message that has been flushed.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 23 }, "hiddenSeries": false, "id": 123275, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_flushed_data_size{_app_name_=\"$clusterId\"}[2m])/120) by (msg_type, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{msg_type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flush Data Size Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "binBps", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of flush requests.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 31 }, "hiddenSeries": false, "id": 123286, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "sum(increase(milvus_datanode_flush_req_count{_app_name_=\"$clusterId\"}[2m])/120) by (status, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{status}}", "queryType": "randomWalk", "range": true, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flush Request Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of writte the data in buffer to storage over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 31 }, "hiddenSeries": false, "id": 123283, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_datanode_save_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_datanode_save_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_datanode_save_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Save Data Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Data Node", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 64 }, "id": 123231, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "per-second increasing rate of index tasks recevied by IndexNode", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 123233, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_index_task_count{_app_name_=\"$clusterId\"}[2m])/120) by (status, pod_name)", "interval": "", "legendFormat": "{{pod_name}}-{{status}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Task Rate", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:101", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency ofr load the FieldData over the last 2 minutes.", "fieldConfig": { "defaults": { "unit": "ms" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, "hiddenSeries": false, "id": 123235, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_indexnode_load_field_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_load_field_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_indexnode_load_field_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Load Field Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:161", "format": "ms", "logBase": 1, "show": true }, { "$$hashKey": "object:162", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of decod the FieldData over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, "hiddenSeries": false, "id": 123238, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_indexnode_decode_field_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_decode_field_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_indexnode_decode_field_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Decode Field Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:190", "format": "short", "label": "", "logBase": 1, "show": true }, { "$$hashKey": "object:191", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of build index over the last 2 minutes.", "fieldConfig": { "defaults": { "unit": "ms" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 123237, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_indexnode_build_index_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_build_index_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_indexnode_build_index_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Build Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:246", "format": "ms", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:247", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile and average latency of encode index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "hiddenSeries": false, "id": 123239, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_indexnode_encode_index_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_encode_index_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_indexnode_encode_index_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Encode Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:275", "format": "short", "logBase": 1, "min": "0", "show": true }, { "$$hashKey": "object:276", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${datasource}", "description": "The 99th percentile of latency for save index over the last 2 minutes.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "hiddenSeries": false, "id": 123240, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.2.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_indexnode_save_index_latency_bucket{_app_name_=\"$clusterId\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "queryType": "randomWalk", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "exemplar": true, "expr": "sum(increase(milvus_indexnode_save_index_latency_sum{_app_name_=\"$clusterId\"}[2m])) by(pod_name) / sum(increase(milvus_indexnode_save_index_latency_count{_app_name_=\"$clusterId\"}[2m])) by(pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Save Index Latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:331", "format": "short", "logBase": 1, "show": true }, { "$$hashKey": "object:332", "format": "short", "logBase": 1, "show": true } ], "yaxis": { "align": false } } ], "title": "Index Node", "type": "row" } ], "schemaVersion": 32, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus-milvus-hangzhou-pre", "value": "Prometheus-milvus-hangzhou-pre" }, "description": null, "error": null, "hide": 2, "includeAll": false, "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "/^Prometheus-milvus/", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": { "selected": false, "text": "c-0dcefe3b4731fffb", "value": "c-0dcefe3b4731fffb" }, "datasource": "${datasource}", "definition": "query_result((count by (_app_name_) (count_over_time(go_info[30m]))))", "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "clusterId", "options": [], "query": { "query": "query_result((count by (_app_name_) (count_over_time(go_info[30m]))))", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": ".*_app_name_=\"(.*)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": "${datasource}", "definition": "query_result((count by (collection_name) (count_over_time(milvus_rootcoord_entity_num{_app_name_=\"$clusterId\",status=\"total\"}[30m]))))", "description": "Collection", "error": null, "hide": 0, "includeAll": false, "label": "Collection", "multi": false, "name": "collection_name", "options": [], "query": { "query": "query_result((count by (collection_name) (count_over_time(milvus_rootcoord_entity_num{_app_name_=\"$clusterId\",status=\"total\"}[30m]))))", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": ".*collection_name=\"(.*)\".*", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": true, "text": [ "datanode", "proxy", "querynode" ], "value": [ "datanode", "proxy", "querynode" ] }, "datasource": "${datasource}", "definition": "query_result(count by (pod_name) (label_replace(count_over_time(go_info{_app_name_=\"$clusterId\"}[30m]),\"pod_name\",\"$1\",\"pod_name\",\"^milvus-milvus-([^-\\n]+)-.*$\")))", "description": "Component", "error": null, "hide": 0, "includeAll": true, "label": "Component", "multi": true, "name": "pod_name", "options": [], "query": { "query": "query_result(count by (pod_name) (label_replace(count_over_time(go_info{_app_name_=\"$clusterId\"}[30m]),\"pod_name\",\"$1\",\"pod_name\",\"^milvus-milvus-([^-\\n]+)-.*$\")))", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": ".*pod_name=.*\"(.*)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Milvus-overview-v2", "uid": "Milvus-overview-v2", "version": 36 }