{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "gnetId": null, "graphTooltip": 0, "id": 264, "iteration": 1765278447481, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 123479, "panels": [], "title": "性能", "type": "row" }, { "datasource": "${datasource}", "description": "每2分钟慢查询增量（超过5s）", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, "id": 123481, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "increase(milvus_proxy_slow_query_count{_app_name_=\"$clusterId\"}[2m])", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod_name}}-{{msg_type}}", "refId": "A" } ], "title": "慢查询 [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟搜索请求QPS", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, "id": 123494, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "sum(increase(milvus_proxy_search_vectors_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "interval": "", "intervalFactor": 1, "legendFormat": "{{pod_name}}", "refId": "A" } ], "title": "搜索QPS [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟写入请求QPS", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, "id": 123495, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "sum(increase(milvus_proxy_insert_vectors_count{_app_name_=\"$clusterId\"}[2m])/120) by (pod_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "refId": "A" } ], "title": "写入QPS [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟统计向量搜索类操作延迟P99和Avg", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, "id": 123486, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le,pod_name) (rate(milvus_proxy_sq_latency_bucket{_app_name_=\"$clusterId\", query_type=\"search\"}[2m])))", "interval": "", "legendFormat": "p99-{{pod_name}}", "refId": "A" }, { "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_latency_sum{_app_name_=\"$clusterId\", query_type=\"search\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_sq_latency_count{_app_name_=\"$clusterId\", query_type=\"search\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "title": "搜索延迟 [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟统计查询类操作延迟P99和Avg", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, "id": 123487, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum by (le, pod_name) (rate(milvus_proxy_sq_latency_bucket{_app_name_=\"$clusterId\", query_type=\"query\"}[2m])))", "hide": false, "interval": "", "legendFormat": "p99-{{pod_name}}", "refId": "A" }, { "exemplar": true, "expr": "sum(increase(milvus_proxy_sq_latency_sum{_app_name_=\"$clusterId\", query_type=\"query\"}[2m])) by (pod_name) / sum(increase(milvus_proxy_sq_latency_count{_app_name_=\"$clusterId\", query_type=\"query\"}[2m])) by (pod_name)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}", "refId": "B" } ], "title": "查询延迟 [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟统计写入类操作延迟P99和Avg", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 9 }, "id": 123488, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99,\n sum by (le, msg_type, pod_name) (\n rate(milvus_proxy_mutation_latency_bucket{_app_name_=\"$clusterId\"}[2m]) > 0\n )\n)\nor\n(\n count by (msg_type, pod_name) (\n milvus_proxy_mutation_latency_bucket{_app_name_=\"$clusterId\"}\n ) * 0\n)\n", "hide": false, "interval": "", "legendFormat": "p99-{{pod_name}}-{{msg_type}}", "refId": "A" }, { "exemplar": true, "expr": "sum(increase(milvus_proxy_mutation_latency_sum{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type) / sum(increase(milvus_proxy_mutation_latency_count{_app_name_=\"$clusterId\"}[2m])) by (pod_name, msg_type)", "hide": false, "interval": "", "legendFormat": "avg-{{pod_name}}-{{msg_type}}", "refId": "B" } ], "title": "写入延迟 [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟请求成功数量", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 17 }, "id": 123483, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{_app_name_=\"$clusterId\", status=\"success\"}[2m])) by(function_name) ", "interval": "", "legendFormat": "{{function_name}}", "refId": "A" } ], "title": "请求成功数 [2m]", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每2分钟请求失败数量", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 17 }, "id": 123484, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "sum(increase(milvus_proxy_req_count{_app_name_=\"$clusterId\", status!~\"success|total\"}[1m])) by(function_name) ", "interval": "", "legendFormat": "{{function_name}}", "refId": "A" } ], "title": "请求失败数 [2m]", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 123472, "panels": [], "title": "资源", "type": "row" }, { "datasource": "${datasource}", "description": "QueryNode/StreamingNode资源使用率", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, "id": 123474, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "mem_util{_app_name_=\"$clusterId\",pod_name=~\"milvus-milvus-querynode.*|milvus-milvus-streamingnode.*|milvus-milvus-standalone.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "refId": "A" } ], "title": "搜索资源水位（QueryNode/StreamingNode）", "type": "timeseries" }, { "datasource": "${datasource}", "description": "DataNode/IndexNode/StreamingNode资源使用率", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, "id": 123475, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "cpu_util{_app_name_=\"$clusterId\",pod_name=~\"milvus-milvus-indexnode.*|milvus-milvus-datanode.*|milvus-milvus-streamingnode.*|milvus-milvus-standalone.*\"}", "interval": "", "intervalFactor": 2, "legendFormat": "{{pod_name}}", "refId": "A" } ], "title": "写入资源水位（DataNode/IndexNode/StreamingNode）", "type": "timeseries" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, "id": 123490, "panels": [], "title": "数据", "type": "row" }, { "datasource": "${datasource}", "description": "每个DB里面Collection数量", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "id": 123492, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.2.7", "targets": [ { "exemplar": true, "expr": "sum(milvus_rootcoord_collection_num{_app_name_=\"$clusterId\"}) by (db_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{db_name}}", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Collection数量", "type": "timeseries" }, { "datasource": "${datasource}", "description": "每个Collection里面Entity数量", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 123493, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "pluginVersion": "8.2.7", "targets": [ { "exemplar": true, "expr": "sum(milvus_rootcoord_entity_num{_app_name_=\"$clusterId\",status=\"total\"}) by (collection_name)", "interval": "", "intervalFactor": 2, "legendFormat": "{{collection_name}}", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Entity数量", "type": "timeseries" } ], "schemaVersion": 32, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus-milvus-hangzhou-pre", "value": "Prometheus-milvus-hangzhou-pre" }, "description": null, "error": null, "hide": 2, "includeAll": false, "label": null, "multi": false, "name": "datasource", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "/^Prometheus-milvus/", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": { "selected": false, "text": "c-0dcefe3b4731fffb", "value": "c-0dcefe3b4731fffb" }, "datasource": "${datasource}", "definition": "query_result((count by (_app_name_) (count_over_time(go_info[30m]))))", "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "clusterId", "options": [], "query": { "query": "query_result((count by (_app_name_) (count_over_time(go_info[30m]))))", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": ".*_app_name_=\"(.*)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Milvus-general-zh-CN", "uid": "Milvus-general-zh-CN", "version": 6 }