{ "__inputs": [ { "name": "DS_LOKI", "label": "Loki", "description": "Loki data source", "type": "datasource", "pluginId": "loki", "pluginName": "Loki" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, { "type": "datasource", "id": "loki", "name": "Loki", "version": "1.0.0" } ], "annotations": { "list": [] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 10, "title": "Overview", "type": "row" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisLabel": "Count", "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, "stacking": { "group": "A", "mode": "normal" } } }, "overrides": [ { "matcher": { "id": "byName", "options": "error" }, "properties": [ { "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "warn" }, "properties": [ { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [ { "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } } ] }, { "matcher": { "id": "byName", "options": "debug" }, "properties": [ { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 1 }, "id": 1, "options": { "legend": { "calcs": [ "sum" ], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "multi" } }, "title": "Log Volume by Level", "type": "timeseries", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=~\"$log_level\" [$__auto])) by (detected_level)", "legendFormat": "{{detected_level}}", "refId": "A" } ] }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 50 } ] }, "mappings": [] } }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 9 }, "id": 2, "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Total Errors (selected range)", "type": "stat", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" [$__auto]))", "refId": "A" } ] }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 50 }, { "color": "orange", "value": 200 } ] } } }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 9 }, "id": 3, "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Total Warnings (selected range)", "type": "stat", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"warn\" [$__auto]))", "refId": "A" } ] }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "blue", "value": null } ] } } }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 9 }, "id": 4, "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "Total Log Lines (selected range)", "type": "stat", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} [$__auto]))", "refId": "A" } ] }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] } } }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 9 }, "id": 5, "options": { "colorMode": "background", "graphMode": "area", "reduceOptions": { "calcs": [ "sum" ], "fields": "", "values": false }, "textMode": "auto" }, "title": "HTTP 4xx/5xx Responses", "type": "stat", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | json | status_code >= 400 [$__auto]))", "refId": "A" } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 11, "title": "Error Analysis", "type": "row" }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "fixedColor": "red", "mode": "fixed" }, "custom": { "axisBorderShow": false, "axisLabel": "errors / sec", "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2, "pointSize": 5 } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, "id": 6, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "multi" } }, "title": "Error Rate Over Time", "type": "timeseries", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "sum(rate({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" [$__auto])) by (service_instance_id)", "legendFormat": "{{service_instance_id}}", "refId": "A" } ] }, { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true } }, "overrides": [ { "matcher": { "id": "byName", "options": "Count" }, "properties": [ { "id": "custom.width", "value": 100 } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 22 }, "id": 7, "options": { "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Count" } ] }, "title": "Top Error Messages", "type": "table", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "topk(10, sum by (service_name, log_message) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" | json | label_format log_message=\"{{or .msg .message}}\" [$__range])))", "queryType": "instant", "refId": "A" } ], "transformations": [ { "id": "labelsToFields", "options": { "mode": "columns" } }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": {}, "renameByName": { "Value": "Count", "Value #A": "Count", "log_message": "Message", "service_name": "Service" } } } ] }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 12, "title": "Log Browser", "type": "row" }, { "id": 8, "type": "logs", "title": "Mattermost Logs", "gridPos": { "x": 0, "y": 33, "h": 14, "w": 24 }, "fieldConfig": { "defaults": {}, "overrides": [] }, "pluginVersion": "11.6.1", "targets": [ { "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "expr": "{service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | json | detected_level=~\"$log_level\" |~ \"$search\"", "refId": "A", "editorMode": "code", "queryType": "range", "direction": "backward" } ], "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "options": { "showTime": true, "showLabels": false, "showCommonLabels": true, "wrapLogMessage": true, "prettifyLogMessage": false, "enableLogDetails": true, "enableInfiniteScrolling": false, "dedupStrategy": "none", "sortOrder": "Descending" } } ], "schemaVersion": 39, "tags": [ "mattermost", "loki", "logs" ], "templating": { "list": [ { "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "definition": "label_values(service_name)", "includeAll": true, "multi": true, "name": "service_name", "query": "label_values(service_name)", "refresh": 2, "sort": 1, "type": "query" }, { "current": { "selected": true, "text": "All", "value": "$__all" }, "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, "definition": "label_values(service_instance_id)", "includeAll": true, "multi": true, "name": "service_instance_id", "query": "label_values(service_instance_id)", "refresh": 2, "sort": 1, "type": "query" }, { "current": { "selected": true, "text": "All", "value": "$__all" }, "description": "Filter by log severity", "hide": 0, "includeAll": true, "multi": true, "name": "log_level", "options": [ { "selected": true, "text": "All", "value": "$__all" }, { "selected": false, "text": "info", "value": "info" }, { "selected": false, "text": "warn", "value": "warn" }, { "selected": false, "text": "error", "value": "error" } ], "query": "info,warn,error", "queryValue": "", "refresh": 2, "skipUrlSync": false, "type": "custom" }, { "current": { "selected": false, "text": "", "value": "" }, "name": "search", "options": [], "query": "", "type": "textbox" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Mattermost Log Aggregation", "uid": "mattermost-loki-logs", "version": 1 }