Some checks failed
Server CI / Postgres (shard 0) (push) Has been skipped
Server CI / Postgres (shard 3) (push) Has been skipped
BuildEnv Docker Image / build-image (push) Failing after 4s
BuildEnv Docker Image / build-image-fips (push) Failing after 4s
Migration Review & Release Notes / Generate migration review & release notes (push) Failing after 3s
Server CI / Build Environment (push) Failing after 3s
Server CI / Postgres (shard 1) (push) Has been skipped
Tools CI / check-style (mattermost-govet) (push) Failing after 4s
Server CI / Postgres (shard 2) (push) Has been skipped
Tools CI / check-style (mmgotool) (push) Failing after 3s
Server CI / Elasticsearch v8 Compatibility (push) Has been skipped
Server CI / OpenSearch v2 Compatibility (push) Has been skipped
Server CI / Postgres FIPS (shard 0) (push) Has been skipped
Tools CI / Test (mattermost-govet) (push) Failing after 3s
Server CI / Postgres FIPS (shard 1) (push) Has been skipped
Server CI / Postgres FIPS (shard 2) (push) Has been skipped
Tools CI / Test (mmgotool) (push) Failing after 3s
Server CI / Postgres FIPS (shard 3) (push) Has been skipped
Server CI / Run mmctl tests (push) Has been skipped
Server CI / Run mmctl tests (FIPS) (push) Has been skipped
Server CI / Merge Postgres Test Results (push) Has been skipped
Server CI / Merge Postgres FIPS Test Results (push) Has been skipped
Server CI / Check backport migrations (push) Has been skipped
Server CI / Check generated files (push) Has been skipped
Server CI / check-style (push) Has been skipped
Server CI / Vet API (push) Has been skipped
Server CI / Build mattermost server app (push) Has been skipped
Server CI / Server CI Complete (push) Failing after 2s
Server CI / Server CI Artifacts (push) Has been skipped
Server CI / Sentry Upload (push) Has been skipped
Server CI / Server CI Report (push) Failing after 5m1s
Server CI Nightly Race / Race Detector (push) Has been skipped
CodeQL / Analyze (go) (push) Has been cancelled
CodeQL / Analyze (javascript) (push) Has been cancelled
Webapp CI Cache Warm / Warm Webapp node_modules Cache (push) Has been cancelled
E2E CI Cache Warm / Warm E2E npm Registry Cache (push) Has been cancelled
API / build (push) Has been cancelled
Web App CI / check-types (push) Has been cancelled
Web App CI / check-lint (push) Has been cancelled
Web App CI / check-i18n (push) Has been cancelled
Web App CI / check-external-links (push) Has been cancelled
Web App CI / test (platform) (push) Has been cancelled
Web App CI / test (mattermost-redux) (push) Has been cancelled
Web App CI / test (channels shard 1/4) (push) Has been cancelled
Web App CI / test (channels shard 2/4) (push) Has been cancelled
Web App CI / test (channels shard 3/4) (push) Has been cancelled
Web App CI / test (channels shard 4/4) (push) Has been cancelled
Web App CI / upload-coverage (push) Has been cancelled
Web App CI / build (push) Has been cancelled
YAML Lint / yamllint (push) Has been cancelled
Push mirrored docker images / cd/Push mirrored docker images (push) Has been cancelled
docs-cd / Build and deploy docs (push) Has been cancelled
Docs CI / build (push) Has been cancelled
E2E Cache Warm: Cypress deps / Warm Cypress deps cache (push) Has been cancelled
Scorecards supply-chain security / Scorecard analysis (push) Has been cancelled
Server CI Weekly / Postgres FIPS (push) Has been skipped
Server CI Weekly / Run mmctl tests (FIPS) (push) Has been skipped
Server CI Weekly / Postgres with binary parameters (push) Has been skipped
Server CI Weekly / Build Environment (push) Failing after 1s
Server CI Cache Warm / Warm Go Module and Build Cache (push) Failing after 2s
Server CI Nightly Race / Build Environment (push) Failing after 1s
718 lines
16 KiB
JSON
718 lines
16 KiB
JSON
{
|
|
"__inputs": [
|
|
{
|
|
"name": "DS_LOKI",
|
|
"label": "Loki",
|
|
"description": "Loki data source",
|
|
"type": "datasource",
|
|
"pluginId": "loki",
|
|
"pluginName": "Loki"
|
|
}
|
|
],
|
|
"__requires": [
|
|
{
|
|
"type": "grafana",
|
|
"id": "grafana",
|
|
"name": "Grafana",
|
|
"version": "10.0.0"
|
|
},
|
|
{
|
|
"type": "datasource",
|
|
"id": "loki",
|
|
"name": "Loki",
|
|
"version": "1.0.0"
|
|
}
|
|
],
|
|
"annotations": {
|
|
"list": []
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": null,
|
|
"links": [],
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 10,
|
|
"title": "Overview",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisLabel": "Count",
|
|
"drawStyle": "bars",
|
|
"fillOpacity": 80,
|
|
"gradientMode": "none",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
}
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "error"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"fixedColor": "red",
|
|
"mode": "fixed"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "warn"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"fixedColor": "yellow",
|
|
"mode": "fixed"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "info"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"fixedColor": "green",
|
|
"mode": "fixed"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "debug"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "color",
|
|
"value": {
|
|
"fixedColor": "blue",
|
|
"mode": "fixed"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 1,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"sum"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"title": "Log Volume by Level",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=~\"$log_level\" [$__auto])) by (detected_level)",
|
|
"legendFormat": "{{detected_level}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 10
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 50
|
|
}
|
|
]
|
|
},
|
|
"mappings": []
|
|
}
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"id": 2,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"sum"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"title": "Total Errors (selected range)",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" [$__auto]))",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "yellow",
|
|
"value": 50
|
|
},
|
|
{
|
|
"color": "orange",
|
|
"value": 200
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 9
|
|
},
|
|
"id": 3,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"sum"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"title": "Total Warnings (selected range)",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"warn\" [$__auto]))",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "blue",
|
|
"value": null
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 9
|
|
},
|
|
"id": 4,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"sum"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"title": "Total Log Lines (selected range)",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} [$__auto]))",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 1
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {
|
|
"h": 4,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 9
|
|
},
|
|
"id": 5,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "area",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"sum"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"title": "HTTP 4xx/5xx Responses",
|
|
"type": "stat",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | json | status_code >= 400 [$__auto]))",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 11,
|
|
"title": "Error Analysis",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"fixedColor": "red",
|
|
"mode": "fixed"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisLabel": "errors / sec",
|
|
"drawStyle": "line",
|
|
"fillOpacity": 20,
|
|
"lineWidth": 2,
|
|
"pointSize": 5
|
|
}
|
|
}
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 6,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"mean",
|
|
"max"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"title": "Error Rate Over Time",
|
|
"type": "timeseries",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "sum(rate({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" [$__auto])) by (service_instance_id)",
|
|
"legendFormat": "{{service_instance_id}}",
|
|
"refId": "A"
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"align": "auto",
|
|
"cellOptions": {
|
|
"type": "auto"
|
|
},
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": [
|
|
{
|
|
"matcher": {
|
|
"id": "byName",
|
|
"options": "Count"
|
|
},
|
|
"properties": [
|
|
{
|
|
"id": "custom.width",
|
|
"value": 100
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"id": 7,
|
|
"options": {
|
|
"showHeader": true,
|
|
"sortBy": [
|
|
{
|
|
"desc": true,
|
|
"displayName": "Count"
|
|
}
|
|
]
|
|
},
|
|
"title": "Top Error Messages",
|
|
"type": "table",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "topk(10, sum by (service_name, log_message) (count_over_time({service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | detected_level=\"error\" | json | label_format log_message=\"{{or .msg .message}}\" [$__range])))",
|
|
"queryType": "instant",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"transformations": [
|
|
{
|
|
"id": "labelsToFields",
|
|
"options": {
|
|
"mode": "columns"
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true
|
|
},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Value": "Count",
|
|
"Value #A": "Count",
|
|
"log_message": "Message",
|
|
"service_name": "Service"
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"id": 12,
|
|
"title": "Log Browser",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"id": 8,
|
|
"type": "logs",
|
|
"title": "Mattermost Logs",
|
|
"gridPos": {
|
|
"x": 0,
|
|
"y": 33,
|
|
"h": 14,
|
|
"w": 24
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {},
|
|
"overrides": []
|
|
},
|
|
"pluginVersion": "11.6.1",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"expr": "{service_name=~\"$service_name\", service_instance_id=~\"$service_instance_id\"} | json | detected_level=~\"$log_level\" |~ \"$search\"",
|
|
"refId": "A",
|
|
"editorMode": "code",
|
|
"queryType": "range",
|
|
"direction": "backward"
|
|
}
|
|
],
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"options": {
|
|
"showTime": true,
|
|
"showLabels": false,
|
|
"showCommonLabels": true,
|
|
"wrapLogMessage": true,
|
|
"prettifyLogMessage": false,
|
|
"enableLogDetails": true,
|
|
"enableInfiniteScrolling": false,
|
|
"dedupStrategy": "none",
|
|
"sortOrder": "Descending"
|
|
}
|
|
}
|
|
],
|
|
"schemaVersion": 39,
|
|
"tags": [
|
|
"mattermost",
|
|
"loki",
|
|
"logs"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"definition": "label_values(service_name)",
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"name": "service_name",
|
|
"query": "label_values(service_name)",
|
|
"refresh": 2,
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"datasource": {
|
|
"type": "loki",
|
|
"uid": "${DS_LOKI}"
|
|
},
|
|
"definition": "label_values(service_instance_id)",
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"name": "service_instance_id",
|
|
"query": "label_values(service_instance_id)",
|
|
"refresh": 2,
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"description": "Filter by log severity",
|
|
"hide": 0,
|
|
"includeAll": true,
|
|
"multi": true,
|
|
"name": "log_level",
|
|
"options": [
|
|
{
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "info",
|
|
"value": "info"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "warn",
|
|
"value": "warn"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "error",
|
|
"value": "error"
|
|
}
|
|
],
|
|
"query": "info,warn,error",
|
|
"queryValue": "",
|
|
"refresh": 2,
|
|
"skipUrlSync": false,
|
|
"type": "custom"
|
|
},
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "",
|
|
"value": ""
|
|
},
|
|
"name": "search",
|
|
"options": [],
|
|
"query": "",
|
|
"type": "textbox"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {},
|
|
"timezone": "",
|
|
"title": "Mattermost Log Aggregation",
|
|
"uid": "mattermost-loki-logs",
|
|
"version": 1
|
|
} |