diff --git a/tdrs-backend/docker-compose.yml b/tdrs-backend/docker-compose.yml index 525335d88f..1feed80e48 100644 --- a/tdrs-backend/docker-compose.yml +++ b/tdrs-backend/docker-compose.yml @@ -94,6 +94,25 @@ services: - web - celery-exporter - postgres-exporter + + promtail: + restart: always + image: grafana/promtail:3.0.1 + ports: + - 9080:9080 + volumes: + - ./plg/promtail/config.yml:/etc/promtail/config.yml + - ~/tdp-logs/nginx:/var/log/nginx + - logs:/logs + command: -config.file=/etc/promtail/config.yml + + loki: + restart: always + image: grafana/loki:3.0.1 + ports: + - 3100:3100 + volumes: + - loki_data:/loki celery-exporter: restart: always @@ -157,6 +176,7 @@ services: - ELASTICSEARCH_LOG_INDEX_SLOW_LEVEL volumes: - .:/tdpapp + - logs:/logs image: tdp build: . command: > @@ -189,6 +209,8 @@ volumes: elastic_data: prometheus_data: grafana_data: + loki_data: + logs: networks: diff --git a/tdrs-backend/plg/grafana/dashboards/loki_test_dashboard.json b/tdrs-backend/plg/grafana/dashboards/loki_test_dashboard.json new file mode 100644 index 0000000000..a037bfee77 --- /dev/null +++ b/tdrs-backend/plg/grafana/dashboards/loki_test_dashboard.json @@ -0,0 +1,155 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "name": "Loki", + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "expr": "{job=\"django\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Django Logs", + "type": "logs" + }, + { + "datasource": { + "name": "Loki", + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "expr": "{job=\"nginx\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Nginx Logs", + "type": "logs" + }, + { + "datasource": { + "name": "Loki", + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "L8E80F9AEF21F6940" + }, + "expr": "{job=\"varlogs\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Promtail varlogs", + "type": "logs" + } + ], + "schemaVersion": 39, + "tags": [ + "Loki", + "Logs", + "Nginx", + "Django" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Loki Test", + "uid": "adx2v8wzehla8e", + "version": 1, + "weekStart": "" + } \ No newline at end of file diff --git a/tdrs-backend/plg/grafana/datasources.yml b/tdrs-backend/plg/grafana/datasources.yml index d4295f750c..5cbf39c98f 100644 --- a/tdrs-backend/plg/grafana/datasources.yml +++ b/tdrs-backend/plg/grafana/datasources.yml @@ -4,15 +4,17 @@ datasources: - name: Prometheus type: prometheus access: proxy - # Access mode - proxy (server in the UI) or direct (browser in the UI). url: http://prometheus:9090 jsonData: httpMethod: POST - secureJsonFields: {} - version: 2 - readOnly: true - basicAuth: false - basicAuthUser: "" - withCredentials: false - isDefault: true + isDefault: false + uid: "PBFA97CFB590B2093" typeLogoUrl: "public/app/plugins/datasource/prometheus/img/prometheus_logo.svg" + - name: Loki + type: loki + access: proxy + url: "http://loki:3100" + jsonData: + maxLines: 1000 + isDefault: false + uid: "L8E80F9AEF21F6940" diff --git a/tdrs-backend/plg/promtail/config.yml b/tdrs-backend/plg/promtail/config.yml new file mode 100644 index 0000000000..56e195ca54 --- /dev/null +++ b/tdrs-backend/plg/promtail/config.yml @@ -0,0 +1,32 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: system + static_configs: + - targets: + - localhost + labels: + job: varlogs + __path__: /var/log/*log + - job_name: django + static_configs: + - targets: + - localhost + labels: + job: django + __path__: /logs/django.log + - job_name: nginx + static_configs: + - targets: + - localhost + labels: + job: nginx + __path__: /var/log/nginx/*log diff --git a/tdrs-backend/tdpservice/settings/common.py b/tdrs-backend/tdpservice/settings/common.py index b571ec89e1..d12dce3cab 100644 --- a/tdrs-backend/tdpservice/settings/common.py +++ b/tdrs-backend/tdpservice/settings/common.py @@ -233,30 +233,35 @@ class Common(Configuration): "class": "logging.StreamHandler", "formatter": "color", }, + "file": { + "class": "logging.FileHandler", + "formatter": "verbose", + "filename": '/logs/django.log', + } }, "loggers": { "tdpservice": { - "handlers": ["application"], + "handlers": ["application", "file"], "propagate": True, "level": LOGGING_LEVEL }, "tdpservice.parsers": { - "handlers": ["application"], + "handlers": ["application", "file"], "propagate": False, "level": LOGGING_LEVEL }, - "django": {"handlers": ["console"], "propagate": True}, + "django": {"handlers": ["console", "file"], "propagate": True}, "django.server": { - "handlers": ["django.server"], + "handlers": ["django.server", "file"], "propagate": False, "level": LOGGING_LEVEL }, "django.request": { - "handlers": ["console"], + "handlers": ["console", "file"], "propagate": False, "level": LOGGING_LEVEL }, - "django.db.backends": {"handlers": ["console"], "level": "INFO"}, + "django.db.backends": {"handlers": ["console", "file"], "level": "INFO"}, }, } es_logger = logging.getLogger('elasticsearch') diff --git a/tdrs-frontend/docker-compose.yml b/tdrs-frontend/docker-compose.yml index 75c3eb92bd..3ee327f3e8 100644 --- a/tdrs-frontend/docker-compose.yml +++ b/tdrs-frontend/docker-compose.yml @@ -24,6 +24,7 @@ services: - default volumes: - ./:/home/node/app + - ~/tdp-logs/nginx:/var/log/nginx environment: - NGINX_FRONTEND=tdp-frontend - BACK_END=web diff --git a/tdrs-frontend/nginx/local/default.conf.template b/tdrs-frontend/nginx/local/default.conf.template index 3609680152..15bcecc2f4 100644 --- a/tdrs-frontend/nginx/local/default.conf.template +++ b/tdrs-frontend/nginx/local/default.conf.template @@ -105,7 +105,8 @@ http { ' Proxy Pass: "$proxy_host" "$upstream_addr"'; access_log /dev/stdout compression; - #access_log stderr compression; + access_log /var/log/nginx/access_log.log compression; + error_log /var/log/nginx/error_log.log warn; # Content caching # saves cached fies in /tmp