From 680d6fe5b17e8c552a6b102a52b9e8fade67c697 Mon Sep 17 00:00:00 2001 From: Fu Zi Xiang Date: Thu, 9 Nov 2023 10:06:42 +0800 Subject: [PATCH] feat: remove moved configuration files --- doc/logging.md | 54 --- doc/monitoring.md | 33 -- docker-compose-logging.yml | 48 --- docker-compose-monitor.yml | 68 ---- docker/filebeat/filebeat.yml | 7 - .../grant_container_logs_permissions.sh | 8 - ...wy_cloud_prometheus_grafana_dashboard.json | 363 ------------------ docker/grafana/datasources/prometheus.yml | 9 - docker/logstash/pipeline/logstash.conf | 14 - docker/prometheus/prometheus.yml | 25 -- 10 files changed, 629 deletions(-) delete mode 100644 doc/logging.md delete mode 100644 doc/monitoring.md delete mode 100644 docker-compose-logging.yml delete mode 100644 docker-compose-monitor.yml delete mode 100644 docker/filebeat/filebeat.yml delete mode 100755 docker/filebeat/grant_container_logs_permissions.sh delete mode 100644 docker/grafana/appflowy_cloud_prometheus_grafana_dashboard.json delete mode 100644 docker/grafana/datasources/prometheus.yml delete mode 100644 docker/logstash/pipeline/logstash.conf delete mode 100644 docker/prometheus/prometheus.yml diff --git a/doc/logging.md b/doc/logging.md deleted file mode 100644 index d9203397..00000000 --- a/doc/logging.md +++ /dev/null @@ -1,54 +0,0 @@ -# Logging -A simple guide to deploy the logging infrastructure for AppFlowy Cloud - -## Components -- Listed in previous directory: `docker-compose-logging.yml` - -## Quick Start -- This assumes that you have already deployed AppFlowy. -- In the previous directory, open a terminal and put the following command: -``` -docker compose --file docker-compose-logging.yml up -d -``` -- Quick check: `docker ps -a` - -## Common deployment problems -### Filebeat did not start successfully -``` -$ docker logs appflowy-cloud-filebeat-1 -Exiting: error loading config file: config file ("filebeat.yml") can only be writable by the owner but the permissions are "-rw-rw-r--" (to fix the permissions use: 'chmod go-w /usr/share/filebeat/filebeat.yml') -``` -- Solution: remove write permission on the file: `chmod -w docker/filebeat/filebeat.yml` - -### No Logs -- Observation: There are no logs in OpenSearch Dashboard -- Possibe Diagnostic: No read permission for `*.log` files in `/var/lib/docker/containers` - -- One Time Solution: give read permission to docker logs -``` -chmod -R a+r /var/lib/docker/containers -``` -- Permanent Solution: give read permission to docker logs every time there's a modification -In the project root directory: `sudo ./docker/filebeat/grant_container_logs_permissions.sh` - - Caveat: Only work on unix like operating system, requires `inotifywait`(`inotify-tools`) to be installed. - MacOS alternative: `fswatch` - -## Credentials -- After deployment, when you go to localhost:5601, both username and password will be `admin` - -## Web UI Setup -- In your browser, go to `localhost:5601` -- Login in with username and password -- Top Left -> Side Panel -> Dashboard Management -> Index Pattern -> Create -- Enter "logstash-logs-*" -> Add Timestamp as filter -> Create - -## Example Usage -- Top Left -> Side Panel -> Discover - -## Final Notes -This is a very simple guide to get started, -You might want to consider the following for security reasons: -- Changing default password -- Use Https protocol instead -- Clustering -For more information, go to Opensearch official website to find out more: [Opensearch](https://opensearch.org/docs/latest/security/configuration/security-admin/) diff --git a/doc/monitoring.md b/doc/monitoring.md deleted file mode 100644 index 05505221..00000000 --- a/doc/monitoring.md +++ /dev/null @@ -1,33 +0,0 @@ -# Monitoring Grafana -A simple guide to deploy the monitoring tools for AppFlowy Cloud - -## Components -- Listed in previous directory: `docker-compose-monitor.yml` - -## Quick Start -- This assumes that you have already deployed AppFlowy. -- Copy the `dev.env` as `.env` if you have not done so -- Change the `GF_SECURITY_ADMIN_USER` and `GF_SECURITY_ADMIN_PASSWORD` field as necessary -- In the previous directory, open a terminal and put the following command: -``` -docker compose --file docker-compose-monitor.yml up -d -``` -- Quick check: `docker ps -a` - -## Web UI Setup -- After deployment, you will have a Grafana dashboard server, sign in with your username and password defined previously -- There no dashboard at the start, you would need to add preconfigured dashboard or configure your own. - -### Steps to import -- Go to myhost/dashboard/import -- Fill in dashboard ID (e.g. 1860 for node-exporter-full) - -### Recommended Publicly available Dashboard -- https://grafana.com/grafana/dashboards/13946-docker-cadvisor/ -- https://grafana.com/grafana/dashboards/1860-node-exporter-full/ -- https://grafana.com/grafana/dashboards/12708-nginx/ - -### AppFlowy Cloud dashboard -- Go to myhost/dashboard/import -- Fill in json model from file (same directory as this doc) - - `appflowy_cloud_prometheus_grafana_dashboard.json` diff --git a/docker-compose-logging.yml b/docker-compose-logging.yml deleted file mode 100644 index 63a141e2..00000000 --- a/docker-compose-logging.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: '3' -services: - # collect logs for docker containers - filebeat: - restart: on-failure - image: docker.elastic.co/beats/filebeat-oss:8.10.4 - volumes: - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock - - ./docker/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml - - # collect scrapped logs - logstash: - restart: on-failure - image: opensearchproject/logstash-oss-with-opensearch-output-plugin - volumes: - - ./docker/logstash/pipeline:/usr/share/logstash/pipeline - - # database to store logs - opensearch: - restart: on-failure - image: opensearchproject/opensearch - environment: - - discovery.type=single-node - - plugins.security.ssl.http.enabled=false - ports: - - 9200:9200 # REST API - - 9600:9600 # Performance Analyzer - volumes: - - opensearch-data:/usr/share/opensearch/data - - # web ui for visualization - opensearch-dashboards: - restart: on-failure - image: opensearchproject/opensearch-dashboards - ports: - - 5601:5601 - environment: - - OPENSEARCH_HOSTS=http://opensearch:9200 - volumes: - - opensearch-dashboards-data:/usr/share/opensearch-dashboards/data - -# logs persistent -volumes: - opensearch-data: - external: false - opensearch-dashboards-data: - external: false diff --git a/docker-compose-monitor.yml b/docker-compose-monitor.yml deleted file mode 100644 index 988c76aa..00000000 --- a/docker-compose-monitor.yml +++ /dev/null @@ -1,68 +0,0 @@ -version: '3' -services: - # Database, Scraper - prometheus: - image: prom/prometheus - ports: - - 9090:9090 - volumes: - - prometheus-data:/prometheus # This volume stores the actual Prometheus data - - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - command: - - --config.file=/etc/prometheus/prometheus.yml - - --enable-feature=exemplar-storage - - # Web UI - grafana: - image: grafana/grafana - restart: unless-stopped - ports: - - 3001:3000 - environment: - - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER} - - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} - - GF_USERS_ALLOW_SIGN_UP=false - volumes: - - grafana-data:/var/lib/grafana # This volume stores Grafana's data - - ./docker/grafana/datasources/:/etc/grafana/provisioning/datasources/ - - # Scrapped by Prometheus for docker container - cadvisor: - image: gcr.io/cadvisor/cadvisor - ports: - - 8080:8080 - volumes: - - /:/rootfs:ro - - /var/run:/var/run:rw - - /sys:/sys:ro - - /var/lib/docker/:/var/lib/docker:ro - - # Scrapped by Prometheus for host machine - node-exporter: - image: prom/node-exporter - restart: unless-stopped - ports: - - 9100:9100 - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - '--path.procfs=/host/proc' - - '--path.rootfs=/rootfs' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - - # Scrapped by Prometheus for nginx routing - nginx-exporter: - image: nginx/nginx-prometheus-exporter - restart: unless-stopped - command: "-nginx.scrape-uri=http://nginx:8080/stub_status" - ports: - - 9114:9113 - -volumes: - prometheus-data: - external: false - grafana-data: - external: false diff --git a/docker/filebeat/filebeat.yml b/docker/filebeat/filebeat.yml deleted file mode 100644 index a7c7f8ed..00000000 --- a/docker/filebeat/filebeat.yml +++ /dev/null @@ -1,7 +0,0 @@ -filebeat.inputs: -- type: container - paths: - - '/var/lib/docker/containers/*/*.log' - -output.logstash: - hosts: ["logstash:5044"] diff --git a/docker/filebeat/grant_container_logs_permissions.sh b/docker/filebeat/grant_container_logs_permissions.sh deleted file mode 100755 index 89eb7455..00000000 --- a/docker/filebeat/grant_container_logs_permissions.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /usr/bin/env bash - -while true -do - inotifywait /var/lib/docker/containers - sleep 1 - sudo chmod -R a+r /var/lib/docker/containers -done diff --git a/docker/grafana/appflowy_cloud_prometheus_grafana_dashboard.json b/docker/grafana/appflowy_cloud_prometheus_grafana_dashboard.json deleted file mode 100644 index 4d69c7ae..00000000 --- a/docker/grafana/appflowy_cloud_prometheus_grafana_dashboard.json +++ /dev/null @@ -1,363 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.2.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "dashboard to monitor appflowy cloud server", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": null, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "(rate of total latency)/(rate of total request made) for every endpoint", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(appflowy_cloud_requests_latency_total[$__interval])/rate(appflowy_cloud_requests_count_total[$__interval])", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Average Response(ms) for endpoints", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "description": "rate of http request with returned response of non 200", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "exemplar": true, - "expr": "rate(appflowy_cloud_requests_result_total{status_code!~\"2..\"}[$__interval])\n", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "non 200 http status code for every endpoint", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 16 - }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "rate(appflowy_cloud_requests_count_total[$__interval])", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Rate of requests per second for each endpoint", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 38, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-30m", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "AppFlowy Cloud", - "uid": "a973a62f-fd57-4ae1-b0ee-2f206e872b74", - "version": 7, - "weekStart": "" -} diff --git a/docker/grafana/datasources/prometheus.yml b/docker/grafana/datasources/prometheus.yml deleted file mode 100644 index 9916d4da..00000000 --- a/docker/grafana/datasources/prometheus.yml +++ /dev/null @@ -1,9 +0,0 @@ -datasources: -- name: Prometheus - access: proxy - type: prometheus - url: http://prometheus:9090 - jsonData: - exemplarTraceIdDestinations: - - name: trace_id - isDefault: true diff --git a/docker/logstash/pipeline/logstash.conf b/docker/logstash/pipeline/logstash.conf deleted file mode 100644 index da12b3b5..00000000 --- a/docker/logstash/pipeline/logstash.conf +++ /dev/null @@ -1,14 +0,0 @@ -input { - beats { - port => 5044 - } -} - -output { - opensearch { - hosts => ["http://opensearch"] - user => "admin" - password => "admin" - index => "logstash-logs-%{+YYYY.MM.dd}" - } -} diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml deleted file mode 100644 index ec86f776..00000000 --- a/docker/prometheus/prometheus.yml +++ /dev/null @@ -1,25 +0,0 @@ -scrape_configs: -- job_name: cadvisor - scrape_interval: 5s - static_configs: - - targets: [cadvisor:8080] - -- job_name: node-exporter - scrape_interval: 5s - static_configs: - - targets: [node-exporter:9100] - -- job_name: appflowy-cloud - scrape_interval: 5s - static_configs: - - targets: [appflowy_cloud:8000] - -- job_name: gotrue - scrape_interval: 5s - static_configs: - - targets: [gotrue:9100] - -- job_name: nginx-exporter - scrape_interval: 5s - static_configs: - - targets: [nginx-exporter:9113]