chk
/
AppFlowy-Cloud
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: remove moved configuration files

This commit is contained in:
Fu Zi Xiang 2023-11-09 10:06:42 +08:00
parent d37fbbf486
commit 680d6fe5b1
No known key found for this signature in database
10 changed files with 0 additions and 629 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
doc/logging.md
View File

@ -1,54 +0,0 @@
# Logging
A simple guide to deploy the logging infrastructure for AppFlowy Cloud
## Components
- Listed in previous directory: `docker-compose-logging.yml`
## Quick Start
- This assumes that you have already deployed AppFlowy.
- In the previous directory, open a terminal and put the following command:
```
docker compose --file docker-compose-logging.yml up -d
```
- Quick check: `docker ps -a`
## Common deployment problems
### Filebeat did not start successfully
```
$ docker logs appflowy-cloud-filebeat-1
Exiting: error loading config file: config file ("filebeat.yml") can only be writable by the owner but the permissions are "-rw-rw-r--" (to fix the permissions use: 'chmod go-w /usr/share/filebeat/filebeat.yml')
```
- Solution: remove write permission on the file: `chmod -w docker/filebeat/filebeat.yml`
### No Logs
- Observation: There are no logs in OpenSearch Dashboard
- Possibe Diagnostic: No read permission for `*.log` files in `/var/lib/docker/containers`
- One Time Solution: give read permission to docker logs
```
chmod -R a+r /var/lib/docker/containers
```
- Permanent Solution: give read permission to docker logs every time there's a modification
In the project root directory: `sudo ./docker/filebeat/grant_container_logs_permissions.sh`
- Caveat: Only work on unix like operating system, requires `inotifywait`(`inotify-tools`) to be installed.
MacOS alternative: `fswatch`
## Credentials
- After deployment, when you go to localhost:5601, both username and password will be `admin`
## Web UI Setup
- In your browser, go to `localhost:5601`
- Login in with username and password
- Top Left -> Side Panel -> Dashboard Management -> Index Pattern -> Create
- Enter "logstash-logs-*" -> Add Timestamp as filter -> Create
## Example Usage
- Top Left -> Side Panel -> Discover
## Final Notes
This is a very simple guide to get started,
You might want to consider the following for security reasons:
- Changing default password
- Use Https protocol instead
- Clustering
For more information, go to Opensearch official website to find out more: [Opensearch](https://opensearch.org/docs/latest/security/configuration/security-admin/)

33
doc/monitoring.md
View File

@ -1,33 +0,0 @@
# Monitoring Grafana
A simple guide to deploy the monitoring tools for AppFlowy Cloud
## Components
- Listed in previous directory: `docker-compose-monitor.yml`
## Quick Start
- This assumes that you have already deployed AppFlowy.
- Copy the `dev.env` as `.env` if you have not done so
- Change the `GF_SECURITY_ADMIN_USER` and `GF_SECURITY_ADMIN_PASSWORD` field as necessary
- In the previous directory, open a terminal and put the following command:
```
docker compose --file docker-compose-monitor.yml up -d
```
- Quick check: `docker ps -a`
## Web UI Setup
- After deployment, you will have a Grafana dashboard server, sign in with your username and password defined previously
- There no dashboard at the start, you would need to add preconfigured dashboard or configure your own.
### Steps to import
- Go to myhost/dashboard/import
- Fill in dashboard ID (e.g. 1860 for node-exporter-full)
### Recommended Publicly available Dashboard
- https://grafana.com/grafana/dashboards/13946-docker-cadvisor/
- https://grafana.com/grafana/dashboards/1860-node-exporter-full/
- https://grafana.com/grafana/dashboards/12708-nginx/
### AppFlowy Cloud dashboard
- Go to myhost/dashboard/import
- Fill in json model from file (same directory as this doc)
- `appflowy_cloud_prometheus_grafana_dashboard.json`

48
docker-compose-logging.yml
View File

@ -1,48 +0,0 @@
version: '3'
services:
# collect logs for docker containers
filebeat:
restart: on-failure
image: docker.elastic.co/beats/filebeat-oss:8.10.4
volumes:
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock
- ./docker/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
# collect scrapped logs
logstash:
restart: on-failure
image: opensearchproject/logstash-oss-with-opensearch-output-plugin
volumes:
- ./docker/logstash/pipeline:/usr/share/logstash/pipeline
# database to store logs
opensearch:
restart: on-failure
image: opensearchproject/opensearch
environment:
- discovery.type=single-node
- plugins.security.ssl.http.enabled=false
ports:
- 9200:9200 # REST API
- 9600:9600 # Performance Analyzer
volumes:
- opensearch-data:/usr/share/opensearch/data
# web ui for visualization
opensearch-dashboards:
restart: on-failure
image: opensearchproject/opensearch-dashboards
ports:
- 5601:5601
environment:
- OPENSEARCH_HOSTS=http://opensearch:9200
volumes:
- opensearch-dashboards-data:/usr/share/opensearch-dashboards/data
# logs persistent
volumes:
opensearch-data:
external: false
opensearch-dashboards-data:
external: false

68
docker-compose-monitor.yml
View File

@ -1,68 +0,0 @@
version: '3'
services:
# Database, Scraper
prometheus:
image: prom/prometheus
ports:
- 9090:9090
volumes:
- prometheus-data:/prometheus # This volume stores the actual Prometheus data
- ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
command:
- --config.file=/etc/prometheus/prometheus.yml
- --enable-feature=exemplar-storage
# Web UI
grafana:
image: grafana/grafana
restart: unless-stopped
ports:
- 3001:3000
environment:
- GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER}
- GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
- GF_USERS_ALLOW_SIGN_UP=false
volumes:
- grafana-data:/var/lib/grafana # This volume stores Grafana's data
- ./docker/grafana/datasources/:/etc/grafana/provisioning/datasources/
# Scrapped by Prometheus for docker container
cadvisor:
image: gcr.io/cadvisor/cadvisor
ports:
- 8080:8080
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
# Scrapped by Prometheus for host machine
node-exporter:
image: prom/node-exporter
restart: unless-stopped
ports:
- 9100:9100
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
# Scrapped by Prometheus for nginx routing
nginx-exporter:
image: nginx/nginx-prometheus-exporter
restart: unless-stopped
command: "-nginx.scrape-uri=http://nginx:8080/stub_status"
ports:
- 9114:9113
volumes:
prometheus-data:
external: false
grafana-data:
external: false

7
docker/filebeat/filebeat.yml
View File

@ -1,7 +0,0 @@
filebeat.inputs:
- type: container
paths:
- '/var/lib/docker/containers/*/*.log'
output.logstash:
hosts: ["logstash:5044"]

8
docker/filebeat/grant_container_logs_permissions.sh
View File

@ -1,8 +0,0 @@
#! /usr/bin/env bash
while true
do
inotifywait /var/lib/docker/containers
sleep 1
sudo chmod -R a+r /var/lib/docker/containers
done

363
docker/grafana/appflowy_cloud_prometheus_grafana_dashboard.json
View File

@ -1,363 +0,0 @@
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "10.2.0"
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "dashboard to monitor appflowy cloud server",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "(rate of total latency)/(rate of total request made) for every endpoint",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": true,
"expr": "rate(appflowy_cloud_requests_latency_total[$__interval])/rate(appflowy_cloud_requests_count_total[$__interval])",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Average Response(ms) for endpoints",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"description": "rate of http request with returned response of non 200",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"exemplar": true,
"expr": "rate(appflowy_cloud_requests_result_total{status_code!~\"2..\"}[$__interval])\n",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "non 200 http status code for every endpoint",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 16
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${DS_PROMETHEUS}"
},
"editorMode": "code",
"expr": "rate(appflowy_cloud_requests_count_total[$__interval])",
"instant": false,
"legendFormat": "__auto",
"range": true,
"refId": "A"
}
],
"title": "Rate of requests per second for each endpoint",
"type": "timeseries"
}
],
"refresh": "",
"schemaVersion": 38,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "AppFlowy Cloud",
"uid": "a973a62f-fd57-4ae1-b0ee-2f206e872b74",
"version": 7,
"weekStart": ""
}

9
docker/grafana/datasources/prometheus.yml
View File

@ -1,9 +0,0 @@
datasources:
- name: Prometheus
access: proxy
type: prometheus
url: http://prometheus:9090
jsonData:
exemplarTraceIdDestinations:
- name: trace_id
isDefault: true

14
docker/logstash/pipeline/logstash.conf
View File

@ -1,14 +0,0 @@
input {
beats {
port => 5044
}
}
output {
opensearch {
hosts => ["http://opensearch"]
user => "admin"
password => "admin"
index => "logstash-logs-%{+YYYY.MM.dd}"
}
}

25
docker/prometheus/prometheus.yml
View File

@ -1,25 +0,0 @@
scrape_configs:
- job_name: cadvisor
scrape_interval: 5s
static_configs:
- targets: [cadvisor:8080]
- job_name: node-exporter
scrape_interval: 5s
static_configs:
- targets: [node-exporter:9100]
- job_name: appflowy-cloud
scrape_interval: 5s
static_configs:
- targets: [appflowy_cloud:8000]
- job_name: gotrue
scrape_interval: 5s
static_configs:
- targets: [gotrue:9100]
- job_name: nginx-exporter
scrape_interval: 5s
static_configs:
- targets: [nginx-exporter:9113]