From 14b2f3c9853690d9fb518d95de1ec4c7d0bd22a0 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Wed, 11 Dec 2024 09:43:47 +0800 Subject: [PATCH] chore: enable running stress on on self-hosted runner (#1053) * chore: run stress test on selfhost runner * chore: Update stress_test.yml * chore: update env * chore: skip stress_test when running integration test --- .github/workflows/integration_test.yml | 2 +- .github/workflows/stress_test.yml | 49 +++++++++++ Cargo.lock | 9 +- deploy.env | 11 ++- docker-compose-stress-test.yml | 97 +++++++++++++++++++++ tests/collab/stress_test.rs | 2 +- xtask/Cargo.toml | 3 +- xtask/src/main.rs | 115 ++++++++++++++++++++----- 8 files changed, 258 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/stress_test.yml create mode 100644 docker-compose-stress-test.yml diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index e420f6ed..5d2a2308 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -130,7 +130,7 @@ jobs: - name: Run Tests run: | echo "Running tests for ${{ matrix.test_service }} with flags: ${{ matrix.test_cmd }}" - RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }} + RUST_LOG="info" DISABLE_CI_TEST_LOG="true" cargo test ${{ matrix.test_cmd }} -- --skip stress_test - name: Docker Logs if: always() diff --git a/.github/workflows/stress_test.yml b/.github/workflows/stress_test.yml new file mode 100644 index 00000000..267a15ee --- /dev/null +++ b/.github/workflows/stress_test.yml @@ -0,0 +1,49 @@ +name: AppFlowy-Cloud Stress Test + +on: [ pull_request ] + +concurrency: + group: stress-test-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +env: + POSTGRES_HOST: localhost + REDIS_HOST: localhost + MINIO_HOST: localhost + SQLX_OFFLINE: true + RUST_TOOLCHAIN: "1.78" + +jobs: + test: + name: Collab Stress Tests + runs-on: self-hosted-appflowy3 + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Install Rust Toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Copy and Rename deploy.env to .env + run: cp deploy.env .env + + - name: Replace Values in .env + run: | + sed -i '' 's|RUST_LOG=.*|RUST_LOG=debug|' .env + sed -i '' 's|API_EXTERNAL_URL=.*|API_EXTERNAL_URL=http://localhost:9999|' .env + sed -i '' 's|APPFLOWY_GOTRUE_BASE_URL=.*|APPFLOWY_GOTRUE_BASE_URL=http://localhost:9999|' .env + shell: bash + + - name: Start Docker Compose Services + run: | + docker compose -f docker-compose-stress-test.yml up -d + docker ps -a + + - name: Install Prerequisites + run: | + brew install protobuf + + - name: Run Server and Test + run: | + cargo run --package xtask -- --stress-test diff --git a/Cargo.lock b/Cargo.lock index 32e2f1c0..2003d484 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3342,9 +3342,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3373,9 +3373,9 @@ checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -8399,6 +8399,7 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "futures", "tokio", ] diff --git a/deploy.env b/deploy.env index afae1ebf..8ec07874 100644 --- a/deploy.env +++ b/deploy.env @@ -4,7 +4,7 @@ # PostgreSQL Settings POSTGRES_HOST=postgres POSTGRES_USER=postgres -POSTGRES_PASSWORD=changepassword +POSTGRES_PASSWORD=password POSTGRES_PORT=5432 POSTGRES_DB=postgres @@ -15,6 +15,10 @@ SUPABASE_PASSWORD=root REDIS_HOST=redis REDIS_PORT=6379 +# Minio Host +MINIO_HOST=minio +MINIO_PORT=9000 + # AppFlowy Cloud ## URL that connects to the gotrue docker container APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999 @@ -66,11 +70,12 @@ GOTRUE_ADMIN_PASSWORD=password # If you are using a different domain, you need to change the redirect_uri in the OAuth2 configuration # Make sure that this domain is accessible to the user # Make sure no endswith / +# Replace with your host name instead of localhost API_EXTERNAL_URL=http://your-host # In docker environment, `postgres` is the hostname of the postgres service # GoTrue connect to postgres using this url -GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:${SUPABASE_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} +GOTRUE_DATABASE_URL=postgres://supabase_auth_admin:${SUPABASE_PASSWORD}@postgres:${POSTGRES_PORT}/${POSTGRES_DB} # Refer to this for details: https://github.com/AppFlowy-IO/AppFlowy-Cloud/blob/main/doc/AUTHENTICATION.md # Google OAuth2 @@ -102,7 +107,7 @@ APPFLOWY_S3_CREATE_BUCKET=true # By default, Minio is used as the default file storage which uses host's file system. # Keep this as true if you are using other S3 compatible storage provider other than AWS. APPFLOWY_S3_USE_MINIO=true -APPFLOWY_S3_MINIO_URL=http://minio:9000 # change this if you are using a different address for minio +APPFLOWY_S3_MINIO_URL=http://${MINIO_HOST}:${MINIO_PORT} # change this if you are using a different address for minio APPFLOWY_S3_ACCESS_KEY=minioadmin APPFLOWY_S3_SECRET_KEY=minioadmin APPFLOWY_S3_BUCKET=appflowy diff --git a/docker-compose-stress-test.yml b/docker-compose-stress-test.yml new file mode 100644 index 00000000..58a65e64 --- /dev/null +++ b/docker-compose-stress-test.yml @@ -0,0 +1,97 @@ +services: + nginx: + restart: on-failure + image: nginx + ports: + - 80:80 # Disable this if you are using TLS + - 443:443 + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./nginx/ssl/certificate.crt:/etc/nginx/ssl/certificate.crt + - ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key + minio: + restart: on-failure + image: minio/minio + ports: + - 9000:9000 + - 9001:9001 + environment: + - MINIO_BROWSER_REDIRECT_URL=http://localhost:9001 + command: server /data --console-address ":9001" + + postgres: + restart: on-failure + image: pgvector/pgvector:pg16 + environment: + - POSTGRES_USER=${POSTGRES_USER:-postgres} + - POSTGRES_DB=${POSTGRES_DB:-postgres} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} + - POSTGRES_HOST=${POSTGRES_HOST:-postgres} + - SUPABASE_USER=${SUPABASE_USER:-supabase_auth_admin} + - SUPABASE_PASSWORD=${SUPABASE_PASSWORD:-root} + ports: + - 5432:5432 + volumes: + - ./migrations/before:/docker-entrypoint-initdb.d + # comment out the following line if you want to persist data when restarting docker + #- postgres_data:/var/lib/postgresql/data + + redis: + restart: on-failure + image: redis + ports: + - 6379:6379 + + gotrue: + restart: on-failure + image: supabase/gotrue:v2.159.1 + depends_on: + - postgres + environment: + # Gotrue config: https://github.com/supabase/gotrue/blob/master/example.env + - GOTRUE_SITE_URL=appflowy-flutter:// # redirected to AppFlowy application + - URI_ALLOW_LIST=* # adjust restrict if necessary + - GOTRUE_JWT_SECRET=${GOTRUE_JWT_SECRET} # authentication secret + - GOTRUE_JWT_EXP=${GOTRUE_JWT_EXP} + - GOTRUE_DB_DRIVER=postgres + - API_EXTERNAL_URL=${API_EXTERNAL_URL} + - DATABASE_URL=${GOTRUE_DATABASE_URL} + - PORT=9999 + - GOTRUE_MAILER_URLPATHS_CONFIRMATION=/verify + - GOTRUE_SMTP_HOST=${GOTRUE_SMTP_HOST} # e.g. smtp.gmail.com + - GOTRUE_SMTP_PORT=${GOTRUE_SMTP_PORT} # e.g. 465 + - GOTRUE_SMTP_USER=${GOTRUE_SMTP_USER} # email sender, e.g. noreply@appflowy.io + - GOTRUE_SMTP_PASS=${GOTRUE_SMTP_PASS} # email password + - GOTRUE_SMTP_ADMIN_EMAIL=${GOTRUE_SMTP_ADMIN_EMAIL} # email with admin privileges e.g. internal@appflowy.io + - GOTRUE_SMTP_MAX_FREQUENCY=${GOTRUE_SMTP_MAX_FREQUENCY:-1ns} # set to 1ns for running tests + - GOTRUE_RATE_LIMIT_EMAIL_SENT=${GOTRUE_RATE_LIMIT_EMAIL_SENT:-100} # number of email sendable per minute + - GOTRUE_MAILER_AUTOCONFIRM=${GOTRUE_MAILER_AUTOCONFIRM:-false} # change this to true to skip email confirmation + # Google OAuth config + - GOTRUE_EXTERNAL_GOOGLE_ENABLED=${GOTRUE_EXTERNAL_GOOGLE_ENABLED} + - GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID=${GOTRUE_EXTERNAL_GOOGLE_CLIENT_ID} + - GOTRUE_EXTERNAL_GOOGLE_SECRET=${GOTRUE_EXTERNAL_GOOGLE_SECRET} + - GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI=${GOTRUE_EXTERNAL_GOOGLE_REDIRECT_URI} + # Apple OAuth config + - GOTRUE_EXTERNAL_APPLE_ENABLED=${GOTRUE_EXTERNAL_APPLE_ENABLED} + - GOTRUE_EXTERNAL_APPLE_CLIENT_ID=${GOTRUE_EXTERNAL_APPLE_CLIENT_ID} + - GOTRUE_EXTERNAL_APPLE_SECRET=${GOTRUE_EXTERNAL_APPLE_SECRET} + - GOTRUE_EXTERNAL_APPLE_REDIRECT_URI=${GOTRUE_EXTERNAL_APPLE_REDIRECT_URI} + # GITHUB OAuth config + - GOTRUE_EXTERNAL_GITHUB_ENABLED=${GOTRUE_EXTERNAL_GITHUB_ENABLED} + - GOTRUE_EXTERNAL_GITHUB_CLIENT_ID=${GOTRUE_EXTERNAL_GITHUB_CLIENT_ID} + - GOTRUE_EXTERNAL_GITHUB_SECRET=${GOTRUE_EXTERNAL_GITHUB_SECRET} + - GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI=${GOTRUE_EXTERNAL_GITHUB_REDIRECT_URI} + # Discord OAuth config + - GOTRUE_EXTERNAL_DISCORD_ENABLED=${GOTRUE_EXTERNAL_DISCORD_ENABLED} + - GOTRUE_EXTERNAL_DISCORD_CLIENT_ID=${GOTRUE_EXTERNAL_DISCORD_CLIENT_ID} + - GOTRUE_EXTERNAL_DISCORD_SECRET=${GOTRUE_EXTERNAL_DISCORD_SECRET} + - GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI=${GOTRUE_EXTERNAL_DISCORD_REDIRECT_URI} + # Prometheus Metrics + - GOTRUE_METRICS_ENABLED=true + - GOTRUE_METRICS_EXPORTER=prometheus + - GOTRUE_MAILER_TEMPLATES_CONFIRMATION=${GOTRUE_MAILER_TEMPLATES_CONFIRMATION} + ports: + - 9999:9999 + +volumes: + postgres_data: diff --git a/tests/collab/stress_test.rs b/tests/collab/stress_test.rs index 9133b98f..7dacb1d9 100644 --- a/tests/collab/stress_test.rs +++ b/tests/collab/stress_test.rs @@ -11,7 +11,7 @@ use client_api_test::{assert_server_collab, TestClient}; use database_entity::dto::AFRole; #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn run_multiple_text_edits() { +async fn stress_test_run_multiple_text_edits() { const READER_COUNT: usize = 1; let test_scenario = Arc::new(TestScenario::open( "./tests/collab/asset/automerge-paper.json.gz", diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 284e1869..728e6a28 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" [dependencies] anyhow = "1.0" -tokio = { version = "1", features = ["full"] } \ No newline at end of file +tokio = { version = "1", features = ["full"] } +futures = "0.3.31" \ No newline at end of file diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0c2ed40f..87d9e66a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,48 +1,112 @@ use anyhow::{anyhow, Context, Result}; +use std::process::Stdio; use tokio::process::Command; use tokio::select; +use tokio::time::{sleep, Duration}; -/// Using 'cargo run --package xtask' to run servers in parallel. -/// 1. AppFlowy Cloud -/// 2. AppFlowy History -/// 3. AppFlowy Indexer +/// Run servers: +/// cargo run --package xtask /// -/// Before running this command, make sure the other dependencies servers are running. For example, -/// Redis, Postgres, etc. +/// Run servers and stress tests: +/// cargo run --package xtask -- --stress-test +/// +/// Note: test start with 'stress_test' will be run as stress tests #[tokio::main] async fn main() -> Result<()> { + let is_stress_test = std::env::args().any(|arg| arg == "--stress-test"); + let appflowy_cloud_bin_name = "appflowy_cloud"; - let worker = "appflowy_worker"; + let worker_bin_name = "appflowy_worker"; + // Step 1: Kill existing processes kill_existing_process(appflowy_cloud_bin_name).await?; - kill_existing_process(worker).await?; + kill_existing_process(worker_bin_name).await?; - let mut appflowy_cloud_cmd = Command::new("cargo") - .args(["run", "--features", "history"]) - .spawn() - .context("Failed to start AppFlowy-Cloud process")?; + // Step 2: Start servers sequentially + println!("Starting {} server...", appflowy_cloud_bin_name); + let mut appflowy_cloud_cmd = spawn_server( + "cargo", + &["run", "--features", "history"], + appflowy_cloud_bin_name, + is_stress_test, + )?; + wait_for_readiness(appflowy_cloud_bin_name).await?; - let mut appflowy_worker_cmd = Command::new("cargo") - .args([ + println!("Starting {} server...", worker_bin_name); + let mut appflowy_worker_cmd = spawn_server( + "cargo", + &[ "run", "--manifest-path", "./services/appflowy-worker/Cargo.toml", - ]) - .spawn() - .context("Failed to start AppFlowy-Worker process")?; + ], + worker_bin_name, + is_stress_test, + )?; + wait_for_readiness(worker_bin_name).await?; + println!("All servers are up and running."); + + // Step 3: Run stress tests if flag is set + let stress_test_cmd = if is_stress_test { + println!("Running stress tests (tests starting with 'stress_test')..."); + Some( + Command::new("cargo") + .args(["test", "stress_test", "--", "--nocapture"]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn() + .context("Failed to start stress test process")?, + ) + } else { + None + }; + + // Step 4: Monitor all processes select! { status = appflowy_cloud_cmd.wait() => { handle_process_exit(status?, appflowy_cloud_bin_name)?; }, status = appflowy_worker_cmd.wait() => { - handle_process_exit(status?, worker)?; - } + handle_process_exit(status?, worker_bin_name)?; + }, + status = async { + if let Some(mut stress_cmd) = stress_test_cmd { + stress_cmd.wait().await + } else { + futures::future::pending().await + } + } => { + if is_stress_test { + handle_process_exit(status?, "cargo test stress_test")?; + } + }, } Ok(()) } +fn spawn_server( + command: &str, + args: &[&str], + name: &str, + suppress_output: bool, +) -> Result { + println!("Spawning {} process...", name); + let mut cmd = Command::new(command); + cmd.args(args); + + if suppress_output { + cmd.stdout(Stdio::null()).stderr(Stdio::null()); + } + + Ok( + cmd + .spawn() + .context(format!("Failed to start {} process", name))?, + ) +} + async fn kill_existing_process(process_identifier: &str) -> Result<()> { let _ = Command::new("pkill") .arg("-f") @@ -59,6 +123,17 @@ fn handle_process_exit(status: std::process::ExitStatus, process_name: &str) -> println!("{} exited normally.", process_name); Ok(()) } else { - Err(anyhow!("{} process failed", process_name)) + Err(anyhow!( + "{} process failed with code {}", + process_name, + status + )) } } + +async fn wait_for_readiness(process_name: &str) -> Result<()> { + println!("Waiting for {} to be ready...", process_name); + sleep(Duration::from_secs(3)).await; + println!("{} is ready.", process_name); + Ok(()) +}