From a02da0762753dc8bc6f213f18c44bdfef1d8ed8d Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Sun, 8 Oct 2023 21:42:25 +0800 Subject: [PATCH] Test deploy docker log (#105) * chore: try to fix logs in docker * ci: test deploy * ci: log level --- .github/workflows/deploy.test.yml | 6 +- .gitignore | 1 + Cargo.lock | 4 +- Cargo.toml | 6 +- dev.env | 2 +- libs/gotrue-entity/src/lib.rs | 2 + libs/realtime/src/collaborate/group.rs | 15 ++--- src/application.rs | 82 +++++++++++++++----------- src/main.rs | 8 ++- tests/realtime/test_client.rs | 3 +- 10 files changed, 71 insertions(+), 58 deletions(-) diff --git a/.github/workflows/deploy.test.yml b/.github/workflows/deploy.test.yml index b46d74f1..89aef600 100644 --- a/.github/workflows/deploy.test.yml +++ b/.github/workflows/deploy.test.yml @@ -1,7 +1,9 @@ name: Deployment for Test Environment on: push: - branches: main + branches: + - 'main' + - 'test_deploy_*' jobs: deploy: @@ -12,7 +14,7 @@ jobs: - name: Set up env vars run: | cp dev.env .env - + # log level sed -i 's|RUST_LOG=.*|RUST_LOG=trace|' .env diff --git a/.gitignore b/.gitignore index 36e06931..0bd4b23d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ node_modules **/libs/AppFlowy-Collab/ data/ .env +.logs \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 6eaa84e2..9b983960 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -866,7 +866,7 @@ dependencies = [ [[package]] name = "collab" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=489cae0#489cae080419bd327282f974efe49ee447390b1d" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=4a4df0b#4a4df0b287f197db99a02748548a5c0836c91588" dependencies = [ "anyhow", "async-trait", @@ -885,7 +885,7 @@ dependencies = [ [[package]] name = "collab-define" version = "0.1.0" -source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=489cae0#489cae080419bd327282f974efe49ee447390b1d" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-Collab?rev=4a4df0b#4a4df0b287f197db99a02748548a5c0836c91588" dependencies = [ "anyhow", "bytes", diff --git a/Cargo.toml b/Cargo.toml index cb230d66..9c65b4b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ sqlx = { version = "0.7", default-features = false, features = ["runtime-tokio-r async-trait = "0.1.73" # collab -collab = { version = "0.1.0" } +collab = { version = "0.1.0", features = ["async-plugin"] } #Local crate token = { path = "libs/token" } @@ -126,8 +126,8 @@ lto = false opt-level = 3 [patch.crates-io] -collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "489cae0" } -collab-define = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "489cae0" } +collab = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "4a4df0b" } +collab-define = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "4a4df0b" } # Comment the above and uncomment the below to use local version of collab by cloning the repo and placing it in libs folder #collab = { path = "libs/AppFlowy-Collab/collab" } diff --git a/dev.env b/dev.env index 958e478e..bb5ba94d 100644 --- a/dev.env +++ b/dev.env @@ -43,4 +43,4 @@ AWS_SECRET_ACCESS_KEY=minioadmin AWS_S3_BUCKET=appflowy AWS_REGION=us-east-1 -RUST_LOG=info \ No newline at end of file +RUST_LOG=info diff --git a/libs/gotrue-entity/src/lib.rs b/libs/gotrue-entity/src/lib.rs index 4f552b04..0b44280d 100644 --- a/libs/gotrue-entity/src/lib.rs +++ b/libs/gotrue-entity/src/lib.rs @@ -100,6 +100,8 @@ pub struct GoTrueError { pub error_id: Option, } +impl std::error::Error for GoTrueError {} + impl Display for GoTrueError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_fmt(format_args!( diff --git a/libs/realtime/src/collaborate/group.rs b/libs/realtime/src/collaborate/group.rs index 6227e081..b7b51598 100644 --- a/libs/realtime/src/collaborate/group.rs +++ b/libs/realtime/src/collaborate/group.rs @@ -1,21 +1,14 @@ use crate::collaborate::{CollabBroadcast, CollabStoragePlugin, Subscription}; - +use crate::entities::RealtimeUser; +use anyhow::Error; use collab::core::collab::MutexCollab; use collab::core::origin::CollabOrigin; use collab::preclude::Collab; use collab_define::CollabType; - -use std::collections::HashMap; - -use anyhow::Error; - use database::collab::CollabStorage; - +use std::collections::HashMap; use std::sync::Arc; - use tokio::sync::RwLock; - -use crate::entities::RealtimeUser; use tokio::task::spawn_blocking; use tracing::{error, warn}; @@ -115,7 +108,7 @@ where Arc::downgrade(&group), ); collab.lock().add_plugin(Arc::new(plugin)); - collab.async_initialize().await; + collab.lock_arc().initialize().await; self .storage diff --git a/src/application.rs b/src/application.rs index 3a556ff6..33a3079a 100644 --- a/src/application.rs +++ b/src/application.rs @@ -11,6 +11,7 @@ use actix_web::cookie::Key; use actix_web::{dev::Server, web, web::Data, App, HttpServer}; use actix::Actor; +use anyhow::{Context, Error}; use openssl::ssl::{SslAcceptor, SslAcceptorBuilder, SslFiletype, SslMethod}; use openssl::x509::X509; use secrecy::{ExposeSecret, Secret}; @@ -114,17 +115,17 @@ fn get_certificate_and_server_key(config: &Config) -> Option<(Secret, Se } } -pub async fn init_state(config: &Config) -> AppState { - let pg_pool = get_connection_pool(&config.database).await; - migrate(&pg_pool).await; +pub async fn init_state(config: &Config) -> Result { + let pg_pool = get_connection_pool(&config.database).await?; + migrate(&pg_pool).await?; - let s3_bucket = get_aws_s3_bucket(&config.s3).await; - let gotrue_client = get_gotrue_client(&config.gotrue).await; - setup_admin_account(&gotrue_client, &pg_pool, &config.gotrue).await; - let redis_client = get_redis_client(config.redis_uri.expose_secret()).await; - let collab_storage = init_storage(config, pg_pool.clone()).await; + let s3_bucket = get_aws_s3_bucket(&config.s3).await?; + let gotrue_client = get_gotrue_client(&config.gotrue).await?; + setup_admin_account(&gotrue_client, &pg_pool, &config.gotrue).await?; + let redis_client = get_redis_client(config.redis_uri.expose_secret()).await?; + let collab_storage = init_storage(config, pg_pool.clone()).await?; - AppState { + Ok(AppState { pg_pool, config: Arc::new(config.clone()), user: Arc::new(Default::default()), @@ -133,17 +134,20 @@ pub async fn init_state(config: &Config) -> AppState { s3_bucket, redis_client, collab_storage, - } + }) } async fn setup_admin_account( gotrue_client: &gotrue::api::Client, pg_pool: &PgPool, gotrue_setting: &GoTrueSetting, -) { +) -> Result<(), Error> { let admin_email = gotrue_setting.admin_email.as_str(); let password = gotrue_setting.admin_password.as_str(); - gotrue_client.sign_up(admin_email, password).await.unwrap(); + gotrue_client + .sign_up(admin_email, password) + .await + .context("failed to sign-up for admin user")?; // Unable to use query! macro here instead // because of the auth is a not default schema @@ -158,25 +162,30 @@ async fn setup_admin_account( .bind(admin_email) .execute(pg_pool) .await - .unwrap(); + .context("failed to update the admin user")?; + Ok(()) } -async fn get_redis_client(redis_uri: &str) -> redis::aio::ConnectionManager { - redis::Client::open(redis_uri) - .unwrap() +async fn get_redis_client(redis_uri: &str) -> Result { + let manager = redis::Client::open(redis_uri) + .context("failed to connect to redis")? .get_tokio_connection_manager() .await - .unwrap() + .context("failed to get the connection manager")?; + Ok(manager) } -async fn get_aws_s3_bucket(s3_setting: &S3Setting) -> s3::Bucket { +async fn get_aws_s3_bucket(s3_setting: &S3Setting) -> Result { let region = { match s3_setting.use_minio { true => s3::Region::Custom { region: "".to_owned(), endpoint: s3_setting.minio_url.to_owned(), }, - false => s3_setting.region.parse::().unwrap(), + false => s3_setting + .region + .parse::() + .context("failed to parser s3 setting")?, } }; @@ -196,16 +205,14 @@ async fn get_aws_s3_bucket(s3_setting: &S3Setting) -> s3::Bucket { ) .await { - Ok(_) => {}, + Ok(_) => Ok(()), Err(e) => match e { - s3::error::S3Error::Http(409, _) => {}, // Bucket already exists - _ => panic!("Failed to create bucket: {:?}", e), + s3::error::S3Error::Http(409, _) => Ok(()), // Bucket already exists + _ => Err(e), }, - } + }?; - s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone()) - .unwrap() - .with_path_style() + Ok(s3::Bucket::new(&s3_setting.bucket, region.clone(), cred.clone())?.with_path_style()) } // async fn get_aws_s3_client() -> aws_sdk_s3::Client { @@ -226,28 +233,28 @@ async fn get_aws_s3_bucket(s3_setting: &S3Setting) -> s3::Bucket { // client // } -async fn get_connection_pool(setting: &DatabaseSetting) -> PgPool { +async fn get_connection_pool(setting: &DatabaseSetting) -> Result { PgPoolOptions::new() .acquire_timeout(std::time::Duration::from_secs(5)) .connect_with(setting.with_db()) .await - .expect("Failed to connect to Postgres") + .context("failed to connect to postgres database") } -async fn migrate(pool: &PgPool) { +async fn migrate(pool: &PgPool) -> Result<(), Error> { sqlx::migrate!("./migrations") .run(pool) .await - .expect("Failed to run migrations"); + .context("failed to run migrations") } -async fn get_gotrue_client(setting: &GoTrueSetting) -> gotrue::api::Client { +async fn get_gotrue_client(setting: &GoTrueSetting) -> Result { let gotrue_client = gotrue::api::Client::new(reqwest::Client::new(), &setting.base_url); gotrue_client .health() .await - .expect("Failed to connect to GoTrue"); - gotrue_client + .context("failed to connect to GoTrue")?; + Ok(gotrue_client) } fn make_ssl_acceptor_builder(certificate: Secret) -> SslAcceptorBuilder { @@ -269,10 +276,13 @@ fn make_ssl_acceptor_builder(certificate: Secret) -> SslAcceptorBuilder builder } -pub async fn init_storage(_config: &Config, pg_pool: PgPool) -> Storage { +pub async fn init_storage( + _config: &Config, + pg_pool: PgPool, +) -> Result, Error> { let collab_storage = CollabPostgresDBStorageImpl::new(pg_pool); let proxy = CollabStorageProxy::new(collab_storage); - Storage { + Ok(Storage { collab_storage: proxy, - } + }) } diff --git a/src/main.rs b/src/main.rs index 4a44410c..8ebea0ad 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,6 +5,8 @@ use appflowy_cloud::telemetry::{get_subscriber, init_subscriber}; #[actix_web::main] async fn main() -> anyhow::Result<()> { let level = std::env::var("RUST_LOG").unwrap_or("info".to_string()); + println!("Start AppFlowy Cloud with RUST_LOG={}", level); + let mut filters = vec![]; filters.push(format!("actix_web={}", level)); filters.push(format!("collab={}", level)); @@ -21,8 +23,10 @@ async fn main() -> anyhow::Result<()> { ); init_subscriber(subscriber); - let configuration = get_configuration().expect("Failed to read configuration."); - let state = init_state(&configuration).await; + let configuration = get_configuration().expect("The configuration should be configured."); + let state = init_state(&configuration) + .await + .expect("The AppState should be initialized"); let application = Application::build(configuration, state).await?; application.run_until_stopped().await?; diff --git a/tests/realtime/test_client.rs b/tests/realtime/test_client.rs index a62238e1..357f6b71 100644 --- a/tests/realtime/test_client.rs +++ b/tests/realtime/test_client.rs @@ -112,6 +112,7 @@ impl TestClient { .to_string() } + #[allow(clippy::await_holding_lock)] pub(crate) async fn create_collab( &mut self, workspace_id: &str, @@ -143,7 +144,7 @@ impl TestClient { ); collab.lock().add_plugin(Arc::new(sync_plugin)); - collab.async_initialize().await; + collab.lock().initialize().await; let test_collab = TestCollab { origin, collab }; self .collab_by_object_id