From a7b259ad205bc3c1ddbf644d8618f77d858777a3 Mon Sep 17 00:00:00 2001 From: Zack <33050391+speed2exe@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:45:42 -0800 Subject: [PATCH] feat: use only env var for appflowy cloud (#224) * feat: use only env var for appflowy cloud * fix: jwt local testing * fix: security audit * feat: update docker deploy configs * fix: test utils dotenvy * fix: test try sqlx offline * fix: add gotrue configs for appflowy * fix: redis uri in docker --- .github/workflows/docker.yml | 2 +- Cargo.lock | 65 +++++--------- Cargo.toml | 4 +- Dockerfile | 4 +- dev.env | 2 +- doc/DEPLOYMENT.md | 4 + docker-compose.yml | 25 +++--- src/application.rs | 21 ++--- src/config/config.rs | 162 ++++++++++++++++------------------- src/main.rs | 25 +++--- tests/user/utils.rs | 2 +- 11 files changed, 146 insertions(+), 170 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3d5b0ff2..7833230a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -43,7 +43,7 @@ jobs: - name: Run Docker-Compose run: | - docker compose up -d + docker compose up -d - name: Run tests run: | diff --git a/Cargo.lock b/Cargo.lock index 1aa62465..4ead858f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -523,11 +523,10 @@ dependencies = [ "collab", "collab-entity", "collab-folder", - "config", "database", "database-entity", "derive_more", - "dotenv", + "dotenvy", "fancy-regex", "futures", "futures-util", @@ -571,6 +570,7 @@ dependencies = [ "tracing-log", "tracing-subscriber", "unicode-segmentation", + "url", "uuid", "validator", "workspace-template", @@ -992,7 +992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b" dependencies = [ "borsh-derive", - "hashbrown 0.11.2", + "hashbrown 0.12.3", ] [[package]] @@ -1403,20 +1403,6 @@ dependencies = [ "tokio-util", ] -[[package]] -name = "config" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7" -dependencies = [ - "async-trait", - "lazy_static", - "nom", - "pathdiff", - "serde", - "yaml-rust", -] - [[package]] name = "const-oid" version = "0.9.5" @@ -2074,9 +2060,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -2593,6 +2579,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_chain" version = "1.0.2" @@ -2807,12 +2803,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.4.7" @@ -3317,12 +3307,6 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" -[[package]] -name = "pathdiff" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" - [[package]] name = "pem" version = "1.1.1" @@ -3343,9 +3327,9 @@ dependencies = [ [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -5698,12 +5682,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna 0.5.0", "percent-encoding", ] @@ -6136,15 +6120,6 @@ dependencies = [ "time", ] -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "yasna" version = "0.5.2" diff --git a/Cargo.toml b/Cargo.toml index ad52407f..89d03207 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ tokio-stream = "0.1.14" tokio-util = { version = "0.7.9", features = ["io"] } futures = "0.3.17" futures-util ={ version = "0.3.26" , features = ["std","io"] } -config = { version = "0.13.3", default-features = false, features = ["yaml"] } once_cell = "1.13.0" chrono = { version = "0.4.23", features = ["serde", "clock"], default-features = false } derive_more = { version = "0.99" } @@ -67,6 +66,8 @@ uuid = "1.4.1" tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] } prost = "0.12.1" casbin = { version = "2.0.9" } +dotenvy = "0.15.7" +url = "2.5.0" # collab collab = { version = "0.1.0", features = ["async-plugin"] } @@ -91,7 +92,6 @@ realtime-entity.workspace = true once_cell = "1.7.2" tempfile = "3.4.0" assert-json-diff = "2.0.2" -dotenv = "0.15.0" scraper = "0.17.1" client-api = { path = "libs/client-api", features = ["collab-sync", "test_util"] } opener = "0.6.1" diff --git a/Dockerfile b/Dockerfile index c8673dad..d4019557 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukemathwalker/cargo-chef:latest-rust-1.69.0 as chef +FROM lukemathwalker/cargo-chef:latest-rust-1.74.0 as chef WORKDIR /app RUN apt update && apt install lld clang -y @@ -18,7 +18,7 @@ ENV SQLX_OFFLINE true # Build the project RUN cargo build --release --bin appflowy_cloud -FROM debian:bullseye-slim AS runtime +FROM debian:bookworm-slim AS runtime WORKDIR /app RUN apt-get update -y \ && apt-get install -y --no-install-recommends openssl \ diff --git a/dev.env b/dev.env index f7fe31f9..145a4c28 100644 --- a/dev.env +++ b/dev.env @@ -15,7 +15,7 @@ GOTRUE_SMTP_HOST=smtp.gmail.com GOTRUE_SMTP_PORT=465 GOTRUE_SMTP_USER=email_sender@some_company.com GOTRUE_SMTP_PASS=email_sender_password -GOTRUE_SMTP_ADMIN_EMAIL=comp_admin@@some_company.com +GOTRUE_SMTP_ADMIN_EMAIL=comp_admin@some_company.com # gotrue admin GOTRUE_ADMIN_EMAIL=admin@example.com diff --git a/doc/DEPLOYMENT.md b/doc/DEPLOYMENT.md index 265f5b27..7879cf87 100644 --- a/doc/DEPLOYMENT.md +++ b/doc/DEPLOYMENT.md @@ -164,3 +164,7 @@ with your own in `nginx/ssl/` directory ## Usage of AppFlowy Application with AppFlowy Cloud - [AppFlowy with AppFlowyCloud](https://docs.appflowy.io/docs/guides/appflowy/self-hosting-appflowy) + +## 5. FAQ +- How do I use a different `postgres`? +> You can set `APPFLOWY_DATABASE_URL` to another postgres url. The default url is using the postgres in docker compose. diff --git a/docker-compose.yml b/docker-compose.yml index 7a99e2b0..2f6864eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,17 +102,20 @@ services: restart: on-failure environment: - RUST_LOG=${RUST_LOG:-info} - - APP_ENVIRONMENT=production - - APP__GOTRUE__JWT_SECRET=${GOTRUE_JWT_SECRET} - - APP__GOTRUE__EXT_URL=${API_EXTERNAL_URL} - - APP__GOTRUE__ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL} - - APP__GOTRUE__ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD} - - APP__S3__USE_MINIO=${USE_MINIO} - - APP__S3__MINIO_URL=${MINIO_URL:-http://minio:9000} - - APP__S3__ACCESS_KEY=${AWS_ACCESS_KEY_ID} - - APP__S3__SECRET_KEY=${AWS_SECRET_ACCESS_KEY} - - APP__S3__BUCKET=${AWS_S3_BUCKET} - - APP__S3__REGION=${AWS_REGION} + - APPFLOWY_ENVIRONMENT=production + - APPFLOWY_DATABASE_URL=postgres://postgres:password@postgres:5432/postgres + - APPFLOWY_REDIS_URI=redis://redis:6379 + - APPFLOWY_GOTRUE_JWT_SECRET=${GOTRUE_JWT_SECRET} + - APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999 + - APPFLOWY_GOTRUE_EXT_URL=${API_EXTERNAL_URL} + - APPFLOWY_GOTRUE_ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL} + - APPFLOWY_GOTRUE_ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD} + - APPFLOWY_S3_USE_MINIO=${USE_MINIO} + - APPFLOWY_S3_MINIO_URL=${MINIO_URL:-http://minio:9000} + - APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY_ID} + - APPFLOWY_S3_SECRET_KEY=${AWS_SECRET_ACCESS_KEY} + - APPFLOWY_S3_BUCKET=${AWS_S3_BUCKET} + - APPFLOWY_S3_REGION=${AWS_REGION} build: context: . dockerfile: Dockerfile diff --git a/src/application.rs b/src/application.rs index 342f18df..c15338af 100644 --- a/src/application.rs +++ b/src/application.rs @@ -2,7 +2,7 @@ use crate::api::metrics::{metrics_registry, metrics_scope}; use crate::biz::casbin::adapter::PgAdapter; use crate::biz::casbin::MODEL_CONF; use crate::component::auth::HEADER_TOKEN; -use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig}; +use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting}; use crate::middleware::cors_mw::default_cors; use crate::middleware::request_id::RequestIdMiddleware; use crate::self_signed::create_self_signed_certificate; @@ -148,17 +148,17 @@ pub async fn run( } fn get_certificate_and_server_key(config: &Config) -> Option<(Secret, Secret)> { - let tls_config = config.application.tls_config.as_ref()?; - match tls_config { - TlsConfig::NoTls => None, - TlsConfig::SelfSigned => Some(create_self_signed_certificate().unwrap()), + if config.application.use_tls { + Some(create_self_signed_certificate().unwrap()) + } else { + None } } pub async fn init_state(config: &Config) -> Result { // Postgres info!("Preparng to run database migrations..."); - let pg_pool = get_connection_pool(&config.database).await?; + let pg_pool = get_connection_pool(&config.db_settings).await?; migrate(&pg_pool).await?; // Bucket storage @@ -204,6 +204,7 @@ pub async fn init_state(config: &Config) -> Result { .await, ); + info!("Application state initialized"); Ok(AppState { pg_pool, config: Arc::new(config.clone()), @@ -337,22 +338,22 @@ async fn get_connection_pool(setting: &DatabaseSetting) -> Result .acquire_timeout(Duration::from_secs(10)) .connect_with(setting.with_db()) .await - .context("failed to connect to postgres database") + .map_err(|e| anyhow::anyhow!("Failed to connect to postgres database: {}", e)) } async fn migrate(pool: &PgPool) -> Result<(), Error> { sqlx::migrate!("./migrations") .run(pool) .await - .context("failed to run migrations") + .map_err(|e| anyhow::anyhow!("Failed to run migrations: {}", e)) } async fn get_gotrue_client(setting: &GoTrueSetting) -> Result { let gotrue_client = gotrue::api::Client::new(reqwest::Client::new(), &setting.base_url); - gotrue_client + let _ = gotrue_client .health() .await - .context("failed to connect to GoTrue")?; + .map_err(|e| anyhow::anyhow!("Failed to connect to GoTrue: {}", e)); Ok(gotrue_client) } diff --git a/src/config/config.rs b/src/config/config.rs index 81172405..2232d1c9 100644 --- a/src/config/config.rs +++ b/src/config/config.rs @@ -1,14 +1,12 @@ -use config::{Config as InnerConfig, FileFormat}; use secrecy::Secret; use serde::Deserialize; -use serde_aux::field_attributes::deserialize_number_from_string; use sqlx::postgres::{PgConnectOptions, PgSslMode}; -use std::convert::TryFrom; -use std::path::PathBuf; +use std::str::FromStr; -#[derive(serde::Deserialize, Clone, Debug)] +#[derive(Clone, Debug)] pub struct Config { - pub database: DatabaseSetting, + pub app_env: Environment, + pub db_settings: DatabaseSetting, pub gotrue: GoTrueSetting, pub application: ApplicationSetting, pub websocket: WebsocketSetting, @@ -49,49 +47,20 @@ pub struct GoTrueSetting { // any network interface. So using 127.0.0.1 for our local development and set // it to 0.0.0.0 in our Docker images. // -#[derive(serde::Deserialize, Clone, Debug)] +#[derive(Clone, Debug)] pub struct ApplicationSetting { - #[serde(deserialize_with = "deserialize_number_from_string")] pub port: u16, pub host: String, - pub data_dir: PathBuf, pub server_key: Secret, - pub tls_config: Option, + pub use_tls: bool, } -impl ApplicationSetting { - pub fn use_https(&self) -> bool { - match &self.tls_config { - None => false, - Some(config) => match config { - TlsConfig::NoTls => false, - TlsConfig::SelfSigned => true, - }, - } - } - - pub fn rocksdb_db_dir(&self) -> PathBuf { - self.data_dir.join("rocksdb") - } -} - -#[derive(serde::Deserialize, Clone, Debug)] -#[serde(rename_all = "snake_case")] -pub enum TlsConfig { - NoTls, - SelfSigned, -} - -#[derive(serde::Deserialize, Clone, Debug)] +#[derive(Clone, Debug)] pub struct DatabaseSetting { - pub username: String, - pub password: String, - #[serde(deserialize_with = "deserialize_number_from_string")] - pub port: u16, - pub host: String, - pub database_name: String, + pub pg_conn_opts: PgConnectOptions, pub require_ssl: bool, pub max_connections: u32, + pub database_name: String, } impl DatabaseSetting { @@ -101,55 +70,73 @@ impl DatabaseSetting { } else { PgSslMode::Prefer }; - PgConnectOptions::new() - .host(&self.host) - .username(&self.username) - .password(&self.password) - .port(self.port) - .ssl_mode(ssl_mode) + let options = self.pg_conn_opts.clone(); + options.ssl_mode(ssl_mode) } pub fn with_db(&self) -> PgConnectOptions { self.without_db().database(&self.database_name) } - - /// Generate a postgresql connection string from the database settings. - pub fn to_pg_url(&self) -> String { - let ssl_mode = if self.require_ssl { - "require" - } else { - "prefer" - }; - format!( - "postgres://{}:{}@{}:{}/{}?sslmode={}", - self.username, self.password, self.host, self.port, self.database_name, ssl_mode - ) - } } -pub fn get_configuration(app_env: &Environment) -> Result { - let base_path = std::env::current_dir().expect("Failed to determine the current directory"); - let configuration_dir = base_path.join("configuration"); +// Default values favor local development. +pub fn get_configuration() -> Result { + let config = Config { + app_env: get_env_var("APPFLOWY_ENVIRONMENT", "local").parse()?, + db_settings: DatabaseSetting { + pg_conn_opts: PgConnectOptions::from_str(&get_env_var( + "APPFLOWY_DATABASE_URL", + "postgres://postgres:password@localhost:5433/postgres", + ))?, + require_ssl: get_env_var("APPFLOWY_DATABASE_REQUIRE_SSL", "false").parse()?, + max_connections: get_env_var("APPFLOWY_DATABASE_MAX_CONNECTIONS", "20").parse()?, + database_name: get_env_var("APPFLOWY_DATABASE_NAME", "postgres"), + }, + gotrue: GoTrueSetting { + base_url: get_env_var("APPFLOWY_GOTRUE_BASE_URL", "http://localhost:9998"), + ext_url: get_env_var("APPFLOWY_GOTRUE_EXT_URL", "http://localhost:9998"), + jwt_secret: get_env_var("APPFLOWY_GOTRUE_JWT_SECRET", "hello456").into(), + admin_email: get_env_var("APPFLOWY_GOTRUE_ADMIN_EMAIL", "admin@example.com"), + admin_password: get_env_var("APPFLOWY_GOTRUE_ADMIN_PASSWORD", "password"), + }, + application: ApplicationSetting { + port: get_env_var("APPFLOWY_APPLICATION_PORT", "8000").parse()?, + host: get_env_var("APPFLOWY_APPLICATION_HOST", "0.0.0.0"), + use_tls: get_env_var("APPFLOWY_APPLICATION_USE_TLS", "false").parse()?, + server_key: get_env_var("APPFLOWY_APPLICATION_SERVER_KEY", "server_key").into(), + }, + websocket: WebsocketSetting { + heartbeat_interval: get_env_var("APPFLOWY_WEBSOCKET_HEARTBEAT_INTERVAL", "6").parse()?, + client_timeout: get_env_var("APPFLOWY_WEBSOCKET_CLIENT_TIMEOUT", "30").parse()?, + }, + redis_uri: get_env_var("APPFLOWY_REDIS_URI", "redis://localhost:6380").into(), + s3: S3Setting { + use_minio: get_env_var("APPFLOWY_S3_USE_MINIO", "true").parse()?, + minio_url: get_env_var("APPFLOWY_S3_MINIO_URL", "http://localhost:9000"), + access_key: get_env_var("APPFLOWY_S3_ACCESS_KEY", "minioadmin"), + secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin"), + bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"), + region: get_env_var("APPFLOWY_S3_REGION", "us-east-1"), + }, + casbin: CasbinSetting { + pool_size: get_env_var("APPFLOWY_CASBIN_POOL_SIZE", "8").parse()?, + }, + }; + Ok(config) +} - let builder = InnerConfig::builder() - .set_default("default", "1")? - .add_source( - config::File::from(configuration_dir.join("base")) - .required(true) - .format(FileFormat::Yaml), - ) - .add_source( - config::File::from(configuration_dir.join(app_env.as_str())) - .required(true) - .format(FileFormat::Yaml), - ) - // Add in settings from environment variables (with a prefix of APP and '__' as - // separator) E.g. `APP__APPLICATION__PORT=5001 would set - // `Settings.application.port` - .add_source(config::Environment::with_prefix("app").separator("__")); - - let config = builder.build()?; - config.try_deserialize() +fn get_env_var(key: &str, default: &str) -> String { + match std::env::var(key) { + Ok(value) => value, + Err(e) => { + tracing::warn!( + "failed to read environment variable: {}, using default value: {}", + e, + default + ); + default.to_owned() + }, + } } /// The possible runtime environment for our application. @@ -168,21 +155,22 @@ impl Environment { } } -impl TryFrom for Environment { - type Error = String; +impl FromStr for Environment { + type Err = anyhow::Error; - fn try_from(s: String) -> Result { + fn from_str(s: &str) -> Result { match s.to_lowercase().as_str() { "local" => Ok(Self::Local), "production" => Ok(Self::Production), - other => Err(format!( + other => anyhow::bail!( "{} is not a supported environment. Use either `local` or `production`.", other - )), + ), } } } -#[derive(serde::Deserialize, Clone, Debug)] + +#[derive(Clone, Debug)] pub struct WebsocketSetting { pub heartbeat_interval: u8, pub client_timeout: u8, diff --git a/src/main.rs b/src/main.rs index eb855682..6edf7a54 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,14 @@ use appflowy_cloud::application::{init_state, Application}; -use appflowy_cloud::config::config::{get_configuration, Environment}; +use appflowy_cloud::config::config::get_configuration; use appflowy_cloud::telemetry::init_subscriber; use std::panic; use tracing::error; #[actix_web::main] async fn main() -> anyhow::Result<()> { + // load from .env + dotenvy::dotenv().ok(); + let level = std::env::var("RUST_LOG").unwrap_or("info".to_string()); println!("AppFlowy Cloud with RUST_LOG={}", level); @@ -19,12 +22,15 @@ async fn main() -> anyhow::Result<()> { filters.push(format!("database={}", level)); filters.push(format!("storage={}", level)); - let app_env: Environment = std::env::var("APP_ENVIRONMENT") - .unwrap_or_else(|_| "local".to_string()) - .try_into() - .expect("Failed to parse APP_ENVIRONMENT."); + // let app_env: Environment = std::env::var("APP_ENVIRONMENT") + // .unwrap_or_else(|_| "local".to_string()) + // .try_into() + // .expect("Failed to parse APP_ENVIRONMENT."); - init_subscriber(&app_env, filters); + let conf = + get_configuration().map_err(|e| anyhow::anyhow!("Failed to read configuration: {}", e))?; + + init_subscriber(&conf.app_env, filters); // Set panic hook panic::set_hook(Box::new(|panic_info| { @@ -44,11 +50,10 @@ async fn main() -> anyhow::Result<()> { }; error!("panic hook: {}\n{}", panic_message, location); })); - let configuration = get_configuration(&app_env).expect("The configuration should be configured."); - let state = init_state(&configuration) + let state = init_state(&conf) .await - .expect("The AppState should be initialized"); - let application = Application::build(configuration, state).await?; + .map_err(|e| anyhow::anyhow!("Failed to initialize application state: {}", e))?; + let application = Application::build(conf, state).await?; application.run_until_stopped().await?; Ok(()) diff --git a/tests/user/utils.rs b/tests/user/utils.rs index 40aaa583..bc35642a 100644 --- a/tests/user/utils.rs +++ b/tests/user/utils.rs @@ -1,5 +1,5 @@ use client_api::Client; -use dotenv::dotenv; +use dotenvy::dotenv; use sqlx::types::Uuid;