feat: use only env var for appflowy cloud (#224)

* feat: use only env var for appflowy cloud

* fix: jwt local testing

* fix: security audit

* feat: update docker deploy configs

* fix: test utils dotenvy

* fix: test try sqlx offline

* fix: add gotrue configs for appflowy

* fix: redis uri in docker

Cargo.lock

@@ -523,11 +523,10 @@ dependencies = [
  "collab",
  "collab-entity",
  "collab-folder",
- "config",
  "database",
  "database-entity",
  "derive_more",
- "dotenv",
+ "dotenvy",
  "fancy-regex",
  "futures",
  "futures-util",
@@ -571,6 +570,7 @@ dependencies = [
  "tracing-log",
  "tracing-subscriber",
  "unicode-segmentation",
+ "url",
  "uuid",
  "validator",
  "workspace-template",
@@ -992,7 +992,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4114279215a005bc675e386011e594e1d9b800918cea18fcadadcce864a2046b"
 dependencies = [
  "borsh-derive",
- "hashbrown 0.11.2",
+ "hashbrown 0.12.3",
 ]
 
 [[package]]
@@ -1403,20 +1403,6 @@ dependencies = [
  "tokio-util",
 ]
 
-[[package]]
-name = "config"
-version = "0.13.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d379af7f68bfc21714c6c7dea883544201741d2ce8274bb12fa54f89507f52a7"
-dependencies = [
- "async-trait",
- "lazy_static",
- "nom",
- "pathdiff",
- "serde",
- "yaml-rust",
-]
-
 [[package]]
 name = "const-oid"
 version = "0.9.5"
@@ -2074,9 +2060,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
 [[package]]
 name = "form_urlencoded"
-version = "1.2.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
 dependencies = [
  "percent-encoding",
 ]
@@ -2593,6 +2579,16 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "idna"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
 [[package]]
 name = "if_chain"
 version = "1.0.2"
@@ -2807,12 +2803,6 @@ dependencies = [
  "vcpkg",
 ]
 
-[[package]]
-name = "linked-hash-map"
-version = "0.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
-
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.7"
@@ -3317,12 +3307,6 @@ version = "1.0.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c"
 
-[[package]]
-name = "pathdiff"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd"
-
 [[package]]
 name = "pem"
 version = "1.1.1"
@@ -3343,9 +3327,9 @@ dependencies = [
 
 [[package]]
 name = "percent-encoding"
-version = "2.3.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
 
 [[package]]
 name = "petgraph"
@@ -5698,12 +5682,12 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
 [[package]]
 name = "url"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
+checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633"
 dependencies = [
  "form_urlencoded",
- "idna 0.4.0",
+ "idna 0.5.0",
  "percent-encoding",
 ]
 
@@ -6136,15 +6120,6 @@ dependencies = [
  "time",
 ]
 
-[[package]]
-name = "yaml-rust"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
-dependencies = [
- "linked-hash-map",
-]
-
 [[package]]
 name = "yasna"
 version = "0.5.2"

Cargo.toml

@@ -34,7 +34,6 @@ tokio-stream = "0.1.14"
 tokio-util = { version = "0.7.9", features = ["io"] }
 futures = "0.3.17"
 futures-util ={ version = "0.3.26" , features = ["std","io"] }
-config = { version = "0.13.3", default-features = false, features = ["yaml"] }
 once_cell = "1.13.0"
 chrono = { version = "0.4.23", features = ["serde", "clock"], default-features = false }
 derive_more = { version = "0.99" }
@@ -67,6 +66,8 @@ uuid = "1.4.1"
 tokio-tungstenite = { version = "0.20.1", features = ["native-tls"] }
 prost = "0.12.1"
 casbin = { version = "2.0.9" }
+dotenvy = "0.15.7"
+url = "2.5.0"
 
 # collab
 collab = { version = "0.1.0", features = ["async-plugin"] }
@@ -91,7 +92,6 @@ realtime-entity.workspace = true
 once_cell = "1.7.2"
 tempfile = "3.4.0"
 assert-json-diff = "2.0.2"
-dotenv = "0.15.0"
 scraper = "0.17.1"
 client-api = { path = "libs/client-api", features = ["collab-sync", "test_util"] }
 opener = "0.6.1"

Dockerfile

@@ -1,4 +1,4 @@
-FROM lukemathwalker/cargo-chef:latest-rust-1.69.0 as chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.74.0 as chef
 
 WORKDIR /app
 RUN apt update && apt install lld clang -y
@@ -18,7 +18,7 @@ ENV SQLX_OFFLINE true
 # Build the project
 RUN cargo build --release --bin appflowy_cloud
 
-FROM debian:bullseye-slim AS runtime
+FROM debian:bookworm-slim AS runtime
 WORKDIR /app
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends openssl \

dev.env

@@ -15,7 +15,7 @@ GOTRUE_SMTP_HOST=smtp.gmail.com
 GOTRUE_SMTP_PORT=465
 GOTRUE_SMTP_USER=email_sender@some_company.com
 GOTRUE_SMTP_PASS=email_sender_password
-GOTRUE_SMTP_ADMIN_EMAIL=comp_admin@@some_company.com
+GOTRUE_SMTP_ADMIN_EMAIL=comp_admin@some_company.com
 
 # gotrue admin
 GOTRUE_ADMIN_EMAIL=admin@example.com

doc/DEPLOYMENT.md

@@ -164,3 +164,7 @@ with your own in `nginx/ssl/` directory
 
 ## Usage of AppFlowy Application with AppFlowy Cloud
 - [AppFlowy with AppFlowyCloud](https://docs.appflowy.io/docs/guides/appflowy/self-hosting-appflowy)
+
+## 5. FAQ
+- How do I use a different `postgres`?
+> You can set `APPFLOWY_DATABASE_URL` to another postgres url. The default url is using the postgres in docker compose.

docker-compose.yml

@@ -102,17 +102,20 @@ services:
     restart: on-failure
     environment:
       - RUST_LOG=${RUST_LOG:-info}
-      - APP_ENVIRONMENT=production
+      - APPFLOWY_ENVIRONMENT=production
-      - APP__GOTRUE__JWT_SECRET=${GOTRUE_JWT_SECRET}
+      - APPFLOWY_DATABASE_URL=postgres://postgres:password@postgres:5432/postgres
-      - APP__GOTRUE__EXT_URL=${API_EXTERNAL_URL}
+      - APPFLOWY_REDIS_URI=redis://redis:6379
-      - APP__GOTRUE__ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL}
+      - APPFLOWY_GOTRUE_JWT_SECRET=${GOTRUE_JWT_SECRET}
-      - APP__GOTRUE__ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD}
+      - APPFLOWY_GOTRUE_BASE_URL=http://gotrue:9999
-      - APP__S3__USE_MINIO=${USE_MINIO}
+      - APPFLOWY_GOTRUE_EXT_URL=${API_EXTERNAL_URL}
-      - APP__S3__MINIO_URL=${MINIO_URL:-http://minio:9000}
+      - APPFLOWY_GOTRUE_ADMIN_EMAIL=${GOTRUE_ADMIN_EMAIL}
-      - APP__S3__ACCESS_KEY=${AWS_ACCESS_KEY_ID}
+      - APPFLOWY_GOTRUE_ADMIN_PASSWORD=${GOTRUE_ADMIN_PASSWORD}
-      - APP__S3__SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
+      - APPFLOWY_S3_USE_MINIO=${USE_MINIO}
-      - APP__S3__BUCKET=${AWS_S3_BUCKET}
+      - APPFLOWY_S3_MINIO_URL=${MINIO_URL:-http://minio:9000}
-      - APP__S3__REGION=${AWS_REGION}
+      - APPFLOWY_S3_ACCESS_KEY=${AWS_ACCESS_KEY_ID}
+      - APPFLOWY_S3_SECRET_KEY=${AWS_SECRET_ACCESS_KEY}
+      - APPFLOWY_S3_BUCKET=${AWS_S3_BUCKET}
+      - APPFLOWY_S3_REGION=${AWS_REGION}
     build:
       context: .
       dockerfile: Dockerfile

src/application.rs

@@ -2,7 +2,7 @@ use crate::api::metrics::{metrics_registry, metrics_scope};
 use crate::biz::casbin::adapter::PgAdapter;
 use crate::biz::casbin::MODEL_CONF;
 use crate::component::auth::HEADER_TOKEN;
-use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting, TlsConfig};
+use crate::config::config::{Config, DatabaseSetting, GoTrueSetting, S3Setting};
 use crate::middleware::cors_mw::default_cors;
 use crate::middleware::request_id::RequestIdMiddleware;
 use crate::self_signed::create_self_signed_certificate;
@@ -148,17 +148,17 @@ pub async fn run(
 }
 
 fn get_certificate_and_server_key(config: &Config) -> Option<(Secret<String>, Secret<String>)> {
-  let tls_config = config.application.tls_config.as_ref()?;
+  if config.application.use_tls {
-  match tls_config {
+    Some(create_self_signed_certificate().unwrap())
-    TlsConfig::NoTls => None,
+  } else {
-    TlsConfig::SelfSigned => Some(create_self_signed_certificate().unwrap()),
+    None
   }
 }
 
 pub async fn init_state(config: &Config) -> Result<AppState, Error> {
   // Postgres
   info!("Preparng to run database migrations...");
-  let pg_pool = get_connection_pool(&config.database).await?;
+  let pg_pool = get_connection_pool(&config.db_settings).await?;
   migrate(&pg_pool).await?;
 
   // Bucket storage
@@ -204,6 +204,7 @@ pub async fn init_state(config: &Config) -> Result<AppState, Error> {
     .await,
   );
 
+  info!("Application state initialized");
   Ok(AppState {
     pg_pool,
     config: Arc::new(config.clone()),
@@ -337,22 +338,22 @@ async fn get_connection_pool(setting: &DatabaseSetting) -> Result<PgPool, Error>
     .acquire_timeout(Duration::from_secs(10))
     .connect_with(setting.with_db())
     .await
-    .context("failed to connect to postgres database")
+    .map_err(|e| anyhow::anyhow!("Failed to connect to postgres database: {}", e))
 }
 
 async fn migrate(pool: &PgPool) -> Result<(), Error> {
   sqlx::migrate!("./migrations")
     .run(pool)
     .await
-    .context("failed to run migrations")
+    .map_err(|e| anyhow::anyhow!("Failed to run migrations: {}", e))
 }
 
 async fn get_gotrue_client(setting: &GoTrueSetting) -> Result<gotrue::api::Client, Error> {
   let gotrue_client = gotrue::api::Client::new(reqwest::Client::new(), &setting.base_url);
-  gotrue_client
+  let _ = gotrue_client
     .health()
     .await
-    .context("failed to connect to GoTrue")?;
+    .map_err(|e| anyhow::anyhow!("Failed to connect to GoTrue: {}", e));
   Ok(gotrue_client)
 }
 

src/config/config.rs

@@ -1,14 +1,12 @@
-use config::{Config as InnerConfig, FileFormat};
 use secrecy::Secret;
 use serde::Deserialize;
-use serde_aux::field_attributes::deserialize_number_from_string;
 use sqlx::postgres::{PgConnectOptions, PgSslMode};
-use std::convert::TryFrom;
+use std::str::FromStr;
-use std::path::PathBuf;
 
-#[derive(serde::Deserialize, Clone, Debug)]
+#[derive(Clone, Debug)]
 pub struct Config {
-  pub database: DatabaseSetting,
+  pub app_env: Environment,
+  pub db_settings: DatabaseSetting,
   pub gotrue: GoTrueSetting,
   pub application: ApplicationSetting,
   pub websocket: WebsocketSetting,
@@ -49,49 +47,20 @@ pub struct GoTrueSetting {
 // any network interface. So using 127.0.0.1 for our local development and set
 // it to 0.0.0.0 in our Docker images.
 //
-#[derive(serde::Deserialize, Clone, Debug)]
+#[derive(Clone, Debug)]
 pub struct ApplicationSetting {
-  #[serde(deserialize_with = "deserialize_number_from_string")]
   pub port: u16,
   pub host: String,
-  pub data_dir: PathBuf,
   pub server_key: Secret<String>,
-  pub tls_config: Option<TlsConfig>,
+  pub use_tls: bool,
 }
 
-impl ApplicationSetting {
+#[derive(Clone, Debug)]
-  pub fn use_https(&self) -> bool {
-    match &self.tls_config {
-      None => false,
-      Some(config) => match config {
-        TlsConfig::NoTls => false,
-        TlsConfig::SelfSigned => true,
-      },
-    }
-  }
-
-  pub fn rocksdb_db_dir(&self) -> PathBuf {
-    self.data_dir.join("rocksdb")
-  }
-}
-
-#[derive(serde::Deserialize, Clone, Debug)]
-#[serde(rename_all = "snake_case")]
-pub enum TlsConfig {
-  NoTls,
-  SelfSigned,
-}
-
-#[derive(serde::Deserialize, Clone, Debug)]
 pub struct DatabaseSetting {
-  pub username: String,
+  pub pg_conn_opts: PgConnectOptions,
-  pub password: String,
-  #[serde(deserialize_with = "deserialize_number_from_string")]
-  pub port: u16,
-  pub host: String,
-  pub database_name: String,
   pub require_ssl: bool,
   pub max_connections: u32,
+  pub database_name: String,
 }
 
 impl DatabaseSetting {
@@ -101,55 +70,73 @@ impl DatabaseSetting {
     } else {
       PgSslMode::Prefer
     };
-    PgConnectOptions::new()
+    let options = self.pg_conn_opts.clone();
-      .host(&self.host)
+    options.ssl_mode(ssl_mode)
-      .username(&self.username)
-      .password(&self.password)
-      .port(self.port)
-      .ssl_mode(ssl_mode)
   }
 
   pub fn with_db(&self) -> PgConnectOptions {
     self.without_db().database(&self.database_name)
   }
+}
 
-  /// Generate a postgresql connection string from the database settings.
+// Default values favor local development.
-  pub fn to_pg_url(&self) -> String {
+pub fn get_configuration() -> Result<Config, anyhow::Error> {
-    let ssl_mode = if self.require_ssl {
+  let config = Config {
-      "require"
+    app_env: get_env_var("APPFLOWY_ENVIRONMENT", "local").parse()?,
-    } else {
+    db_settings: DatabaseSetting {
-      "prefer"
+      pg_conn_opts: PgConnectOptions::from_str(&get_env_var(
+        "APPFLOWY_DATABASE_URL",
+        "postgres://postgres:password@localhost:5433/postgres",
+      ))?,
+      require_ssl: get_env_var("APPFLOWY_DATABASE_REQUIRE_SSL", "false").parse()?,
+      max_connections: get_env_var("APPFLOWY_DATABASE_MAX_CONNECTIONS", "20").parse()?,
+      database_name: get_env_var("APPFLOWY_DATABASE_NAME", "postgres"),
+    },
+    gotrue: GoTrueSetting {
+      base_url: get_env_var("APPFLOWY_GOTRUE_BASE_URL", "http://localhost:9998"),
+      ext_url: get_env_var("APPFLOWY_GOTRUE_EXT_URL", "http://localhost:9998"),
+      jwt_secret: get_env_var("APPFLOWY_GOTRUE_JWT_SECRET", "hello456").into(),
+      admin_email: get_env_var("APPFLOWY_GOTRUE_ADMIN_EMAIL", "admin@example.com"),
+      admin_password: get_env_var("APPFLOWY_GOTRUE_ADMIN_PASSWORD", "password"),
+    },
+    application: ApplicationSetting {
+      port: get_env_var("APPFLOWY_APPLICATION_PORT", "8000").parse()?,
+      host: get_env_var("APPFLOWY_APPLICATION_HOST", "0.0.0.0"),
+      use_tls: get_env_var("APPFLOWY_APPLICATION_USE_TLS", "false").parse()?,
+      server_key: get_env_var("APPFLOWY_APPLICATION_SERVER_KEY", "server_key").into(),
+    },
+    websocket: WebsocketSetting {
+      heartbeat_interval: get_env_var("APPFLOWY_WEBSOCKET_HEARTBEAT_INTERVAL", "6").parse()?,
+      client_timeout: get_env_var("APPFLOWY_WEBSOCKET_CLIENT_TIMEOUT", "30").parse()?,
+    },
+    redis_uri: get_env_var("APPFLOWY_REDIS_URI", "redis://localhost:6380").into(),
+    s3: S3Setting {
+      use_minio: get_env_var("APPFLOWY_S3_USE_MINIO", "true").parse()?,
+      minio_url: get_env_var("APPFLOWY_S3_MINIO_URL", "http://localhost:9000"),
+      access_key: get_env_var("APPFLOWY_S3_ACCESS_KEY", "minioadmin"),
+      secret_key: get_env_var("APPFLOWY_S3_SECRET_KEY", "minioadmin"),
+      bucket: get_env_var("APPFLOWY_S3_BUCKET", "appflowy"),
+      region: get_env_var("APPFLOWY_S3_REGION", "us-east-1"),
+    },
+    casbin: CasbinSetting {
+      pool_size: get_env_var("APPFLOWY_CASBIN_POOL_SIZE", "8").parse()?,
+    },
   };
-    format!(
+  Ok(config)
-      "postgres://{}:{}@{}:{}/{}?sslmode={}",
-      self.username, self.password, self.host, self.port, self.database_name, ssl_mode
-    )
-  }
 }
 
-pub fn get_configuration(app_env: &Environment) -> Result<Config, config::ConfigError> {
+fn get_env_var(key: &str, default: &str) -> String {
-  let base_path = std::env::current_dir().expect("Failed to determine the current directory");
+  match std::env::var(key) {
-  let configuration_dir = base_path.join("configuration");
+    Ok(value) => value,
-
+    Err(e) => {
-  let builder = InnerConfig::builder()
+      tracing::warn!(
-        .set_default("default", "1")?
+        "failed to read environment variable: {}, using default value: {}",
-        .add_source(
+        e,
-            config::File::from(configuration_dir.join("base"))
+        default
-                .required(true)
+      );
-                .format(FileFormat::Yaml),
+      default.to_owned()
-        )
+    },
-        .add_source(
+  }
-            config::File::from(configuration_dir.join(app_env.as_str()))
-                .required(true)
-                .format(FileFormat::Yaml),
-        )
-        // Add in settings from environment variables (with a prefix of APP and '__' as
-        // separator) E.g. `APP__APPLICATION__PORT=5001 would set
-        // `Settings.application.port`
-        .add_source(config::Environment::with_prefix("app").separator("__"));
-
-  let config = builder.build()?;
-  config.try_deserialize()
 }
 
 /// The possible runtime environment for our application.
@@ -168,21 +155,22 @@ impl Environment {
   }
 }
 
-impl TryFrom<String> for Environment {
+impl FromStr for Environment {
-  type Error = String;
+  type Err = anyhow::Error;
 
-  fn try_from(s: String) -> Result<Self, Self::Error> {
+  fn from_str(s: &str) -> Result<Self, Self::Err> {
     match s.to_lowercase().as_str() {
       "local" => Ok(Self::Local),
       "production" => Ok(Self::Production),
-      other => Err(format!(
+      other => anyhow::bail!(
         "{} is not a supported environment. Use either `local` or `production`.",
         other
-      )),
+      ),
     }
   }
 }
-#[derive(serde::Deserialize, Clone, Debug)]
+
+#[derive(Clone, Debug)]
 pub struct WebsocketSetting {
   pub heartbeat_interval: u8,
   pub client_timeout: u8,

src/main.rs

@@ -1,11 +1,14 @@
 use appflowy_cloud::application::{init_state, Application};
-use appflowy_cloud::config::config::{get_configuration, Environment};
+use appflowy_cloud::config::config::get_configuration;
 use appflowy_cloud::telemetry::init_subscriber;
 use std::panic;
 use tracing::error;
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
+  // load from .env
+  dotenvy::dotenv().ok();
+
   let level = std::env::var("RUST_LOG").unwrap_or("info".to_string());
   println!("AppFlowy Cloud with RUST_LOG={}", level);
 
@@ -19,12 +22,15 @@ async fn main() -> anyhow::Result<()> {
   filters.push(format!("database={}", level));
   filters.push(format!("storage={}", level));
 
-  let app_env: Environment = std::env::var("APP_ENVIRONMENT")
+  // let app_env: Environment = std::env::var("APP_ENVIRONMENT")
-    .unwrap_or_else(|_| "local".to_string())
+  //   .unwrap_or_else(|_| "local".to_string())
-    .try_into()
+  //   .try_into()
-    .expect("Failed to parse APP_ENVIRONMENT.");
+  //   .expect("Failed to parse APP_ENVIRONMENT.");
 
-  init_subscriber(&app_env, filters);
+  let conf =
+    get_configuration().map_err(|e| anyhow::anyhow!("Failed to read configuration: {}", e))?;
+
+  init_subscriber(&conf.app_env, filters);
 
   // Set panic hook
   panic::set_hook(Box::new(|panic_info| {
@@ -44,11 +50,10 @@ async fn main() -> anyhow::Result<()> {
     };
     error!("panic hook: {}\n{}", panic_message, location);
   }));
-  let configuration = get_configuration(&app_env).expect("The configuration should be configured.");
+  let state = init_state(&conf)
-  let state = init_state(&configuration)
     .await
-    .expect("The AppState should be initialized");
+    .map_err(|e| anyhow::anyhow!("Failed to initialize application state: {}", e))?;
-  let application = Application::build(configuration, state).await?;
+  let application = Application::build(conf, state).await?;
   application.run_until_stopped().await?;
 
   Ok(())

tests/user/utils.rs

@@ -1,5 +1,5 @@
 use client_api::Client;
-use dotenv::dotenv;
+use dotenvy::dotenv;
 
 use sqlx::types::Uuid;