From 5bebc6a2b2c6efa419b85c0ca05c77f95c953b08 Mon Sep 17 00:00:00 2001 From: "Nathan.fooo" <86001920+appflowy@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:23:40 +0800 Subject: [PATCH] chore: Integrate appflowy ai (#504) * chore: remove client api * chore: add summarize row test * chore: merge main * chore: fix ci * chore: update docker compose file --- .github/workflows/integration_test.yml | 4 +- Cargo.lock | 241 +++++++++++++++--- Cargo.toml | 8 +- deploy.env | 3 + dev.env | 3 + docker-compose-ci.yml | 15 +- docker-compose-dev.yml | 9 +- docker-compose.yml | 13 +- libs/appflowy-ai/Cargo.toml | 16 -- libs/appflowy-ai/src/client.rs | 82 ------ libs/appflowy-ai/src/entity.rs | 11 - libs/appflowy-ai/src/error.rs | 33 --- libs/appflowy-ai/src/lib.rs | 3 - libs/appflowy-ai/tests/main.rs | 1 - libs/appflowy-ai/tests/row_test/mod.rs | 1 - .../tests/row_test/summarize_test.rs | 10 - libs/client-api/src/http.rs | 24 ++ libs/shared-entity/src/dto/ai_dto.rs | 29 +++ libs/shared-entity/src/dto/mod.rs | 1 + nginx/nginx.conf | 7 + src/api/workspace.rs | 28 ++ src/application.rs | 11 +- src/main.rs | 1 + src/state.rs | 4 +- tests/ai_test/mod.rs | 1 + tests/ai_test/summarize_row.rs | 22 ++ tests/collab/pending_write_test.rs | 2 +- tests/main.rs | 2 + xtask/src/main.rs | 2 +- 29 files changed, 366 insertions(+), 221 deletions(-) delete mode 100644 libs/appflowy-ai/Cargo.toml delete mode 100644 libs/appflowy-ai/src/client.rs delete mode 100644 libs/appflowy-ai/src/entity.rs delete mode 100644 libs/appflowy-ai/src/error.rs delete mode 100644 libs/appflowy-ai/src/lib.rs delete mode 100644 libs/appflowy-ai/tests/main.rs delete mode 100644 libs/appflowy-ai/tests/row_test/mod.rs delete mode 100644 libs/appflowy-ai/tests/row_test/summarize_test.rs create mode 100644 libs/shared-entity/src/dto/ai_dto.rs create mode 100644 tests/ai_test/mod.rs create mode 100644 tests/ai_test/summarize_row.rs diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index d97190b7..16db99fd 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -20,7 +20,7 @@ env: jobs: test: - name: Docker + name: Integration Tests runs-on: ubuntu-latest steps: @@ -45,6 +45,7 @@ jobs: sed -i 's/GOTRUE_MAILER_AUTOCONFIRM=.*/GOTRUE_MAILER_AUTOCONFIRM=false/' .env sed -i 's/API_EXTERNAL_URL=http:\/\/your-host/API_EXTERNAL_URL=http:\/\/localhost/' .env sed -i 's/GOTRUE_RATE_LIMIT_EMAIL_SENT=100/GOTRUE_RATE_LIMIT_EMAIL_SENT=1000/' .env + sed -i 's/OPENAI_API_KEY=.*/OPENAI_API_KEY=${{ secrets.CI_OPENAI_API_KEY }}/' .env - name: Update Nginx Configuration run: | @@ -59,6 +60,7 @@ jobs: - name: Run Docker-Compose run: | docker compose -f docker-compose-ci.yml up -d + docker ps -a - name: Run tests run: | diff --git a/Cargo.lock b/Cargo.lock index 076f7d27..71a96760 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,7 +399,7 @@ dependencies = [ "human_bytes", "jwt", "redis 0.25.2", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "shared-entity", @@ -554,7 +554,7 @@ dependencies = [ "anyhow", "bincode", "getrandom 0.2.12", - "reqwest", + "reqwest 0.11.27", "rust-s3", "serde", "serde_json", @@ -570,15 +570,16 @@ dependencies = [ ] [[package]] -name = "appflowy-ai" +name = "appflowy-ai-client" version = "0.1.0" +source = "git+https://github.com/AppFlowy-IO/AppFlowy-AI?rev=6da7ac710fa8f7c9d70b33adb5bd86eb6f8790d9#6da7ac710fa8f7c9d70b33adb5bd86eb6f8790d9" dependencies = [ "anyhow", - "reqwest", + "reqwest 0.12.4", "serde", "serde_json", "thiserror", - "tokio", + "tracing", ] [[package]] @@ -598,7 +599,7 @@ dependencies = [ "actix-web-actors", "anyhow", "app-error", - "appflowy-ai", + "appflowy-ai-client", "appflowy-collaborate", "argon2", "assert-json-diff", @@ -641,7 +642,7 @@ dependencies = [ "rand 0.8.5", "rcgen", "redis 0.25.2", - "reqwest", + "reqwest 0.11.27", "rust-s3", "scraper", "secrecy", @@ -926,11 +927,11 @@ dependencies = [ "http 0.2.11", "log", "native-tls", - "rustls", + "rustls 0.21.12", "serde", "serde_json", "url", - "webpki-roots", + "webpki-roots 0.25.3", ] [[package]] @@ -1153,6 +1154,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -1546,7 +1553,7 @@ dependencies = [ "mime", "parking_lot 0.12.1", "prost", - "reqwest", + "reqwest 0.11.27", "scraper", "semver", "serde", @@ -1588,7 +1595,7 @@ dependencies = [ "mime", "once_cell", "opener", - "reqwest", + "reqwest 0.11.27", "scraper", "serde_json", "shared-entity", @@ -2688,7 +2695,7 @@ dependencies = [ "getrandom 0.2.12", "gotrue-entity", "infra", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "tokio", @@ -3011,6 +3018,7 @@ dependencies = [ "itoa", "pin-project-lite", "tokio", + "want", ] [[package]] @@ -3022,9 +3030,26 @@ dependencies = [ "futures-util", "http 0.2.11", "hyper 0.14.28", - "rustls", + "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +dependencies = [ + "futures-util", + "http 1.0.0", + "hyper 1.1.0", + "hyper-util", + "rustls 0.22.4", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.25.0", + "tower-service", ] [[package]] @@ -3052,6 +3077,22 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.1.0", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.3" @@ -3059,6 +3100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", + "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", @@ -3066,6 +3108,9 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -3177,7 +3222,7 @@ name = "infra" version = "0.1.0" dependencies = [ "anyhow", - "reqwest", + "reqwest 0.11.27", "serde", "serde_json", "tracing", @@ -4662,8 +4707,8 @@ dependencies = [ "http 0.2.11", "http-body 0.4.6", "hyper 0.14.28", - "hyper-rustls", - "hyper-tls", + "hyper-rustls 0.24.2", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -4672,8 +4717,8 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -4681,7 +4726,7 @@ dependencies = [ "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-util", "tower-service", "url", @@ -4689,8 +4734,57 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", - "winreg", + "webpki-roots 0.25.3", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.0", + "bytes", + "cookie 0.17.0", + "cookie_store", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.4", + "http 1.0.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.1.0", + "hyper-rustls 0.26.0", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls 0.22.4", + "rustls-pemfile 2.1.2", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-rustls 0.25.0", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots 0.26.1", + "winreg 0.52.0", ] [[package]] @@ -4836,7 +4930,7 @@ dependencies = [ "hmac", "http 0.2.11", "hyper 0.14.28", - "hyper-tls", + "hyper-tls 0.5.0", "log", "maybe-async", "md5", @@ -4917,10 +5011,24 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.7", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.7", + "rustls-pki-types", + "rustls-webpki 0.102.3", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.6.3" @@ -4928,7 +5036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "schannel", "security-framework", ] @@ -4942,6 +5050,22 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4952,6 +5076,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3bce581c0dd41bce533ce695a1437fa16a7ab5ac3ccfa99fe1a620a7885eabf" +dependencies = [ + "ring 0.17.7", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -5113,9 +5248,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" dependencies = [ "serde_derive", ] @@ -5133,9 +5268,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", "quote", @@ -5279,7 +5414,7 @@ dependencies = [ "collab-entity", "database-entity", "gotrue-entity", - "reqwest", + "reqwest 0.11.27", "rust-s3", "serde", "serde_json", @@ -5487,8 +5622,8 @@ dependencies = [ "paste", "percent-encoding", "rust_decimal", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "sha2", @@ -5500,7 +5635,7 @@ dependencies = [ "tracing", "url", "uuid", - "webpki-roots", + "webpki-roots 0.25.3", ] [[package]] @@ -6000,7 +6135,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", "tokio", ] @@ -6025,13 +6171,13 @@ dependencies = [ "futures-util", "log", "native-tls", - "rustls", + "rustls 0.21.12", "rustls-native-certs", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.24.1", "tungstenite", - "webpki-roots", + "webpki-roots 0.25.3", ] [[package]] @@ -6338,7 +6484,7 @@ dependencies = [ "log", "native-tls", "rand 0.8.5", - "rustls", + "rustls 0.21.12", "sha1", "thiserror", "url", @@ -6703,6 +6849,15 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +[[package]] +name = "webpki-roots" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "wee_alloc" version = "0.4.5" @@ -6934,6 +7089,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "workspace-template" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 10aad7d1..955b06f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,9 @@ prost.workspace = true tonic-proto.workspace = true appflowy-collaborate = { path = "services/appflowy-collaborate" } +# ai +appflowy-ai-client = { version = "0.1.0" } + # collab collab = { version = "0.1.0" } collab-document = { version = "0.1.0" } @@ -81,7 +84,6 @@ collab-entity = { version = "0.1.0" } #Local crate snowflake = { path = "libs/snowflake" } database.workspace = true -appflowy-ai = { path = "libs/appflowy-ai" } database-entity.workspace = true gotrue = { path = "libs/gotrue" } gotrue-entity = { path = "libs/gotrue-entity" } @@ -144,7 +146,6 @@ members = [ "libs/client-websocket", "libs/client-api-test-util", "libs/wasm-test", - "libs/appflowy-ai", "libs/client-api-wasm", # services "services/appflowy-history", @@ -214,6 +215,7 @@ collab-entity = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = collab-folder = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "85580a5c0e95b5dae4787336faa751da44365760" } collab-document = { git = "https://github.com/AppFlowy-IO/AppFlowy-Collab", rev = "85580a5c0e95b5dae4787336faa751da44365760" } +appflowy-ai-client = { git = "https://github.com/AppFlowy-IO/AppFlowy-AI", rev = "6da7ac710fa8f7c9d70b33adb5bd86eb6f8790d9" } + [features] -ai_enable = [] history = [] diff --git a/deploy.env b/deploy.env index 9f3e585d..60c9e9fd 100644 --- a/deploy.env +++ b/deploy.env @@ -101,6 +101,9 @@ CLOUDFLARE_TUNNEL_TOKEN= # AppFlowy AI OPENAI_API_KEY= +APPFLOWY_AI_URL=http://appflowy_ai:5001 +# The SERVER_NAME is environment variable for AppFlowy AI server. By default, it is localhost:5001 +APPFLOWY_AI_SERVER_NAME=appflowy_ai:5001 # AppFlowy History APPFLOWY_HISTORY_URL=http://history:50051 diff --git a/dev.env b/dev.env index 56fa0cb2..5dfb125f 100644 --- a/dev.env +++ b/dev.env @@ -93,5 +93,8 @@ CLOUDFLARE_TUNNEL_TOKEN= # AppFlowy AI OPENAI_API_KEY= +APPFLOWY_AI_URL=http://localhost:5001 +# The SERVER_NAME is environment variable for AppFlowy AI server. By default, it is localhost:5001 +APPFLOWY_AI_SERVER_NAME=localhost:5001 APPFLOWY_HISTORY_DATABASE_URL=postgres://postgres:password@localhost:5432/postgres diff --git a/docker-compose-ci.yml b/docker-compose-ci.yml index adb7e13b..78261095 100644 --- a/docker-compose-ci.yml +++ b/docker-compose-ci.yml @@ -39,7 +39,7 @@ services: - POSTGRES_HOST=${POSTGRES_HOST:-postgres} volumes: - ./migrations/before:/docker-entrypoint-initdb.d - - postgres_data:/var/lib/postgresql/data + # - postgres_data:/var/lib/postgresql/data redis: restart: on-failure @@ -111,8 +111,8 @@ services: - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY} - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET} - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION} - - APPFLOWY_AI_URL=${APPFLOWY_AI_URL} - APPFLOWY_ACCESS_CONTROL=${APPFLOWY_ACCESS_CONTROL} + - APPFLOWY_AI_URL=${APPFLOWY_AI_URL} build: context: . dockerfile: Dockerfile @@ -131,6 +131,17 @@ services: - ADMIN_FRONTEND_GOTRUE_URL=${ADMIN_FRONTEND_GOTRUE_URL:-http://gotrue:9999} - ADMIN_FRONTEND_APPFLOWY_CLOUD_URL=${ADMIN_FRONTEND_APPFLOWY_CLOUD_URL:-http://appflowy_cloud:8000} + appflowy_ai: + restart: on-failure + image: appflowyinc/appflowy_ai:${APPFLOWY_AI_VERSION:-latest} + ports: + - "5001:5001" + environment: + - FLASK_DEBUG=false + - FLASK_SKIP_DOTENV=true + - OPENAI_API_KEY=${OPENAI_API_KEY} + - SERVER_NAME=${APPFLOWY_AI_SERVER_NAME} + appflowy_history: restart: on-failure build: diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 1fee71be..b3f30383 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -110,14 +110,7 @@ services: - FLASK_DEBUG=false - FLASK_SKIP_DOTENV=true - OPENAI_API_KEY=${OPENAI_API_KEY} + - SERVER_NAME=${APPFLOWY_AI_SERVER_NAME} -# collab_history: -# restart: on-failure -# build: -# context: . -# dockerfile: ./libs/appflowy-history/Dockerfile -# environment: -# - RUST_LOG=${RUST_LOG:-info} -# - APPFLOWY_HISTORY_REDIS_URL=${APPFLOWY_HISTORY_REDIS_URL:-redis://redis:6379} volumes: postgres_data: diff --git a/docker-compose.yml b/docker-compose.yml index 655c4c4d..4d2f465d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,8 +107,8 @@ services: - APPFLOWY_S3_SECRET_KEY=${APPFLOWY_S3_SECRET_KEY} - APPFLOWY_S3_BUCKET=${APPFLOWY_S3_BUCKET} - APPFLOWY_S3_REGION=${APPFLOWY_S3_REGION} - - APPFLOWY_AI_URL=${APPFLOWY_AI_URL} - APPFLOWY_ACCESS_CONTROL=${APPFLOWY_ACCESS_CONTROL} + - APPFLOWY_AI_URL=${APPFLOWY_AI_URL} build: context: . dockerfile: Dockerfile @@ -128,6 +128,17 @@ services: - ADMIN_FRONTEND_GOTRUE_URL=${ADMIN_FRONTEND_GOTRUE_URL:-http://gotrue:9999} - ADMIN_FRONTEND_APPFLOWY_CLOUD_URL=${ADMIN_FRONTEND_APPFLOWY_CLOUD_URL:-http://appflowy_cloud:8000} + appflowy_ai: + restart: on-failure + image: appflowyinc/appflowy_ai:${APPFLOWY_AI_VERSION:-latest} + ports: + - "5001:5001" + environment: + - FLASK_DEBUG=false + - FLASK_SKIP_DOTENV=true + - OPENAI_API_KEY=${OPENAI_API_KEY} + - SERVER_NAME=${APPFLOWY_AI_SERVER_NAME} + volumes: postgres_data: minio_data: diff --git a/libs/appflowy-ai/Cargo.toml b/libs/appflowy-ai/Cargo.toml deleted file mode 100644 index 81bd08f4..00000000 --- a/libs/appflowy-ai/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "appflowy-ai" -version = "0.1.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -reqwest = { workspace = true, features = ["json", "rustls-tls", "cookies"] } -serde.workspace = true -serde_json.workspace = true -thiserror = "1.0.58" -anyhow = "1.0.81" - -[dev-dependencies] -tokio = { workspace = true, features = ["macros", "test-util"] } diff --git a/libs/appflowy-ai/src/client.rs b/libs/appflowy-ai/src/client.rs deleted file mode 100644 index a1a46bda..00000000 --- a/libs/appflowy-ai/src/client.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::entity::{SummarizeRow, TranslateRow}; -use crate::error::AIError; -use reqwest::{Method, RequestBuilder}; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::borrow::Cow; - -#[derive(Clone, Debug)] -pub struct AppFlowyAIClient { - client: reqwest::Client, - url: String, -} - -impl AppFlowyAIClient { - pub fn new(url: &str) -> Self { - let url = url.to_string(); - let client = reqwest::Client::new(); - Self { client, url } - } - - pub async fn summarize_row(&self, json: Value) -> Result { - let url = format!("{}/summarize_row", self.url); - let resp = self - .http_client(Method::POST, &url)? - .json(&json) - .send() - .await?; - AIResponse::::from_response(resp) - .await? - .into_data() - } - - pub async fn translate_row(&self, json: Value) -> Result { - let url = format!("{}/translate_row", self.url); - let resp = self - .http_client(Method::POST, &url)? - .json(&json) - .send() - .await?; - AIResponse::::from_response(resp) - .await? - .into_data() - } - - fn http_client(&self, method: Method, url: &str) -> Result { - let request_builder = self.client.request(method, url); - Ok(request_builder) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AIResponse { - #[serde(skip_serializing_if = "Option::is_none")] - pub data: Option, - - #[serde(default)] - pub message: Cow<'static, str>, -} - -impl AIResponse -where - T: serde::de::DeserializeOwned + 'static, -{ - pub async fn from_response(resp: reqwest::Response) -> Result { - let status_code = resp.status(); - if !status_code.is_success() { - let body = resp.text().await?; - anyhow::bail!("got error code: {}, body: {}", status_code, body) - } - - let bytes = resp.bytes().await?; - let resp = serde_json::from_slice(&bytes)?; - Ok(resp) - } - - pub fn into_data(self) -> Result { - match self.data { - None => Err(AIError::InvalidRequest("Empty payload".to_string())), - Some(data) => Ok(data), - } - } -} diff --git a/libs/appflowy-ai/src/entity.rs b/libs/appflowy-ai/src/entity.rs deleted file mode 100644 index 829b5c8f..00000000 --- a/libs/appflowy-ai/src/entity.rs +++ /dev/null @@ -1,11 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SummarizeRow { - text: String, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct TranslateRow { - text: String, -} diff --git a/libs/appflowy-ai/src/error.rs b/libs/appflowy-ai/src/error.rs deleted file mode 100644 index f23920a9..00000000 --- a/libs/appflowy-ai/src/error.rs +++ /dev/null @@ -1,33 +0,0 @@ -use reqwest::StatusCode; - -#[derive(Debug, thiserror::Error)] -pub enum AIError { - #[error(transparent)] - Internal(#[from] anyhow::Error), - - #[error("Request timeout:{0}")] - RequestTimeout(String), - - #[error("Payload too large:{0}")] - PayloadTooLarge(String), - - #[error("Invalid request:{0}")] - InvalidRequest(String), -} - -impl From for AIError { - fn from(error: reqwest::Error) -> Self { - if error.is_timeout() { - return AIError::RequestTimeout(error.to_string()); - } - - if error.is_request() { - return if error.status() == Some(StatusCode::PAYLOAD_TOO_LARGE) { - AIError::PayloadTooLarge(error.to_string()) - } else { - AIError::InvalidRequest(error.to_string()) - }; - } - AIError::Internal(error.into()) - } -} diff --git a/libs/appflowy-ai/src/lib.rs b/libs/appflowy-ai/src/lib.rs deleted file mode 100644 index 0d1ec3ac..00000000 --- a/libs/appflowy-ai/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod client; -mod entity; -pub mod error; diff --git a/libs/appflowy-ai/tests/main.rs b/libs/appflowy-ai/tests/main.rs deleted file mode 100644 index 5f897e25..00000000 --- a/libs/appflowy-ai/tests/main.rs +++ /dev/null @@ -1 +0,0 @@ -// mod row_test; diff --git a/libs/appflowy-ai/tests/row_test/mod.rs b/libs/appflowy-ai/tests/row_test/mod.rs deleted file mode 100644 index d9fd84d6..00000000 --- a/libs/appflowy-ai/tests/row_test/mod.rs +++ /dev/null @@ -1 +0,0 @@ -mod summarize_test; diff --git a/libs/appflowy-ai/tests/row_test/summarize_test.rs b/libs/appflowy-ai/tests/row_test/summarize_test.rs deleted file mode 100644 index 1ea332a8..00000000 --- a/libs/appflowy-ai/tests/row_test/summarize_test.rs +++ /dev/null @@ -1,10 +0,0 @@ -use appflowy_ai::client::AppFlowyAIClient; -use serde_json::json; - -#[tokio::test] -async fn summarize_row_test() { - let client = AppFlowyAIClient::new("http://localhost:5001"); - let json = json!({"name": "Jack", "age": 25, "city": "New York"}); - let result = client.summarize_row(json).await.unwrap(); - println!("{:?}", result); -} diff --git a/libs/client-api/src/http.rs b/libs/client-api/src/http.rs index 48813f5e..a2531bdf 100644 --- a/libs/client-api/src/http.rs +++ b/libs/client-api/src/http.rs @@ -44,6 +44,7 @@ use url::Url; use crate::ws::ConnectInfo; use gotrue_entity::dto::SignUpResponse::{Authenticated, NotAuthenticated}; use gotrue_entity::dto::{GotrueTokenResponse, UpdateGotrueUserParams, User}; +use shared_entity::dto::ai_dto::{SummarizeRowParams, SummarizeRowResponse}; pub const X_COMPRESSION_TYPE: &str = "X-Compression-Type"; pub const X_COMPRESSION_BUFFER_SIZE: &str = "X-Compression-Buffer-Size"; @@ -1293,6 +1294,29 @@ impl Client { Ok(()) } + #[instrument(level = "info", skip_all)] + pub async fn summarize_row( + &self, + params: SummarizeRowParams, + ) -> Result { + let url = format!( + "{}/api/workspace/{}/summarize_row", + self.base_url, params.workspace_id + ); + + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .json(¶ms) + .send() + .await?; + + log_request_id(&resp); + AppResponse::::from_response(resp) + .await? + .into_data() + } + #[instrument(level = "debug", skip_all, err)] pub async fn http_client_with_auth( &self, diff --git a/libs/shared-entity/src/dto/ai_dto.rs b/libs/shared-entity/src/dto/ai_dto.rs new file mode 100644 index 00000000..94209d17 --- /dev/null +++ b/libs/shared-entity/src/dto/ai_dto.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SummarizeRowParams { + pub workspace_id: String, + pub data: SummarizeRowData, +} + +/// Represents different types of content that can be used to summarize a database row. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum SummarizeRowData { + /// Specifies the identity of the row within the database. + Identity { database_id: String, row_id: String }, + /// Content of the row provided as key-value pairs. + /// For example: + /// ```json + /// { + /// "name": "Jack", + /// "age": 25, + /// "city": "New York" + /// } + Content(Map), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SummarizeRowResponse { + pub text: String, +} diff --git a/libs/shared-entity/src/dto/mod.rs b/libs/shared-entity/src/dto/mod.rs index 34ddea15..6c492672 100644 --- a/libs/shared-entity/src/dto/mod.rs +++ b/libs/shared-entity/src/dto/mod.rs @@ -1,2 +1,3 @@ +pub mod ai_dto; pub mod auth_dto; pub mod workspace_dto; diff --git a/nginx/nginx.conf b/nginx/nginx.conf index b286c1b3..60a4c2e2 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -92,6 +92,13 @@ http { add_header 'Access-Control-Max-Age' 3600 always; } + # AppFlowy AI + location /appflowy_ai/ { + proxy_pass http://appflowy_ai:5001; + proxy_set_header Host $http_host; + proxy_pass_request_headers on; + } + # Minio Web UI # Derive from: https://min.io/docs/minio/linux/integrations/setup-nginx-proxy-with-minio.html # Optional Module, comment this section if you are did not deploy minio in docker-compose.yml diff --git a/src/api/workspace.rs b/src/api/workspace.rs index 9074aba6..3302097a 100644 --- a/src/api/workspace.rs +++ b/src/api/workspace.rs @@ -23,6 +23,7 @@ use database::collab::CollabStorage; use database::user::select_uid_from_email; use database_entity::dto::*; use prost::Message as ProstMessage; +use shared_entity::dto::ai_dto::{SummarizeRowData, SummarizeRowParams, SummarizeRowResponse}; use shared_entity::dto::workspace_dto::*; use shared_entity::response::AppResponseError; use shared_entity::response::{AppResponse, JsonAppResponse}; @@ -128,6 +129,9 @@ pub fn workspace_scope() -> Scope { .service( web::resource("/{workspace_id}/collab_list").route(web::get().to(batch_get_collab_handler)), ) + .service( + web::resource("/{workspace_id}/summarize_row").route(web::post().to(summary_row_handler)), + ) } pub fn collab_scope() -> Scope { @@ -1001,3 +1005,27 @@ async fn parser_realtime_msg( ))), } } + +#[instrument(level = "debug", skip(state, payload), err)] +async fn summary_row_handler( + state: Data, + payload: Json, +) -> Result>> { + let params = payload.into_inner(); + match params.data { + SummarizeRowData::Identity { .. } => { + return Err(AppError::InvalidRequest("Identity data is not supported".to_string()).into()); + }, + SummarizeRowData::Content(content) => { + let text = state + .ai_client + .summarize_row(&content) + .await + .map_err(|err| AppError::InvalidRequest(err.to_string()))? + .text; + + let resp = SummarizeRowResponse { text }; + Ok(AppResponse::Ok().with_data(resp).into()) + }, + } +} diff --git a/src/application.rs b/src/application.rs index a24b258b..2cc2f2c4 100644 --- a/src/application.rs +++ b/src/application.rs @@ -4,8 +4,6 @@ use crate::api::file_storage::file_storage_scope; use crate::api::user::user_scope; use crate::api::workspace::{collab_scope, workspace_scope}; use crate::api::ws::ws_scope; -use access_control::access::{enable_access_control, AccessControl}; - use crate::biz::actix_ws::server::RealtimeServerActor; use crate::biz::casbin::{ CollabAccessControlImpl, RealtimeCollabAccessControlImpl, WorkspaceAccessControlImpl, @@ -24,6 +22,7 @@ use crate::middleware::metrics_mw::MetricsMiddleware; use crate::middleware::request_id::RequestIdMiddleware; use crate::self_signed::create_self_signed_certificate; use crate::state::{AppMetrics, AppState, GoTrueAdmin, UserCache}; +use access_control::access::{enable_access_control, AccessControl}; use actix::Supervisor; use actix_identity::IdentityMiddleware; use actix_session::storage::RedisSessionStore; @@ -31,6 +30,7 @@ use actix_session::SessionMiddleware; use actix_web::cookie::Key; use actix_web::{dev::Server, web, web::Data, App, HttpServer}; use anyhow::{Context, Error}; +use appflowy_ai_client::client::AppFlowyAIClient; use appflowy_collaborate::command::{RTCommandReceiver, RTCommandSender}; use appflowy_collaborate::CollaborationServer; use database::file::bucket_s3_impl::S3BucketStorage; @@ -185,9 +185,7 @@ pub async fn init_state(config: &Config, rt_cmd_tx: RTCommandSender) -> Result Result anyhow::Result<()> { filters.push(format!("storage={}", level)); filters.push(format!("gotrue={}", level)); filters.push(format!("appflowy_collaborate={}", level)); + filters.push(format!("appflowy_ai_client={}", level)); // Load environment variables from .env file dotenvy::dotenv().ok(); diff --git a/src/state.rs b/src/state.rs index 95697449..2876191b 100644 --- a/src/state.rs +++ b/src/state.rs @@ -9,6 +9,7 @@ use crate::config::config::Config; use access_control::access::AccessControl; use access_control::metrics::AccessControlMetrics; use app_error::AppError; +use appflowy_ai_client::client::AppFlowyAIClient; use appflowy_collaborate::CollabRealtimeMetrics; use dashmap::DashMap; use database::file::bucket_s3_impl::S3BucketStorage; @@ -41,8 +42,7 @@ pub struct AppState { pub access_control: AccessControl, pub metrics: AppMetrics, pub gotrue_admin: GoTrueAdmin, - #[cfg(feature = "ai_enable")] - pub appflowy_ai_client: appflowy_ai::client::AppFlowyAIClient, + pub ai_client: AppFlowyAIClient, #[cfg(feature = "history")] pub grpc_history_client: tonic_proto::history::history_client::HistoryClient, diff --git a/tests/ai_test/mod.rs b/tests/ai_test/mod.rs new file mode 100644 index 00000000..8a5982f6 --- /dev/null +++ b/tests/ai_test/mod.rs @@ -0,0 +1 @@ +mod summarize_row; diff --git a/tests/ai_test/summarize_row.rs b/tests/ai_test/summarize_row.rs new file mode 100644 index 00000000..65a02a07 --- /dev/null +++ b/tests/ai_test/summarize_row.rs @@ -0,0 +1,22 @@ +use client_api_test_util::TestClient; +use serde_json::json; +use shared_entity::dto::ai_dto::{SummarizeRowData, SummarizeRowParams}; + +#[tokio::test] +async fn summarize_row_test() { + let test_client = TestClient::new_user().await; + let workspace_id = test_client.workspace_id().await; + + let params = SummarizeRowParams { + workspace_id: workspace_id.clone(), + data: SummarizeRowData::Content( + json!({"name": "Jack", "age": 25, "city": "New York"}) + .as_object() + .unwrap() + .clone(), + ), + }; + + let resp = test_client.api_client.summarize_row(params).await.unwrap(); + assert!(resp.text.contains("Jack")); +} diff --git a/tests/collab/pending_write_test.rs b/tests/collab/pending_write_test.rs index c11af3f5..742003b3 100644 --- a/tests/collab/pending_write_test.rs +++ b/tests/collab/pending_write_test.rs @@ -31,7 +31,7 @@ async fn simulate_small_data_set_write(pool: PgPool) { let storage_queue = StorageQueue::new(collab_cache.clone(), conn, &queue_name); let queries = Arc::new(Mutex::new(Vec::new())); - for i in 0..50 { + for i in 0..10 { // sleep random seconds less than 2 seconds. because the runtime is single-threaded, // we need sleep a little time to let the runtime switch to other tasks. sleep(Duration::from_millis(i % 2)).await; diff --git a/tests/main.rs b/tests/main.rs index 8c0a1175..46f82e6a 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -5,3 +5,5 @@ mod sql_test; mod user; mod websocket; mod workspace; + +mod ai_test; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e778a734..d89c8a56 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -17,7 +17,7 @@ async fn main() -> Result<()> { kill_existing_process(appflowy_history_bin_name).await?; let mut appflowy_cloud_cmd = Command::new("cargo") - .args(["run", "--features", "ai_enable"]) + .args(["run", "--features", ""]) .spawn() .context("Failed to start AppFlowy-Cloud process")?;