chore: fix import test (#941)

* chore: fix import test

* chore: enable test

* chore: fix build

* chore: update test

.github/workflows/integration_test.yml

@@ -43,6 +43,7 @@ jobs:
 
       - name: Build Docker Images
         run: |
+          export DOCKER_DEFAULT_PLATFORM=linux/amd64
           docker compose build appflowy_cloud appflowy_history appflowy_worker
 
       - name: Push docker images to docker hub
@@ -56,7 +57,7 @@ jobs:
           docker push appflowyinc/appflowy_worker:${GITHUB_SHA}
           APPFLOWY_HISTORY_VERSION=${GITHUB_SHA}
           APPFLOWY_WORKER_VERSION=${GITHUB_SHA}
-          APPFLOWY_CLOUD_VERSION=0.1.1
+          APPFLOWY_CLOUD_VERSION=${GITHUB_SHA}
 
   test:
     name: Integration Tests

docker-compose-ci.yml

@@ -13,11 +13,13 @@ services:
       - ./nginx/ssl/private_key.key:/etc/nginx/ssl/private_key.key
 
   # You do not need this if you have configured to use your own s3 file storage
+  # You can try to access http://localhost/minio/browser/appflowy in your browser
   minio:
     restart: on-failure
     image: minio/minio
     ports:
-      - "9000:9000"
+      - 9000:9000
+      - 9001:9001
     environment:
       - MINIO_BROWSER_REDIRECT_URL=http://localhost/minio
       - MINIO_ROOT_USER=${APPFLOWY_S3_ACCESS_KEY:-minioadmin}

libs/appflowy-ai-client/Cargo.toml

@@ -26,5 +26,5 @@ infra.workspace = true
 
 [features]
 default = ["client-api"]
-client-api = ["dto", "reqwest", "serde", "serde_json", "tracing", "serde_repr"]
+client-api = ["dto", "reqwest", "serde", "serde_json", "tracing", "serde_repr", "infra/request_util"]
 dto = ["serde", "serde_json", "serde_repr"]

libs/client-api-test/src/client.rs

@@ -4,7 +4,7 @@ use std::borrow::Cow;
 use std::env;
 use tracing::warn;
 use uuid::Uuid;
-//
+
 #[cfg(not(target_arch = "wasm32"))]
 lazy_static! {
   pub static ref LOCALHOST_URL: Cow<'static, str> =

libs/client-api/src/http_chat.rs

@@ -9,9 +9,7 @@ use futures_core::{ready, Stream};
 use pin_project::pin_project;
 use reqwest::Method;
 use serde_json::Value;
-use shared_entity::dto::ai_dto::{
-  CreateTextChatContext, RepeatedRelatedQuestion, STREAM_ANSWER_KEY, STREAM_METADATA_KEY,
-};
+use shared_entity::dto::ai_dto::{RepeatedRelatedQuestion, STREAM_ANSWER_KEY, STREAM_METADATA_KEY};
 use shared_entity::response::{AppResponse, AppResponseError};
 use std::pin::Pin;
 use std::task::{Context, Poll};
@@ -217,25 +215,6 @@ impl Client {
       .await?
       .into_data()
   }
-
-  pub async fn create_chat_context(
-    &self,
-    workspace_id: &str,
-    params: CreateTextChatContext,
-  ) -> Result<(), AppResponseError> {
-    let url = format!(
-      "{}/api/chat/{workspace_id}/{}/context/text",
-      self.base_url, params.chat_id
-    );
-    let resp = self
-      .http_client_with_auth(Method::POST, &url)
-      .await?
-      .json(&params)
-      .send()
-      .await?;
-    log_request_id(&resp);
-    AppResponse::<()>::from_response(resp).await?.into_error()
-  }
 }
 
 #[pin_project]

libs/database-entity/src/dto.rs

@@ -8,6 +8,7 @@ use collab_entity::proto;
 use collab_entity::CollabType;
 use prost::Message;
 use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
 use serde_repr::{Deserialize_repr, Serialize_repr};
 use std::cmp::Ordering;
 use std::collections::HashMap;
@@ -777,6 +778,17 @@ pub struct ChatMetadataData {
   pub size: i64,
 }
 
+impl ChatMetadataData {
+  pub fn from_text(text: String) -> Self {
+    let size = text.len() as i64;
+    Self {
+      content: text,
+      content_type: ChatMetadataContentType::Text,
+      size,
+    }
+  }
+}
+
 impl ChatMetadataData {
   /// Validates the `ChatMetadataData` instance.
   ///
@@ -885,8 +897,14 @@ impl CreateChatMessageParams {
     }
   }
 
-  pub fn with_metadata(mut self, metadata: serde_json::Value) -> Self {
-    self.metadata = Some(metadata);
+  pub fn with_metadata<T: Serialize>(mut self, metadata: T) -> Self {
+    if let Ok(metadata) = serde_json::to_value(&metadata) {
+      if !matches!(metadata, Value::Array(_)) {
+        self.metadata = Some(json!([metadata]));
+      } else {
+        self.metadata = Some(metadata);
+      }
+    }
     self
   }
 }

script/run_ci_server.sh

@@ -4,30 +4,56 @@ set -eo pipefail
 
 cd "$(dirname "$0")/.."
 
-# This script simulates the continuous integration (CI) environment on a local machine. It
-# requires a `.env` file to be located in the project's root directory. The values in the `.env`
-# file must be updated to reflect the specifications of the CI environment.
 # Check if .env file exists in the current directory
-
 if [ -f ".env" ]; then
   echo ".env file exists"
 else
-  echo ".env file does not exist. Copying deploy.env to .env and update the values"
+  echo ".env file does not exist. Please copy deploy.env to .env and update the values."
   exit 1  # Exit with an error code to indicate failure
 fi
 
-# Make sure to update the test client configuration in libs/client-api-test-util/src/client.rs
-# export LOCALHOST_URL="http://localhost"
-# export LOCALHOST_WS_URL="ws://localhost/ws/v1"
-# export LOCALHOST_GOTRUE_URL="http://localhost:gotrue"
+IMAGE_VERSION="${1:-latest}"  # Default to 'latest' if no argument is provided
 
+
+# Stop and remove running containers
+docker ps -q --filter "network=appflowy-cloud_default" | xargs -r docker stop
+docker ps -aq --filter "network=appflowy-cloud_default" | xargs -r docker rm
 docker compose down
-docker compose -f docker-compose-ci.yml pull
 
-# SKIP_BUILD_APPFLOWY_CLOUD=true.
-if [[ -z "${SKIP_BUILD_APPFLOWY_CLOUD+x}" ]]
-then
-  docker build -t appflowy_cloud . && docker build -t appflowy_worker .
-fi
+# Build amd64 images with a new local tag
+# Before running following command, make sure you have the .env file with the correct values
+# For example: SKIP_BUILD=true  ./script/run_ci_server.sh 0.6.51-amd64
+if [[ -z "${SKIP_BUILD+x}" ]]; then
+  docker build --platform=linux/amd64 -t appflowyinc/appflowy_cloud_local:$IMAGE_VERSION -f Dockerfile .
+  docker build --platform=linux/amd64 -t appflowyinc/appflowy_worker_local:$IMAGE_VERSION -f ./services/appflowy-worker/Dockerfile .
+  
+  cat > docker-compose.override.yml <<EOF
+version: '3'
+services:
+  appflowy_cloud:
+    image: appflowyinc/appflowy_cloud:$IMAGE_VERSION
+  appflowy_worker:
+    image: appflowyinc/appflowy_worker:$IMAGE_VERSION
+EOF
 
-docker compose -f docker-compose-ci.yml up -d
+
+  export RUST_LOG=trace
+  docker compose -f docker-compose-ci.yml -f docker-compose.override.yml up -d --build
+  rm docker-compose.override.yml
+else
+  echo "Skipping the build process for appflowy services..."
+  echo "Using image version: $IMAGE_VERSION"
+
+  # Set the image version to the input value
+  export RUST_LOG=trace
+  export APPFLOWY_CLOUD_VERSION=$IMAGE_VERSION
+  export APPFLOWY_WORKER_VERSION=$IMAGE_VERSION
+  export APPFLOWY_HISTORY_VERSION=$IMAGE_VERSION
+  export APPFLOWY_ADMIN_FRONTEND_VERSION=$IMAGE_VERSION
+  docker compose -f docker-compose-ci.yml pull
+
+  echo "Printing the appflowy_cloud image version:"
+  docker images appflowyinc/appflowy_cloud --format "{{.Repository}}:{{.Tag}} (Created: {{.CreatedSince}}, Size: {{.Size}})"
+
+  docker compose -f docker-compose-ci.yml up -d
+fi

tests/ai_test/chat_test.rs

@@ -1,5 +1,5 @@
 use crate::ai_test::util::read_text_from_asset;
-use appflowy_ai_client::dto::{ChatContextLoader, CreateTextChatContext};
+
 use assert_json_diff::assert_json_eq;
 use client_api::entity::{QuestionStream, QuestionStreamValue};
 use client_api_test::{local_ai_test_enabled, TestClient};
@@ -228,22 +228,16 @@ async fn create_chat_context_test() {
     .await
     .unwrap();
 
-  let context = CreateTextChatContext {
-    chat_id: chat_id.clone(),
-    context_loader: ChatContextLoader::Txt,
-    content: "Lacus have lived in the US for five years".to_string(),
-    chunk_size: 1000,
-    chunk_overlap: 20,
-    metadata: Default::default(),
+  let content = "Lacus have lived in the US for five years".to_string();
+  let metadata = ChatMessageMetadata {
+    data: ChatMetadataData::from_text(content),
+    id: chat_id.clone(),
+    name: "".to_string(),
+    source: "appflowy".to_string(),
+    extract: None,
   };
 
-  test_client
-    .api_client
-    .create_chat_context(&workspace_id, context)
-    .await
-    .unwrap();
-
-  let params = CreateChatMessageParams::new_user("Where Lacus live?");
+  let params = CreateChatMessageParams::new_user("Where Lacus live?").with_metadata(metadata);
   let question = test_client
     .api_client
     .create_question(&workspace_id, &chat_id, params)

tests/workspace/import_test.rs

@@ -2,39 +2,37 @@ use anyhow::Error;
 use client_api_test::TestClient;
 use collab_document::importer::define::{BlockType, URL_FIELD};
 use collab_folder::ViewLayout;
-use futures_util::future::join_all;
+
 use std::path::PathBuf;
 use std::time::Duration;
 
-#[tokio::test]
-async fn import_blog_post_four_times_test() {
-  let mut handles = vec![];
-  // Simulate 4 clients, each uploading 3 files concurrently.
-  for _ in 0..4 {
-    let handle = tokio::spawn(async {
-      let client = TestClient::new_user().await;
-      for _ in 0..3 {
-        let _ = upload_file(&client, "blog_post.zip", None).await.unwrap();
-      }
-
-      // the default concurrency limit is 3, so the fourth import should fail
-      let result = upload_file(&client, "blog_post.zip", None).await;
-      assert!(result.is_err());
-      wait_until_num_import_task_complete(&client, 3).await;
-    });
-    handles.push(handle);
-  }
-
-  for result in join_all(handles).await {
-    result.unwrap();
-  }
-}
+// #[tokio::test]
+// async fn import_blog_post_four_times_test() {
+//   let mut handles = vec![];
+//   // Simulate 4 clients, each uploading 3 files concurrently.
+//   for _ in 0..4 {
+//     let handle = tokio::spawn(async {
+//       let client = TestClient::new_user().await;
+//       for _ in 0..3 {
+//         let _ = upload_file(&client, "blog_post.zip", None).await.unwrap();
+//       }
+//
+//       // the default concurrency limit is 3, so the fourth import should fail
+//       upload_file(&client, "blog_post.zip", None).await.unwrap();
+//       wait_until_num_import_task_complete(&client, 3).await;
+//     });
+//     handles.push(handle);
+//   }
+//
+//   for result in join_all(handles).await {
+//     result.unwrap();
+//   }
+// }
 
 #[tokio::test]
 async fn import_blog_post_test() {
   // Step 1: Import the blog post zip
-  let (client, imported_workspace_id) =
-    import_notion_zip_until_complete("blog_post.zip", Some(10)).await;
+  let (client, imported_workspace_id) = import_notion_zip_until_complete("blog_post.zip").await;
 
   // Step 2: Fetch the folder and views
   let folder = client.get_folder(&imported_workspace_id).await;
@@ -104,8 +102,7 @@ async fn import_blog_post_test() {
 
 #[tokio::test]
 async fn import_project_and_task_zip_test() {
-  let (client, imported_workspace_id) =
-    import_notion_zip_until_complete("project&task.zip", None).await;
+  let (client, imported_workspace_id) = import_notion_zip_until_complete("project&task.zip").await;
   let folder = client.get_folder(&imported_workspace_id).await;
   let workspace_database = client.get_workspace_database(&imported_workspace_id).await;
   let space_views = folder.get_views_belong_to(&imported_workspace_id);
@@ -203,6 +200,7 @@ async fn imported_workspace_do_not_become_latest_visit_workspace_test() {
   );
 }
 
+#[allow(dead_code)]
 async fn upload_file(
   client: &TestClient,
   name: &str,
@@ -231,12 +229,10 @@ async fn upload_file(
 }
 
 // upload_after_secs: simulate the delay of uploading the file
-async fn import_notion_zip_until_complete(
-  name: &str,
-  upload_after_secs: Option<u64>,
-) -> (TestClient, String) {
+async fn import_notion_zip_until_complete(name: &str) -> (TestClient, String) {
   let client = TestClient::new_user().await;
-  upload_file(&client, name, upload_after_secs).await.unwrap();
+  let file_path = PathBuf::from(format!("tests/workspace/asset/{name}"));
+  client.api_client.import_file(&file_path).await.unwrap();
   let default_workspace_id = client.workspace_id().await;
 
   // when importing a file, the workspace for the file should be created and it's
@@ -266,7 +262,7 @@ async fn wait_until_num_import_task_complete(client: &TestClient, num: usize) {
   let mut task_completed = false;
   let max_retries = 12;
   let mut retries = 0;
-  while !task_completed && retries < max_retries {
+  while !task_completed {
     tokio::time::sleep(Duration::from_secs(10)).await;
     let tasks = client.api_client.get_import_list().await.unwrap().tasks;
     assert_eq!(tasks.len(), num);
@@ -274,6 +270,11 @@ async fn wait_until_num_import_task_complete(client: &TestClient, num: usize) {
       task_completed = true;
     }
     retries += 1;
+
+    if retries > max_retries {
+      eprintln!("{:?}", tasks);
+      break;
+    }
   }
 
   assert!(

tests/workspace/mod.rs

@@ -1,7 +1,7 @@
 mod access_request;
 mod default_user_workspace;
 mod edit_workspace;
-// mod import_test;
+mod import_test;
 mod invitation_crud;
 mod member_crud;
 mod page_view;