From ab0fa6e7fc3fd8c98db0493eddcd519602000089 Mon Sep 17 00:00:00 2001 From: Khor Shu Heng <32997938+khorshuheng@users.noreply.github.com> Date: Fri, 3 Jan 2025 21:28:48 +0800 Subject: [PATCH] feat: api for publish page to web (#1108) --- libs/client-api/src/http_publish.rs | 25 ++- libs/client-api/src/http_view.rs | 23 +- libs/shared-entity/src/dto/workspace_dto.rs | 6 + src/api/workspace.rs | 39 +++- src/biz/collab/ops.rs | 21 +- src/biz/collab/utils.rs | 94 ++++++++ src/biz/workspace/page_view.rs | 224 +++++++++++++++++++- tests/workspace/page_view.rs | 72 ++++++- 8 files changed, 481 insertions(+), 23 deletions(-) diff --git a/libs/client-api/src/http_publish.rs b/libs/client-api/src/http_publish.rs index faff6c8a..37fb6d28 100644 --- a/libs/client-api/src/http_publish.rs +++ b/libs/client-api/src/http_publish.rs @@ -1,5 +1,5 @@ use bytes::Bytes; -use client_api_entity::workspace_dto::PublishInfoView; +use client_api_entity::workspace_dto::{PublishInfoView, PublishedView}; use client_api_entity::{workspace_dto::PublishedDuplicate, PublishInfo, UpdatePublishNamespace}; use client_api_entity::{ CreateGlobalCommentParams, CreateReactionParams, DeleteGlobalCommentParams, DeleteReactionParams, @@ -301,6 +301,29 @@ impl Client { .into_data() } + #[instrument(level = "debug", skip_all)] + pub async fn get_published_outline( + &self, + publish_namespace: &str, + ) -> Result { + let url = format!( + "{}/api/workspace/published-outline/{}", + self.base_url, publish_namespace, + ); + + let resp = self + .cloud_client + .get(&url) + .send() + .await? + .error_for_status()?; + + log_request_id(&resp); + AppResponse::::from_response(resp) + .await? + .into_data() + } + #[instrument(level = "debug", skip_all)] pub async fn get_default_published_collab( &self, diff --git a/libs/client-api/src/http_view.rs b/libs/client-api/src/http_view.rs index 75e446e8..ad1cfc8d 100644 --- a/libs/client-api/src/http_view.rs +++ b/libs/client-api/src/http_view.rs @@ -1,6 +1,6 @@ use client_api_entity::workspace_dto::{ - CreatePageParams, CreateSpaceParams, MovePageParams, Page, PageCollab, Space, UpdatePageParams, - UpdateSpaceParams, + CreatePageParams, CreateSpaceParams, MovePageParams, Page, PageCollab, PublishPageParams, Space, + UpdatePageParams, UpdateSpaceParams, }; use reqwest::Method; use serde_json::json; @@ -169,6 +169,25 @@ impl Client { .into_data() } + pub async fn publish_page( + &self, + workspace_id: Uuid, + view_id: &str, + params: &PublishPageParams, + ) -> Result<(), AppResponseError> { + let url = format!( + "{}/api/workspace/{}/page-view/{}/publish", + self.base_url, workspace_id, view_id + ); + let resp = self + .http_client_with_auth(Method::POST, &url) + .await? + .json(params) + .send() + .await?; + AppResponse::<()>::from_response(resp).await?.into_error() + } + pub async fn create_space( &self, workspace_id: Uuid, diff --git a/libs/shared-entity/src/dto/workspace_dto.rs b/libs/shared-entity/src/dto/workspace_dto.rs index 0597fbf6..174da343 100644 --- a/libs/shared-entity/src/dto/workspace_dto.rs +++ b/libs/shared-entity/src/dto/workspace_dto.rs @@ -281,6 +281,12 @@ pub struct PublishInfoView { pub info: PublishInfo, } +#[derive(Debug, Serialize, Deserialize)] +pub struct PublishPageParams { + pub publish_name: Option, + pub visible_database_view_ids: Option>, +} + #[derive(Eq, PartialEq, Debug, Hash, Clone, Serialize_repr, Deserialize_repr)] #[repr(u8)] pub enum IconType { diff --git a/src/api/workspace.rs b/src/api/workspace.rs index fc85e299..25293e70 100644 --- a/src/api/workspace.rs +++ b/src/api/workspace.rs @@ -13,8 +13,8 @@ use crate::biz::workspace::ops::{ }; use crate::biz::workspace::page_view::{ create_page, create_space, delete_all_pages_from_trash, delete_trash, get_page_view_collab, - move_page, move_page_to_trash, restore_all_pages_from_trash, restore_page_from_trash, - update_page, update_page_collab_data, update_space, + move_page, move_page_to_trash, publish_page, restore_all_pages_from_trash, + restore_page_from_trash, update_page, update_page_collab_data, update_space, }; use crate::biz::workspace::publish::get_workspace_default_publish_view_info_meta; use crate::biz::workspace::quick_note::{ @@ -189,6 +189,10 @@ pub fn workspace_scope() -> Scope { web::resource("/{workspace_id}/delete-all-pages-from-trash") .route(web::post().to(delete_all_pages_from_trash_handler)), ) + .service( + web::resource("/{workspace_id}/page-view/{view_id}/publish") + .route(web::post().to(publish_page_handler)), + ) .service( web::resource("/{workspace_id}/batch/collab") .route(web::post().to(batch_create_collab_handler)), @@ -1272,6 +1276,37 @@ async fn delete_all_pages_from_trash_handler( Ok(Json(AppResponse::Ok())) } +async fn publish_page_handler( + user_uuid: UserUuid, + path: web::Path<(Uuid, String)>, + payload: Json, + state: Data, +) -> Result>> { + let (workspace_id, view_id) = path.into_inner(); + let uid = state + .user_cache + .get_user_uid(&user_uuid) + .await + .map_err(AppResponseError::from)?; + let PublishPageParams { + publish_name, + visible_database_view_ids, + } = payload.into_inner(); + publish_page( + &state.pg_pool, + &state.collab_access_control_storage, + state.published_collab_store.as_ref(), + uid, + *user_uuid, + workspace_id, + &view_id, + visible_database_view_ids, + publish_name, + ) + .await?; + Ok(Json(AppResponse::Ok())) +} + async fn update_page_view_handler( user_uuid: UserUuid, path: web::Path<(Uuid, String)>, diff --git a/src/biz/collab/ops.rs b/src/biz/collab/ops.rs index b75b71cf..f125004e 100644 --- a/src/biz/collab/ops.rs +++ b/src/biz/collab/ops.rs @@ -76,14 +76,12 @@ use super::folder_view::section_items_to_recent_folder_view; use super::folder_view::section_items_to_trash_folder_view; use super::folder_view::to_dto_folder_view_miminal; use super::publish_outline::collab_folder_to_published_outline; -use super::utils::collab_from_doc_state; use super::utils::collab_to_bin; use super::utils::create_row_document; use super::utils::field_by_id_name_uniq; use super::utils::get_latest_collab; use super::utils::get_latest_collab_database_body; use super::utils::get_latest_collab_database_row_body; -use super::utils::get_latest_collab_encoded; use super::utils::get_latest_collab_folder; use super::utils::get_row_details_serde; use super::utils::type_option_reader_by_id; @@ -434,17 +432,14 @@ pub async fn get_latest_workspace_database( workspace_id: Uuid, ) -> Result<(String, WorkspaceDatabase), AppError> { let workspace_database_oid = select_workspace_database_oid(pg_pool, &workspace_id).await?; - let workspace_database_collab = { - let encoded_collab = get_latest_collab_encoded( - collab_storage, - collab_origin, - &workspace_id.to_string(), - &workspace_database_oid, - CollabType::WorkspaceDatabase, - ) - .await?; - collab_from_doc_state(encoded_collab.doc_state.to_vec(), &workspace_database_oid)? - }; + let workspace_database_collab = get_latest_collab( + collab_storage, + collab_origin, + &workspace_id.to_string(), + &workspace_database_oid, + CollabType::WorkspaceDatabase, + ) + .await?; let workspace_database = WorkspaceDatabase::open(workspace_database_collab) .map_err(|err| AppError::Unhandled(format!("failed to open workspace database: {}", err)))?; diff --git a/src/biz/collab/utils.rs b/src/biz/collab/utils.rs index 666ee319..987a3ef4 100644 --- a/src/biz/collab/utils.rs +++ b/src/biz/collab/utils.rs @@ -19,19 +19,26 @@ use collab_database::rows::RowId; use collab_database::rows::RowMetaKey; use collab_database::template::timestamp_parse::TimestampCellData; use collab_database::workspace_database::NoPersistenceDatabaseCollabService; +use collab_database::workspace_database::WorkspaceDatabaseBody; use collab_document::document::Document; use collab_document::importer::md_importer::MDImporter; use collab_entity::CollabType; use collab_entity::EncodedCollab; use collab_folder::CollabOrigin; use collab_folder::Folder; +use database::collab::select_workspace_database_oid; use database::collab::CollabStorage; use database::collab::GetCollabOrigin; use database_entity::dto::QueryCollab; use database_entity::dto::QueryCollabParams; +use database_entity::dto::QueryCollabResult; +use rayon::iter::IntoParallelIterator; +use rayon::iter::ParallelIterator; +use sqlx::PgPool; use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use uuid::Uuid; use yrs::Map; pub const DEFAULT_SPACE_ICON: &str = "interface_essential/home-3"; @@ -262,6 +269,53 @@ pub async fn get_latest_collab_encoded( .await } +pub async fn batch_get_latest_collab_encoded( + collab_storage: &CollabAccessControlStorage, + collab_origin: GetCollabOrigin, + workspace_id: &str, + oid_list: &[String], + collab_type: CollabType, +) -> Result, AppError> { + let uid = match collab_origin { + GetCollabOrigin::User { uid } => uid, + _ => 0, + }; + let queries: Vec = oid_list + .iter() + .map(|row_id| QueryCollab { + object_id: row_id.to_string(), + collab_type: collab_type.clone(), + }) + .collect(); + let query_collab_results = collab_storage + .batch_get_collab(&uid, workspace_id, queries, true) + .await; + let encoded_collabs = tokio::task::spawn_blocking(move || { + let collabs: HashMap = query_collab_results + .into_par_iter() + .filter_map(|(oid, query_collab_result)| match query_collab_result { + QueryCollabResult::Success { encode_collab_v1 } => { + let decoded_result = EncodedCollab::decode_from_bytes(&encode_collab_v1); + match decoded_result { + Ok(decoded) => Some((oid, decoded)), + Err(err) => { + tracing::error!("Failed to decode collab for row {}: {}", oid, err); + None + }, + } + }, + QueryCollabResult::Failed { error } => { + tracing::error!("Failed to get collab: {:?}", error); + None + }, + }) + .collect(); + collabs + }) + .await?; + Ok(encoded_collabs) +} + pub async fn get_latest_collab( storage: &CollabAccessControlStorage, origin: GetCollabOrigin, @@ -280,6 +334,31 @@ pub async fn get_latest_collab( Ok(collab) } +pub async fn get_latest_collab_workspace_database_body( + pg_pool: &PgPool, + storage: &CollabAccessControlStorage, + origin: GetCollabOrigin, + workspace_id: &str, +) -> Result { + let workspace_uuid = Uuid::parse_str(workspace_id)?; + let ws_db_oid = select_workspace_database_oid(pg_pool, &workspace_uuid).await?; + let mut collab = get_latest_collab( + storage, + origin, + workspace_id, + &ws_db_oid, + CollabType::WorkspaceDatabase, + ) + .await?; + let ws_db = WorkspaceDatabaseBody::open(&mut collab).map_err(|err| { + AppError::Internal(anyhow::anyhow!( + "Failed to open workspace database body: {}", + err + )) + })?; + Ok(ws_db) +} + pub async fn get_latest_collab_folder( collab_storage: &CollabAccessControlStorage, collab_origin: GetCollabOrigin, @@ -344,6 +423,21 @@ pub async fn collab_to_bin(collab: Collab, collab_type: CollabType) -> Result Result, AppError> { + tokio::task::spawn_blocking(move || { + let bin = collab + .encode_collab_v1(|collab| collab_type.validate_require_data(collab)) + .map_err(|e| AppError::Unhandled(e.to_string()))? + .doc_state + .to_vec(); + Ok(bin) + }) + .await? +} + pub fn collab_from_doc_state(doc_state: Vec, object_id: &str) -> Result { let collab = Collab::new_with_source( CollabOrigin::Server, diff --git a/src/biz/workspace/page_view.rs b/src/biz/workspace/page_view.rs index 124cde07..57743742 100644 --- a/src/biz/workspace/page_view.rs +++ b/src/biz/workspace/page_view.rs @@ -6,7 +6,8 @@ use crate::biz::collab::folder_view::{ }; use crate::biz::collab::ops::get_latest_workspace_database; use crate::biz::collab::utils::{ - collab_from_doc_state, get_latest_collab_encoded, get_latest_collab_folder, + batch_get_latest_collab_encoded, collab_from_doc_state, collab_to_doc_state, + get_latest_collab_database_body, get_latest_collab_encoded, get_latest_collab_folder, }; use actix_web::web::Data; use anyhow::anyhow; @@ -41,21 +42,27 @@ use collab_rt_entity::user::RealtimeUser; use database::collab::{select_workspace_database_oid, CollabStorage, GetCollabOrigin}; use database::publish::select_published_view_ids_for_workspace; use database::user::select_web_user_from_uid; -use database_entity::dto::{CollabParams, QueryCollab, QueryCollabResult}; +use database_entity::dto::{ + CollabParams, PublishCollabItem, PublishCollabMetadata, QueryCollab, QueryCollabResult, +}; +use fancy_regex::Regex; use itertools::Itertools; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde_json::json; +use shared_entity::dto::publish_dto::{PublishDatabaseData, PublishViewInfo, PublishViewMetaData}; use shared_entity::dto::workspace_dto::{ FolderView, Page, PageCollab, PageCollabData, Space, SpacePermission, ViewIcon, ViewLayout, }; use sqlx::PgPool; use std::collections::{HashMap, HashSet}; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::time::{Duration, Instant}; use tokio::time::timeout_at; use tracing::instrument; use uuid::Uuid; +use super::publish::PublishedCollabStore; + #[allow(clippy::too_many_arguments)] pub async fn update_space( appflowy_web_metrics: &AppFlowyWebMetrics, @@ -1036,6 +1043,217 @@ pub async fn update_page( Ok(()) } +static INVALID_URL_CHARS: LazyLock = LazyLock::new(|| Regex::new(r"[^\w-]").unwrap()); + +fn replace_invalid_url_chars(input: &str) -> String { + INVALID_URL_CHARS.replace_all(input, "-").to_string() +} + +fn generate_publish_name(view_id: &str, name: &str) -> String { + let id_len = view_id.len(); + let name = replace_invalid_url_chars(name); + let name_len = name.len(); + // The backend limits the publish name to a maximum of 50 characters. + // If the combined length of the ID and the name exceeds 50 characters, + // we will truncate the name to ensure the final result is within the limit. + // The name should only contain alphanumeric characters and hyphens. + let result = format!( + "{}-{}", + &name[..std::cmp::min(49 - id_len, name_len)], + view_id + ); + result +} + +#[allow(clippy::too_many_arguments)] +pub async fn publish_page( + pg_pool: &PgPool, + collab_access_control_storage: &CollabAccessControlStorage, + publish_collab_store: &dyn PublishedCollabStore, + uid: i64, + user_uuid: Uuid, + workspace_id: Uuid, + view_id: &str, + visible_database_view_ids: Option>, + publish_name: Option, +) -> Result<(), AppError> { + let folder = get_latest_collab_folder( + collab_access_control_storage, + GetCollabOrigin::User { uid }, + &workspace_id.to_string(), + ) + .await?; + let view = folder + .get_view(view_id) + .ok_or(AppError::InvalidFolderView(format!( + "View {} not found", + view_id + )))?; + let icon = view + .icon + .as_ref() + .map(|icon| to_dto_view_icon(icon.clone())); + let metadata = PublishViewMetaData { + view: PublishViewInfo { + view_id: view_id.to_string(), + name: view.name.clone(), + icon, + layout: to_dto_view_layout(&view.layout), + extra: view.extra.clone(), + created_by: view.created_by, + last_edited_by: view.last_edited_by, + last_edited_time: view.last_edited_time, + created_at: view.created_at, + child_views: None, + }, + // Note: The use of child views and ancestor views are going to be deprecated in + // appflowy web as there is now endpoint to obtain published outline. + child_views: vec![], + ancestor_views: vec![], + }; + + let publish_data = match view.layout { + collab_folder::ViewLayout::Document => { + generate_publish_data_for_document(collab_access_control_storage, uid, workspace_id, view_id) + .await + }, + collab_folder::ViewLayout::Grid + | collab_folder::ViewLayout::Board + | collab_folder::ViewLayout::Calendar => { + generate_publish_data_for_database( + pg_pool, + collab_access_control_storage, + uid, + workspace_id, + view_id, + visible_database_view_ids, + ) + .await + }, + collab_folder::ViewLayout::Chat => Err(AppError::InvalidRequest( + "AI Chat cannot be published".to_string(), + )), + }?; + publish_collab_store + .publish_collabs( + vec![PublishCollabItem { + meta: PublishCollabMetadata { + view_id: Uuid::parse_str(view_id).unwrap(), + publish_name: publish_name + .map(|name| name.to_string()) + .unwrap_or_else(|| generate_publish_name(view_id, &view.name)), + metadata: serde_json::value::to_value(metadata).unwrap(), + }, + data: publish_data, + }], + &workspace_id, + &user_uuid, + ) + .await?; + Ok(()) +} + +async fn generate_publish_data_for_document( + collab_access_control_storage: &CollabAccessControlStorage, + uid: i64, + workspace_id: Uuid, + view_id: &str, +) -> Result, AppError> { + let collab = get_latest_collab_encoded( + collab_access_control_storage, + GetCollabOrigin::User { uid }, + &workspace_id.to_string(), + view_id, + CollabType::Document, + ) + .await?; + Ok(collab.doc_state.to_vec()) +} + +async fn generate_publish_data_for_database( + pg_pool: &PgPool, + collab_storage: &CollabAccessControlStorage, + uid: i64, + workspace_id: Uuid, + view_id: &str, + visible_database_view_ids: Option>, +) -> Result, AppError> { + let (_, ws_db) = get_latest_workspace_database( + collab_storage, + pg_pool, + GetCollabOrigin::User { uid }, + workspace_id, + ) + .await?; + let db_oid = { + ws_db + .get_database_meta_with_view_id(view_id) + .ok_or(AppError::NoRequiredData(format!( + "Database view {} not found", + view_id + )))? + .database_id + }; + let (db_collab, db_body) = + get_latest_collab_database_body(collab_storage, &workspace_id.to_string(), &db_oid).await?; + let inline_view_id = { + let txn = db_collab.transact(); + db_body.get_inline_view_id(&txn) + }; + let row_ids: Vec = { + let txn = db_collab.transact(); + db_body + .views + .get_row_orders(&txn, &inline_view_id) + .iter() + .map(|ro| ro.id.to_string()) + .collect() + }; + let encoded_rows = batch_get_latest_collab_encoded( + collab_storage, + GetCollabOrigin::User { uid }, + &workspace_id.to_string(), + &row_ids, + CollabType::DatabaseRow, + ) + .await?; + let row_data: HashMap> = encoded_rows + .into_iter() + .map(|(oid, encoded_collab)| (oid, encoded_collab.doc_state.to_vec())) + .collect(); + + let row_document_ids = row_ids + .iter() + .filter_map(|row_id| { + db_body + .block + .get_row_document_id(&RowId::from(row_id.to_owned())) + .map(|doc_id| doc_id.to_string()) + }) + .collect_vec(); + let encoded_row_documents = batch_get_latest_collab_encoded( + collab_storage, + GetCollabOrigin::User { uid }, + &workspace_id.to_string(), + &row_document_ids, + CollabType::Document, + ) + .await?; + let row_document_data: HashMap> = encoded_row_documents + .into_iter() + .map(|(oid, encoded_collab)| (oid, encoded_collab.doc_state.to_vec())) + .collect(); + + let data = PublishDatabaseData { + database_collab: collab_to_doc_state(db_collab, CollabType::Database).await?, + database_row_collabs: row_data, + database_row_document_collabs: row_document_data, + visible_database_view_ids: visible_database_view_ids.unwrap_or(vec![view_id.to_string()]), + database_relations: HashMap::from([(db_oid, view_id.to_string())]), + }; + Ok(serde_json::ser::to_vec(&data)?) +} + pub async fn get_page_view_collab( pg_pool: &PgPool, collab_access_control_storage: &CollabAccessControlStorage, diff --git a/tests/workspace/page_view.rs b/tests/workspace/page_view.rs index 6b4efe43..46892710 100644 --- a/tests/workspace/page_view.rs +++ b/tests/workspace/page_view.rs @@ -9,8 +9,8 @@ use collab_entity::CollabType; use collab_folder::{CollabOrigin, Folder}; use serde_json::{json, Value}; use shared_entity::dto::workspace_dto::{ - CreatePageParams, CreateSpaceParams, IconType, MovePageParams, SpacePermission, UpdatePageParams, - UpdateSpaceParams, ViewIcon, ViewLayout, + CreatePageParams, CreateSpaceParams, IconType, MovePageParams, PublishPageParams, + SpacePermission, UpdatePageParams, UpdateSpaceParams, ViewIcon, ViewLayout, }; use tokio::time::sleep; use uuid::Uuid; @@ -727,3 +727,71 @@ async fn create_space() { assert_eq!(space_info["space_icon"].as_str().unwrap(), "space_icon_3"); assert_eq!(space_info["space_icon_color"].as_str().unwrap(), "#000000"); } + +#[tokio::test] +async fn publish_page() { + let registered_user = generate_unique_registered_user().await; + let web_client = TestClient::user_with_new_device(registered_user.clone()).await; + let workspace_id = web_client.workspace_id().await; + let folder_view = web_client + .api_client + .get_workspace_folder(&workspace_id, Some(2), None) + .await + .unwrap(); + let general_space = &folder_view + .children + .into_iter() + .find(|v| v.name == "General") + .unwrap(); + let database_page_id = general_space + .children + .iter() + .find(|v| v.name == "To-dos") + .unwrap() + .view_id + .clone(); + let document_page_id = general_space + .children + .iter() + .find(|v| v.name == "Getting started") + .unwrap() + .view_id + .clone(); + let page_to_be_published = vec![database_page_id, document_page_id]; + for view_id in &page_to_be_published { + web_client + .api_client + .publish_page( + Uuid::parse_str(&workspace_id).unwrap(), + view_id, + &PublishPageParams { + publish_name: None, + visible_database_view_ids: None, + }, + ) + .await + .unwrap(); + } + let publish_namespace = web_client + .api_client + .get_workspace_publish_namespace(&workspace_id) + .await + .unwrap(); + let published_view = web_client + .api_client + .get_published_outline(&publish_namespace) + .await + .unwrap(); + let published_view_ids: HashSet = published_view + .children + .iter() + .find(|v| v.name == "General") + .unwrap() + .children + .iter() + .map(|v| v.view_id.clone()) + .collect(); + for view_id in &page_to_be_published { + assert!(published_view_ids.contains(view_id)); + } +}