From 312411cca2d2cc21a33e0026b78f1b984a7d5460 Mon Sep 17 00:00:00 2001 From: Zack Fu Zi Xiang Date: Thu, 5 Sep 2024 00:06:04 +0800 Subject: [PATCH 1/2] fix: slow query deleting collab embeddings --- ...2e5eeeaabda218c3709540e4a4c2df0180047.json | 15 +++++++++++++++ ...9fda9e0e9323806a6f42a5d8f73243f349b85.json | 15 --------------- ...7ded940d66320270d5ab4431271dd50a9d50b.json | 14 ++++++++++++++ ...03f63e80e4f05a45a0790dd0bde892429ef14.json | 14 -------------- libs/database/src/workspace.rs | 19 ++++++++++++------- ...904000000_af_collab_embeddings_indices.sql | 1 + 6 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 .sqlx/query-1e36d9b3adf957524af88f997f12e5eeeaabda218c3709540e4a4c2df0180047.json delete mode 100644 .sqlx/query-4c985215efcd09915403a7f76449fda9e0e9323806a6f42a5d8f73243f349b85.json create mode 100644 .sqlx/query-cbe43eb734e0afd865a7c1082ba7ded940d66320270d5ab4431271dd50a9d50b.json delete mode 100644 .sqlx/query-e5d51236e5c96ac91c70cd1d1b603f63e80e4f05a45a0790dd0bde892429ef14.json create mode 100644 migrations/20240904000000_af_collab_embeddings_indices.sql diff --git a/.sqlx/query-1e36d9b3adf957524af88f997f12e5eeeaabda218c3709540e4a4c2df0180047.json b/.sqlx/query-1e36d9b3adf957524af88f997f12e5eeeaabda218c3709540e4a4c2df0180047.json new file mode 100644 index 00000000..3934d291 --- /dev/null +++ b/.sqlx/query-1e36d9b3adf957524af88f997f12e5eeeaabda218c3709540e4a4c2df0180047.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE af_workspace\n SET settings = $1\n WHERE workspace_id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "1e36d9b3adf957524af88f997f12e5eeeaabda218c3709540e4a4c2df0180047" +} diff --git a/.sqlx/query-4c985215efcd09915403a7f76449fda9e0e9323806a6f42a5d8f73243f349b85.json b/.sqlx/query-4c985215efcd09915403a7f76449fda9e0e9323806a6f42a5d8f73243f349b85.json deleted file mode 100644 index 45063005..00000000 --- a/.sqlx/query-4c985215efcd09915403a7f76449fda9e0e9323806a6f42a5d8f73243f349b85.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE af_workspace SET settings = $1 WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "4c985215efcd09915403a7f76449fda9e0e9323806a6f42a5d8f73243f349b85" -} diff --git a/.sqlx/query-cbe43eb734e0afd865a7c1082ba7ded940d66320270d5ab4431271dd50a9d50b.json b/.sqlx/query-cbe43eb734e0afd865a7c1082ba7ded940d66320270d5ab4431271dd50a9d50b.json new file mode 100644 index 00000000..e2c64532 --- /dev/null +++ b/.sqlx/query-cbe43eb734e0afd865a7c1082ba7ded940d66320270d5ab4431271dd50a9d50b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM af_collab_embeddings e\n USING af_collab c\n WHERE e.oid = c.oid\n AND e.partition_key = c.partition_key\n AND c.workspace_id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "cbe43eb734e0afd865a7c1082ba7ded940d66320270d5ab4431271dd50a9d50b" +} diff --git a/.sqlx/query-e5d51236e5c96ac91c70cd1d1b603f63e80e4f05a45a0790dd0bde892429ef14.json b/.sqlx/query-e5d51236e5c96ac91c70cd1d1b603f63e80e4f05a45a0790dd0bde892429ef14.json deleted file mode 100644 index 256dd3e3..00000000 --- a/.sqlx/query-e5d51236e5c96ac91c70cd1d1b603f63e80e4f05a45a0790dd0bde892429ef14.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM af_collab_embeddings e WHERE e.oid in (\n SELECT c.oid\n FROM af_collab c\n WHERE c.partition_key = e.partition_key\n AND c.oid = e.oid\n AND c.workspace_id = $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "e5d51236e5c96ac91c70cd1d1b603f63e80e4f05a45a0790dd0bde892429ef14" -} diff --git a/libs/database/src/workspace.rs b/libs/database/src/workspace.rs index 98dd82ee..ba52f9cd 100644 --- a/libs/database/src/workspace.rs +++ b/libs/database/src/workspace.rs @@ -907,7 +907,11 @@ pub async fn upsert_workspace_settings( ) -> Result<(), AppError> { let json = serde_json::to_value(settings)?; sqlx::query!( - r#"UPDATE af_workspace SET settings = $1 WHERE workspace_id = $2"#, + r#" + UPDATE af_workspace + SET settings = $1 + WHERE workspace_id = $2 + "#, json, workspace_id ) @@ -916,12 +920,13 @@ pub async fn upsert_workspace_settings( if settings.disable_search_indexing { sqlx::query!( - r#"DELETE FROM af_collab_embeddings e WHERE e.oid in ( - SELECT c.oid - FROM af_collab c - WHERE c.partition_key = e.partition_key - AND c.oid = e.oid - AND c.workspace_id = $1)"#, + r#" + DELETE FROM af_collab_embeddings e + USING af_collab c + WHERE e.oid = c.oid + AND e.partition_key = c.partition_key + AND c.workspace_id = $1 + "#, workspace_id ) .execute(tx.deref_mut()) diff --git a/migrations/20240904000000_af_collab_embeddings_indices.sql b/migrations/20240904000000_af_collab_embeddings_indices.sql new file mode 100644 index 00000000..338e6c99 --- /dev/null +++ b/migrations/20240904000000_af_collab_embeddings_indices.sql @@ -0,0 +1 @@ +CREATE INDEX af_collab_embeddings_oid_idx ON public.af_collab_embeddings (oid); From 206c2cadaf25259ab56280f75820cb7350fb877a Mon Sep 17 00:00:00 2001 From: Zack Fu Zi Xiang Date: Thu, 5 Sep 2024 12:06:57 +0800 Subject: [PATCH 2/2] chore: cargo deny check advisories --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57478019..343621fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4125,9 +4125,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" dependencies = [ "cc", "pkg-config", @@ -6482,9 +6482,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27144619c6e5802f1380337a209d2ac1c431002dd74c6e60aebff3c506dc4f0c" +checksum = "93334716a037193fac19df402f8571269c84a00852f6a7066b5d2616dcd64d3e" dependencies = [ "sqlx-core", "sqlx-macros", @@ -6495,9 +6495,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a999083c1af5b5d6c071d34a708a19ba3e02106ad82ef7bbd69f5e48266b613b" +checksum = "d4d8060b456358185f7d50c55d9b5066ad956956fddec42ee2e8567134a8936e" dependencies = [ "atoi", "byteorder", @@ -6522,8 +6522,8 @@ dependencies = [ "paste", "percent-encoding", "rust_decimal", - "rustls 0.21.12", - "rustls-pemfile 1.0.4", + "rustls 0.23.12", + "rustls-pemfile 2.1.2", "serde", "serde_json", "sha2", @@ -6535,14 +6535,14 @@ dependencies = [ "tracing", "url", "uuid", - "webpki-roots 0.25.4", + "webpki-roots 0.26.3", ] [[package]] name = "sqlx-macros" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23217eb7d86c584b8cbe0337b9eacf12ab76fe7673c513141ec42565698bb88" +checksum = "cac0692bcc9de3b073e8d747391827297e075c7710ff6276d9f7a1f3d58c6657" dependencies = [ "proc-macro2", "quote", @@ -6553,9 +6553,9 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a099220ae541c5db479c6424bdf1b200987934033c2584f79a0e1693601e776" +checksum = "1804e8a7c7865599c9c79be146dc8a9fd8cc86935fa641d3ea58e5f0688abaa5" dependencies = [ "dotenvy", "either", @@ -6579,9 +6579,9 @@ dependencies = [ [[package]] name = "sqlx-mysql" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5afe4c38a9b417b6a9a5eeffe7235d0a106716495536e7727d1c7f4b1ff3eba6" +checksum = "64bb4714269afa44aef2755150a0fc19d756fb580a67db8885608cf02f47d06a" dependencies = [ "atoi", "base64 0.22.1", @@ -6624,9 +6624,9 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1dbb157e65f10dbe01f729339c06d239120221c9ad9fa0ba8408c4cc18ecf21" +checksum = "6fa91a732d854c5d7726349bb4bb879bb9478993ceb764247660aee25f67c2f8" dependencies = [ "atoi", "base64 0.22.1", @@ -6665,9 +6665,9 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2cdd83c008a622d94499c0006d8ee5f821f36c89b7d625c900e5dc30b5c5ee" +checksum = "d5b2cf34a45953bfd3daaf3db0f7a7878ab9b7a6b91b422d24a7a9e4c857b680" dependencies = [ "atoi", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 1230f96e..eaf18d98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -253,7 +253,7 @@ infra = { path = "libs/infra" } tracing = { version = "0.1", features = ["log"] } gotrue = { path = "libs/gotrue" } redis = "0.25.2" -sqlx = { version = "0.8.0", default-features = false } +sqlx = { version = "0.8.1", default-features = false } dashmap = "5.5.3" futures = "0.3.30" async-stream = "0.3.5"