diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index a032a87239021..c22d884730a9b 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -29,8 +29,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json b/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json index 43c68f8c5addd..f52739a9b00f9 100644 --- a/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json +++ b/backend/.sqlx/query-19b59c478744d029c6006b01f04243ad2e0aef485a780daea5d76b0be2bb2ea2.json @@ -39,8 +39,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd35d9..36ddb8ab9fd94 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json b/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json index c45bc18ab4050..eb0dabe8208d2 100644 --- a/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json +++ b/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json @@ -239,8 +239,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json b/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json index cd795e6fecce9..e1ca614dec5fb 100644 --- a/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json +++ b/backend/.sqlx/query-757ef6215d3d385cb3a69e26ee4ca846dd5e7fe7ceb1aa8b3fcd26a2bd30eb2c.json @@ -39,8 +39,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json b/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json index d2dcde86e5db3..326d16b58b4d0 100644 --- a/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json +++ b/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json @@ -154,8 +154,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json b/backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json deleted file mode 100644 index c5a256e0d7a1f..0000000000000 --- a/backend/.sqlx/query-99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO tutorial_progress VALUES ($2, $1::bigint::bit(64)) ON CONFLICT (email) DO UPDATE SET progress = EXCLUDED.progress", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "99e6bffe177e69448b09e82b30d24af00edc86a9bc498f319c1e5bee55d77a8a" -} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index 79c8f0b45d02b..7d00a6f2c7ede 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -29,8 +29,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index da6d213748ec8..af115b609573a 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -184,8 +184,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json b/backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json deleted file mode 100644 index 0972b8b282fe2..0000000000000 --- a/backend/.sqlx/query-a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT progress::bigint FROM tutorial_progress WHERE email = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "progress", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a82eec879838b02e3e0722352fba9f537374dc0658733f7e15bfe42f646d22e2" -} diff --git a/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json b/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json index 19d6878850762..eb9dfd792369e 100644 --- a/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json +++ b/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json @@ -154,8 +154,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json b/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json index b1610d61d65c5..44d146f2088dd 100644 --- a/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json +++ b/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json @@ -121,8 +121,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json index 94acdfb2d882e..1b70af508e19b 100644 --- a/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json +++ b/backend/.sqlx/query-b1a11db5617e8282f5d8256f193be955155b7d0344dfa908451faf2a0ba9269b.json @@ -104,8 +104,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json b/backend/.sqlx/query-c94cd50ff1233025b170efee6489e6637676a8b0435a4612a7efbfff6ea2543d.json similarity index 55% rename from backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json rename to backend/.sqlx/query-c94cd50ff1233025b170efee6489e6637676a8b0435a4612a7efbfff6ea2543d.json index 035e9665e7c4c..64285294c1404 100644 --- a/backend/.sqlx/query-92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6.json +++ b/backend/.sqlx/query-c94cd50ff1233025b170efee6489e6637676a8b0435a4612a7efbfff6ea2543d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', asset.path,\n 'kind', asset.kind,\n 'usages', ARRAY_AGG(jsonb_build_object(\n 'path', asset.usage_path,\n 'kind', asset.usage_kind,\n 'access_type', asset.usage_access_type\n )),\n 'metadata', (CASE\n WHEN asset.kind = 'resource' THEN\n jsonb_build_object('resource_type', resource.resource_type)\n ELSE\n NULL\n END\n )\n )) as \"list!: _\"\n FROM asset\n LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1\n WHERE asset.workspace_id = $1\n AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)\n AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))\n AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1))\n GROUP BY asset.path, asset.kind, resource.resource_type\n ORDER BY asset.path, asset.kind", + "query": "SELECT\n jsonb_strip_nulls(jsonb_build_object(\n 'path', asset.path,\n 'kind', asset.kind,\n 'usages', ARRAY_AGG(jsonb_build_object(\n 'path', asset.usage_path,\n 'kind', asset.usage_kind,\n 'access_type', asset.usage_access_type\n )),\n 'metadata', (CASE\n WHEN asset.kind = 'resource' THEN\n jsonb_build_object('resource_type', resource.resource_type)\n ELSE\n NULL\n END\n )\n )) as \"list!: _\"\n FROM asset\n LEFT JOIN resource ON asset.kind = 'resource'\n AND array_to_string((string_to_array(asset.path, '/'))[1:3], '/') = resource.path -- With specific table, asset path can be e.g u/diego/pg_db/table_name\n AND resource.workspace_id = $1\n WHERE asset.workspace_id = $1\n AND (asset.kind <> 'resource' OR resource.path IS NOT NULL)\n AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1))\n AND (asset.usage_kind <> 'script' OR asset.usage_path = ANY(SELECT path FROM script WHERE workspace_id = $1))\n GROUP BY asset.path, asset.kind, resource.resource_type\n ORDER BY asset.path, asset.kind", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "92f03f4df5e86eb40b255ad0f2cc85e0302c37b0f312366098104cd280a91ef6" + "hash": "c94cd50ff1233025b170efee6489e6637676a8b0435a4612a7efbfff6ea2543d" } diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 5b07bcd9c94ca..33ea7709abe13 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -184,8 +184,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json index 7f581135ed170..8308fdb273b3a 100644 --- a/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json +++ b/backend/.sqlx/query-fa9c2c75b622b23008ef1cdba3cb691ef7d1b0ad9eb2596ccfa7227721a0784f.json @@ -104,8 +104,7 @@ "postgres", "sqs", "gcp", - "mqtt", - "nextcloud" + "mqtt" ] } } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 046504f3f989d..db674bfe8ef6d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2822,7 +2822,7 @@ dependencies = [ "parquet", "rand 0.8.5", "regex", - "sqlparser", + "sqlparser 0.55.0", "tempfile", "tokio", "url", @@ -2899,7 +2899,7 @@ dependencies = [ "parquet", "paste", "recursive", - "sqlparser", + "sqlparser 0.55.0", "tokio", "web-time", ] @@ -3075,7 +3075,7 @@ dependencies = [ "paste", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.55.0", ] [[package]] @@ -3371,7 +3371,7 @@ dependencies = [ "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.55.0", ] [[package]] @@ -8602,15 +8602,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" -dependencies = [ - "memchr", -] - [[package]] name = "notify" version = "6.1.1" @@ -12264,6 +12255,17 @@ dependencies = [ "sqlparser_derive", ] +[[package]] +name = "sqlparser" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + [[package]] name = "sqlparser_derive" version = "0.3.0" @@ -15184,7 +15186,6 @@ dependencies = [ "lazy_static", "libloading 0.8.9", "memchr", - "nom 8.0.0", "object_store", "once_cell", "pep440_rs", @@ -15642,6 +15643,7 @@ dependencies = [ "rustpython-parser", "serde_json", "windmill-parser", + "windmill-parser-sql", ] [[package]] @@ -15705,11 +15707,11 @@ version = "1.591.2" dependencies = [ "anyhow", "lazy_static", - "nom 8.0.0", "regex", "regex-lite", "serde", "serde_json", + "sqlparser 0.59.0", "windmill-parser", ] @@ -15729,6 +15731,7 @@ dependencies = [ "triomphe", "wasm-bindgen", "windmill-parser", + "windmill-parser-sql", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 261f83ca3a862..b7ca6faa9cc98 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -158,7 +158,6 @@ kube.workspace = true k8s-openapi.workspace = true libloading.workspace = true bitflags.workspace = true -nom.workspace = true globset.workspace = true @@ -378,7 +377,6 @@ pg_escape = "0.1.1" async-nats = "0.38.0" nkeys = "0.4.4" nu-parser = { version = "0.101.0", default-features = false } -nom = "8.0.0" globset = "0.4.16" process-wrap = { version = "8.2.1", features = ["tokio1"] } diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 0c50e0f8d8328..2a4f95e49a3da 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,6 +10,7 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true +windmill-parser-sql.workspace = true rustpython-parser.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py/src/asset_parser.rs b/backend/parsers/windmill-parser-py/src/asset_parser.rs index 17a898655bda8..1a3f13c42d1ec 100644 --- a/backend/parsers/windmill-parser-py/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-py/src/asset_parser.rs @@ -2,12 +2,12 @@ use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; use rustpython_parser::{ast::Suite, Parse}; use std::collections::HashMap; use windmill_parser::asset_parser::{ - detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, + asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, }; use AssetUsageAccessType::*; -pub fn parse_assets(input: &str) -> anyhow::Result>> { +pub fn parse_assets(input: &str) -> anyhow::Result> { let ast = Suite::parse(input, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; @@ -15,17 +15,13 @@ pub fn parse_assets(input: &str) -> anyhow::Result ast.into_iter() .for_each(|stmt| assets_finder.visit_stmt(stmt)); - for (kind, name) in assets_finder.var_identifiers.into_values() { + for (kind, path) in assets_finder.var_identifiers.into_values() { // if a db = wmill.datatable() was never used (e.g db.query(...)), // we still want to register the asset as unknown access type - if assets_finder - .assets - .iter() - .all(|a| !(a.kind == kind && a.path == name)) - { + if asset_was_used(&assets_finder.assets, (kind, &path)) == false { assets_finder .assets - .push(ParseAssetsResult { kind, access_type: None, path: name }); + .push(ParseAssetsResult { kind, access_type: None, path }); } } @@ -33,7 +29,7 @@ pub fn parse_assets(input: &str) -> anyhow::Result } struct AssetsFinder { - assets: Vec>, + assets: Vec, var_identifiers: HashMap, } @@ -41,41 +37,27 @@ impl Visitor for AssetsFinder { // Handle assignment statements like: x = wmill.datatable('name') fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { // Check if the value is a call to a tracked function - if let Some((kind, name)) = self.extract_asset_from_call(&node.value) { - // Track all target variables - for target in &node.targets { - if let Expr::Name(name_expr) = target { - let Ok(var_name) = name_expr.id.parse::(); - self.var_identifiers - .insert(var_name, (kind.clone(), name.clone())); - } - } - } else { - // If not wmill.datatable or similar, remove any tracked variables - // It means the identifier is no longer refering to an asset - for target in &node.targets { - if let Expr::Name(name_expr) = target { - let Ok(var_name) = name_expr.id.parse::(); - let removed = self.var_identifiers.remove(&var_name); - // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), - // we still want to register the asset as unknown access type - match removed { - Some((kind, name)) => { - if self - .assets - .iter() - .all(|a| !(a.kind == kind && a.path == name)) - { - self.assets.push(ParseAssetsResult { - kind, - access_type: None, - path: name, - }); - } - } - None => {} + if let Some(Expr::Name(expr_name)) = node.targets.first() { + // Remove any tracked variables with that name in case of reassignment + let Ok(var_name) = expr_name.id.parse::(); + let removed = self.var_identifiers.remove(&var_name); + // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), + // we still want to register the asset as unknown access type + match removed { + Some((kind, path)) => { + if !asset_was_used(&self.assets, (kind, &path)) { + self.assets + .push(ParseAssetsResult { kind, access_type: None, path }); } } + None => {} + } + + if let Some((kind, name)) = self.extract_asset_from_call(&node.value) { + // Track target variable + let Ok(var_name) = expr_name.id.parse::(); + self.var_identifiers + .insert(var_name, (kind.clone(), name.clone())); } } // Continue with generic visit to catch any other assets in the expression @@ -87,7 +69,7 @@ impl Visitor for AssetsFinder { fn visit_expr_constant(&mut self, node: ExprConstant) { match node.value { Constant::Str(s) => { - if let Some((kind, path)) = parse_asset_syntax(&s) { + if let Some((kind, path)) = parse_asset_syntax(&s, false) { self.assets.push(ParseAssetsResult { kind, path: path.to_string(), @@ -108,7 +90,7 @@ impl Visitor for AssetsFinder { if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = &keyword.value { - if let Some((kind, path)) = parse_asset_syntax(s) { + if let Some((kind, path)) = parse_asset_syntax(s, false) { self.assets.push(ParseAssetsResult { kind, path: path.to_string(), @@ -184,8 +166,8 @@ impl AssetsFinder { value, .. }) = node.func.as_ref() { - if let Expr::Name(name_expr) = value.as_ref() { - name_expr.id.parse().map_err(|_| ())? + if let Expr::Name(expr_name) = value.as_ref() { + expr_name.id.parse().map_err(|_| ())? } else { return Err(()); } @@ -197,26 +179,31 @@ impl AssetsFinder { // Continue } else if let Some((kind, ref path)) = self.var_identifiers.get(&obj_name) { if ident == "query" { - let name_expr = node.args.get(0).or_else(|| { + let expr_name = node.args.get(0).or_else(|| { node.keywords .iter() .find(|kw| kw.arg.as_deref() == Some("name")) .map(|kw| &kw.value) }); - match name_expr { - Some(Expr::Constant(ExprConstant { - value: Constant::Str(sql_query), .. - })) => { - let access_type = detect_sql_access_type(&sql_query); - self.assets.push(ParseAssetsResult { - kind: *kind, - path: path.to_string(), - access_type, - }); - return Ok(()); - } + let sql = match expr_name { + Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql, _ => return Err(()), }; + let duckdb_conn_prefix = match kind { + AssetKind::DataTable => "datatable", + AssetKind::Ducklake => "ducklake", + _ => return Ok(()), + }; + let sql = format!("ATTACH '{duckdb_conn_prefix}://{path}' AS dt; USE dt; {sql}"); + + // We use the SQL parser to detect if it's a read or write query + match windmill_parser_sql::parse_assets(&sql) { + Ok(sql_assets) => { + self.assets.extend(sql_assets); + } + _ => {} + } + return Ok(()); } else { return Err(()); } @@ -249,7 +236,9 @@ impl AssetsFinder { match arg_val { Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { - let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); + let path = parse_asset_syntax(&value, false) + .map(|(_, p)| p) + .unwrap_or(&value); self.assets .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); } @@ -259,5 +248,157 @@ impl AssetsFinder { } } -struct Arg(usize, &'static str); // Positional arguments in python can also be used by their name +struct Arg(usize, &'static str); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_py_asset_parser_load_s3() { + let input = r#" +import wmill +def main(): + wmill.load_s3_file('s3:///test.csv') +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/test.csv".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_py_asset_parser_unused_sql() { + let input = r#" +import wmill +def main(): + db = wmill.datatable() +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main".to_string(), + access_type: None + },]) + ); + } + + #[test] + fn test_py_asset_parser_sql_read() { + let input = r#" +import wmill +def main(x: int): + db = wmill.datatable('dt') + return db.query('SELECT * FROM friends WHERE age = $1', x).fetch() +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/friends".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_py_asset_parser_sql_read_write() { + let input = r#" +import wmill +def main(x: int): + db = wmill.datatable('dt') + db.query('UPDATE friends SET x = $1', x).fetch() + db.query('SELECT * FROM friends WHERE age = $1', x).fetch_one() + db.query('SELECT * FROM analytics').fetch() +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/analytics".to_string(), + access_type: Some(R) + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/friends".to_string(), + access_type: Some(RW) + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_multiple_sql_scopes() { + let input = r#" +import wmill +def main(): + def f(x: int): + db = wmill.datatable() + return db.query('SELECT * FROM friends WHERE age = $1', x) + db = wmill.datatable('another1') + return db.query('INSERT INTO customers VALUES ($1)', 0) + +def g(): + db = wmill.ducklake('another2') +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "another1/customers".to_string(), + access_type: Some(W) + }, + ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "another2".to_string(), + access_type: None + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/friends".to_string(), + access_type: Some(R) + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_overriden_var_identifier() { + let input = r#" +import wmill +def main(): + db = wmill.datatable('another1') +def g(): + db = wmill.ducklake() +"#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "another1".to_string(), + access_type: None + }, + ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main".to_string(), + access_type: None + }, + ]) + ); + } +} diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index 8c853df1455f7..41533a6014fe4 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -20,4 +20,4 @@ anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true serde.workspace = true -nom.workspace = true +sqlparser = { version = "0.59.0", features = ["visitor"] } \ No newline at end of file diff --git a/backend/parsers/windmill-parser-sql/src/asset_parser.rs b/backend/parsers/windmill-parser-sql/src/asset_parser.rs index 0323bcb062be4..5ab4bb6a9ff83 100644 --- a/backend/parsers/windmill-parser-sql/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql/src/asset_parser.rs @@ -1,151 +1,626 @@ +use std::collections::HashMap; + +use sqlparser::{ + ast::{ + CopyTarget, Expr, ObjectName, TableFactor, TableObject, Value, ValueWithSpan, Visit, + Visitor, + }, + dialect::DuckDbDialect, + parser::Parser, +}; use windmill_parser::asset_parser::{ - merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, + asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, + ParseAssetsResult, }; use AssetUsageAccessType::*; -use nom::{ - branch::alt, - bytes::complete::{tag, tag_no_case, take_while}, - character::complete::{char, multispace0}, - combinator::opt, - sequence::preceded, - IResult, Parser, -}; +pub fn parse_assets(input: &str) -> anyhow::Result> { + let statements = Parser::parse_sql(&DuckDbDialect, input)?; -pub fn parse_assets<'a>(input: &str) -> anyhow::Result>> { - let mut assets = Vec::new(); - let mut remaining = input; + let mut collector = AssetCollector::new(); + for statement in statements { + let _ = statement.visit(&mut collector); + } - while !remaining.trim().is_empty() { - if let Ok((rest, _)) = parse_comment(remaining) { - remaining = rest; // skip comment - } - if let Ok((rest, res)) = parse_asset(remaining) { - assets.push(res); - remaining = rest; - } else { - remaining = &remaining[1..]; // skip 1 char and continue + for (_, (kind, path)) in collector.var_identifiers { + if !asset_was_used(&collector.assets, (kind, &path)) { + collector + .assets + .push(ParseAssetsResult { kind, access_type: None, path: path }); } } - Ok(merge_assets(assets)) + Ok(merge_assets(collector.assets)) } -fn parse_asset(input: &str) -> IResult<&str, ParseAssetsResult<&str>> { - alt(( - parse_s3_object_read.map(|path| ParseAssetsResult { - path, - kind: AssetKind::S3Object, - access_type: Some(R), - }), - parse_s3_object_write.map(|path| ParseAssetsResult { - path, - kind: AssetKind::S3Object, - access_type: Some(W), - }), - // Parse ambiguous access_types at the end if we could not find precisely read or copy - parse_s3_object_lit.map(|path| ParseAssetsResult { - path, - kind: AssetKind::S3Object, - access_type: None, - }), - parse_resource_lit.map(|path| ParseAssetsResult { - path, - kind: AssetKind::Resource, - access_type: None, - }), - parse_ducklake_lit.map(|path| ParseAssetsResult { - path, - kind: AssetKind::Ducklake, - access_type: None, - }), - parse_datatable_lit.map(|path| ParseAssetsResult { - path, - kind: AssetKind::DataTable, - access_type: None, - }), - )) - .parse(input) +/// Visitor that collects S3 asset literals from SQL statements +struct AssetCollector { + assets: Vec, + // e.g set to Read when we are inside a SELECT ... FROM ... statement + current_access_type_stack: Vec, + // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } + var_identifiers: HashMap, + // e.g USE dl; + currently_used_asset: Option<(AssetKind, String)>, } -/// Any expression that reads an s3 asset -fn parse_s3_object_read(input: &str) -> IResult<&str, &str> { - alt((parse_s3_object_read_fn, parse_s3_object_select_from)).parse(input) -} +impl AssetCollector { + fn new() -> Self { + Self { + assets: Vec::new(), + current_access_type_stack: Vec::with_capacity(8), + var_identifiers: HashMap::new(), + currently_used_asset: None, + } + } -/// Any expression that writes to an s3 asset -fn parse_s3_object_write(input: &str) -> IResult<&str, &str> { - // COPY (...) TO 's3://...' - let (input, _) = (tag_no_case("TO"), multispace0).parse(input)?; - let (input, path) = parse_s3_object_lit(input)?; - Ok((input, path)) -} + // Detect when we do 'a.b' and 'a' is associated with an asset in var_identifiers + // Or when we access 'b' and we did USE a; + fn get_associated_asset_from_obj_name(&self, name: &ObjectName) -> Option { + let access_type = self.current_access_type_stack.last().copied(); + if name.0.len() == 1 { + let ident = name.0.first()?.as_ident()?; + if ident.quote_style.is_some() { + return None; + } + let specific_table = &ident.value; + // We don't want to infer that any simple identifier refers to an asset if + // we are not in a known R/W context + if access_type.is_none() { + return None; + } -/// read_parquet('s3://...') -fn parse_s3_object_read_fn(input: &str) -> IResult<&str, &str> { - let (input, _) = alt(( - tag_no_case("read_parquet"), - tag_no_case("read_csv"), - tag_no_case("read_json"), - )) - .parse(input)?; - let (input, _) = multispace0(input)?; - let (input, _) = char('(')(input)?; - let (input, _) = multispace0(input)?; - let (input, path) = parse_s3_object_lit(input)?; - let (input, _) = multispace0(input)?; - let (input, _) = char(')')(input)?; - Ok((input, path)) -} + if let Some((kind, path)) = &self.currently_used_asset { + let path = format!("{}/{}", path, specific_table); + return Some(ParseAssetsResult { kind: *kind, access_type, path }); + } + } -/// SELECT ... FROM 's3://...' -fn parse_s3_object_select_from(input: &str) -> IResult<&str, &str> { - let (input, _) = tag_no_case("FROM").parse(input)?; - let (input, _) = multispace0(input)?; - let (input, path) = parse_s3_object_lit(input)?; - Ok((input, path)) -} -/// 's3://...' -fn parse_s3_object_lit(input: &str) -> IResult<&str, &str> { - let (input, _) = quote(input)?; - let (input, _) = tag("s3://").parse(input)?; - let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?; - let (input, _) = quote(input)?; - Ok((input, path)) + // Check if the first part of the name (the a in a.b) is associated with an asset + if name.0.len() < 2 { + return None; + } + let ident = name.0.first()?.as_ident()?; + let (kind, path) = self.var_identifiers.get(&ident.value)?; + let path = if name.0.len() == 2 { + let specific_table = &name.0.get(1)?.as_ident()?.value; + format!("{}/{}", path, specific_table) + } else { + path.clone() + }; + Some(ParseAssetsResult { kind: *kind, access_type, path }) + } + + fn handle_string_literal(&mut self, s: &str) { + // Check if the string matches our asset syntax patterns + if let Some((kind, path)) = parse_asset_syntax(s, false) { + if kind == AssetKind::S3Object { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: self.current_access_type_stack.last().copied(), + }); + } + } + } + + fn handle_obj_name_pre(&mut self, name: &ObjectName) { + if let Some(fname) = get_trivial_obj_name(name) { + if is_read_fn(fname) { + self.current_access_type_stack.push(R); + } + } + if let Some(str_lit) = get_str_lit_from_obj_name(name) { + self.handle_string_literal(str_lit); + } + + // Writes to tables should be handled directly when visiting the statement + if self.current_access_type_stack.last() == Some(&R) { + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + } + } + + fn handle_obj_name_post(&mut self, name: &ObjectName) { + if self.current_access_type_stack.is_empty() { + return; + } + if let Some(fname) = get_trivial_obj_name(name) { + if is_read_fn(fname) { + self.current_access_type_stack.pop(); + } + } + } + + fn handle_table_with_joins(&mut self, table_with_joins: &sqlparser::ast::TableWithJoins) { + if let TableFactor::Table { name, .. } = &table_with_joins.relation { + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + } + for join in &table_with_joins.joins { + if let TableFactor::Table { name, .. } = &join.relation { + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + } + } + } } -fn quote(input: &str) -> IResult<&str, char> { - alt((char('\''), char('\"'))).parse(input) +impl Visitor for AssetCollector { + type Break = (); + + fn pre_visit_table_factor( + &mut self, + table_factor: &TableFactor, + ) -> std::ops::ControlFlow { + match table_factor { + TableFactor::Table { name, args, .. } => { + if args.is_none() { + // Avoid Table Functions + self.handle_obj_name_pre(name); + } + } + _ => {} + } + std::ops::ControlFlow::Continue(()) + } + + fn post_visit_table_factor( + &mut self, + table_factor: &TableFactor, + ) -> std::ops::ControlFlow { + match table_factor { + TableFactor::Table { name, .. } => self.handle_obj_name_post(name), + _ => {} + } + std::ops::ControlFlow::Continue(()) + } + + fn pre_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow { + match expr { + Expr::Value(ValueWithSpan { value: Value::SingleQuotedString(s), .. }) => { + self.handle_string_literal(s) + } + Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => { + self.handle_string_literal(s); + } + _ => {} + } + std::ops::ControlFlow::Continue(()) + } + + fn post_visit_expr(&mut self, expr: &Expr) -> std::ops::ControlFlow { + match expr { + Expr::Function(func) => self.handle_obj_name_post(&func.name), + _ => {} + } + std::ops::ControlFlow::Continue(()) + } + + fn pre_visit_statement( + &mut self, + statement: &sqlparser::ast::Statement, + ) -> std::ops::ControlFlow { + match statement { + sqlparser::ast::Statement::Query(_) => { + // don't forget pop() in post_visit_statement + self.current_access_type_stack.push(R); + } + + sqlparser::ast::Statement::Insert(insert) => { + let access_type = if insert.returning.is_some() { RW } else { W }; + self.current_access_type_stack.push(access_type); + match insert.table { + TableObject::TableName(ref name) => { + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + } + _ => {} + } + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::Update { returning, table, from, .. } => { + if let Some(from_tables) = from { + let from_tables = match from_tables { + sqlparser::ast::UpdateTableFromKind::AfterSet(tables) => tables, + sqlparser::ast::UpdateTableFromKind::BeforeSet(tables) => tables, + }; + self.current_access_type_stack.push(R); + for table_with_joins in from_tables { + self.handle_table_with_joins(table_with_joins); + } + self.current_access_type_stack.pop(); + } + + let access_type = if returning.is_some() { RW } else { W }; + self.current_access_type_stack.push(access_type); + + self.handle_table_with_joins(table); + + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::Delete(delete) => { + let access_type = if delete.returning.is_some() { RW } else { W }; + self.current_access_type_stack.push(access_type); + for name in &delete.tables { + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + } + let tables = match &delete.from { + sqlparser::ast::FromTable::WithFromKeyword(tables) => tables, + sqlparser::ast::FromTable::WithoutKeyword(tables) => tables, + }; + for table_with_joins in tables { + self.handle_table_with_joins(table_with_joins); + } + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::CreateTable(create_table) => { + self.current_access_type_stack.push(W); + if let Some(asset) = self.get_associated_asset_from_obj_name(&create_table.name) { + self.assets.push(asset); + } + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::CreateView { name, .. } => { + self.current_access_type_stack.push(W); + if let Some(asset) = self.get_associated_asset_from_obj_name(name) { + self.assets.push(asset); + } + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { + self.current_access_type_stack.push(W); + self.handle_string_literal(filename); + self.current_access_type_stack.pop(); + } + + sqlparser::ast::Statement::AttachDuckDBDatabase { + database_path, + database_alias, + .. + } => { + if let Some((kind, path)) = parse_asset_syntax(&database_path.value, true) { + if kind == AssetKind::Ducklake + || kind == AssetKind::DataTable + || kind == AssetKind::Resource + { + if let Some(database_alias) = database_alias { + self.var_identifiers + .insert(database_alias.value.clone(), (kind, path.to_string())); + } + } + } + } + + sqlparser::ast::Statement::DetachDuckDBDatabase { database_alias, .. } => { + let asset = self.var_identifiers.remove(&database_alias.value); + if self.currently_used_asset == asset { + self.currently_used_asset = None; + } + } + + sqlparser::ast::Statement::Use(sqlparser::ast::Use::Object(obj_name)) => { + if let Some((kind, path)) = self.var_identifiers.get(&obj_name.to_string()) { + self.currently_used_asset = Some((*kind, path.clone())); + } else { + self.currently_used_asset = None; + } + } + + _ => {} + } + + std::ops::ControlFlow::Continue(()) + } + + fn post_visit_statement( + &mut self, + statement: &sqlparser::ast::Statement, + ) -> std::ops::ControlFlow { + match statement { + sqlparser::ast::Statement::Query(_) => { + self.current_access_type_stack.pop(); + } + _ => {} + } + std::ops::ControlFlow::Continue(()) + } + + fn pre_visit_query( + &mut self, + _query: &sqlparser::ast::Query, + ) -> std::ops::ControlFlow { + self.current_access_type_stack.push(R); + std::ops::ControlFlow::Continue(()) + } + + fn post_visit_query( + &mut self, + _query: &sqlparser::ast::Query, + ) -> std::ops::ControlFlow { + self.current_access_type_stack.pop(); + std::ops::ControlFlow::Continue(()) + } + + // We do not use pre_visit_relation because we cannot know if an ObjectName is a table or a function } -fn parse_resource_lit(input: &str) -> IResult<&str, &str> { - let (input, _) = quote(input)?; - let (input, _) = alt((tag("$res:"), tag("res://"))).parse(input)?; - let (input, path) = take_while(|c| c != '\'' && c != '"')(input)?; - let (input, _) = quote(input)?; - Ok((input, path)) +fn is_read_fn(fname: &str) -> bool { + fname.eq_ignore_ascii_case("read_parquet") + || fname.eq_ignore_ascii_case("read_csv") + || fname.eq_ignore_ascii_case("read_json") } -fn parse_ducklake_lit(input: &str) -> IResult<&str, &str> { - let (input, _) = quote(input)?; - let (input, _) = tag("ducklake").parse(input)?; - let (input, path) = - opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?; - let (input, _) = quote(input)?; - Ok((input, path.unwrap_or("main"))) +fn get_trivial_obj_name(name: &sqlparser::ast::ObjectName) -> Option<&str> { + if name.0.len() != 1 { + return None; + } + Some(name.0.first()?.as_ident()?.value.as_str()) } -fn parse_datatable_lit(input: &str) -> IResult<&str, &str> { - let (input, _) = quote(input)?; - let (input, _) = tag("datatable").parse(input)?; - let (input, path) = - opt(preceded(tag("://"), take_while(|c| c != '\'' && c != '"'))).parse(input)?; - let (input, _) = quote(input)?; - Ok((input, path.unwrap_or("main"))) +fn get_str_lit_from_obj_name(name: &ObjectName) -> Option<&str> { + if name.0.len() != 1 { + return None; + } + let ident = name.0.first()?.as_ident()?; + if ident.quote_style != Some('\'') { + return None; + } + Some(ident.value.as_str()) } -fn parse_comment(input: &str) -> IResult<&str, &str> { - let (input, _) = tag("--").parse(input)?; - let (input, comment) = take_while(|c| c != '\n')(input)?; - Ok((input, comment)) +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn test_sql_asset_parser_s3_literals() { + let input = r#" + SELECT * FROM read_parquet('s3:///a.parquet'); + COPY (SELECT * FROM 's3://snd/b.parquet') TO 's3:///c.parquet'; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/a.parquet".to_string(), + access_type: Some(R) + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/c.parquet".to_string(), + access_type: Some(W) + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "snd/b.parquet".to_string(), + access_type: Some(R) + }, + ]) + ); + } + + #[test] + fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT 2; + USE dl; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "my_dl".to_string(), + access_type: None + },]) + ); + } + + #[test] + fn test_sql_asset_parser_attach_dot_notation_read() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + SELECT * FROM dl.table1; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "my_dl/table1".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_attach_dot_notation_write() { + let input = r#" + ATTACH 'datatable://my_dt' AS dt; + SELECT dt.read_bait FROM unrelated_table; -- dt. doesn't access the asset + INSERT INTO dt.table1 VALUES ('test'); + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "my_dt/table1".to_string(), + access_type: Some(W) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_detach() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + DETACH dl; + SELECT * FROM dl.table1; + "#; + let s = parse_assets(input); + assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![])); + } + + #[test] + fn test_sql_asset_parser_implicit_use_asset() { + let input = r#" + ATTACH 'ducklake://my_dl' AS dl; + USE dl; + INSERT INTO table1 VALUES ('test'); + USE memory; + SELECT * FROM table1; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "my_dl/table1".to_string(), + access_type: Some(W) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_default_main() { + let input = r#" + ATTACH 'datatable' AS dl; + INSERT INTO dl.table1 VALUES ('test'); + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/table1".to_string(), + access_type: Some(W) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_create_table() { + let input = r#" + ATTACH 'ducklake' AS dl; USE dl; + CREATE TABLE friends ( + name text, + age int + ); + INSERT INTO friends VALUES ($name, $age); + SELECT * FROM friends; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main/friends".to_string(), + access_type: Some(RW) + },]) + ); + } + + // Make sure a_function is not detected as main/a_function + #[test] + fn test_sql_asset_parser_function_table() { + let input = r#" + ATTACH 'ducklake' AS dl; USE dl; + SELECT * FROM a_function(''); + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main".to_string(), + access_type: None + },]) + ); + } + + #[test] + fn test_sql_asset_parser_delete() { + let input = r#" + ATTACH 'ducklake' AS dl; + USE dl; + DELETE FROM table1; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main/table1".to_string(), + access_type: Some(W) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_update() { + let input = r#" + ATTACH 'ducklake' AS dl; + USE dl; + UPDATE table1 SET id = NULL; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main/table1".to_string(), + access_type: Some(W) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_resource() { + let input = r#" + ATTACH 'res://u/user/pg_resource' AS db (TYPE postgres); + USE db; + SELECT * FROM table1; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg_resource/table1".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_sql_asset_parser_update_with_dot_notation() { + let input = r#" + ATTACH 'ducklake' AS dl; + UPDATE dl.table1 SET id = NULL; + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main/table1".to_string(), + access_type: Some(W) + },]) + ); + } } diff --git a/backend/parsers/windmill-parser-ts/Cargo.toml b/backend/parsers/windmill-parser-ts/Cargo.toml index 5f04b14f3b1b8..b8d577b79f8bc 100644 --- a/backend/parsers/windmill-parser-ts/Cargo.toml +++ b/backend/parsers/windmill-parser-ts/Cargo.toml @@ -15,6 +15,7 @@ serde-wasm-bindgen.workspace = true [dependencies] windmill-parser.workspace = true +windmill-parser-sql.workspace = true swc_common.workspace = true triomphe.workspace = true swc_ecma_parser.workspace = true diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 6ac4c4dd17c40..c702d4259c2be 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -5,12 +5,12 @@ use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, Str}; use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; use swc_ecma_visit::{Visit, VisitWith}; use windmill_parser::asset_parser::{ - detect_sql_access_type, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, + asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, ParseAssetsResult, }; use AssetUsageAccessType::*; -pub fn parse_assets(code: &str) -> anyhow::Result>> { +pub fn parse_assets(code: &str) -> anyhow::Result> { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.ts".into()).into(), code.into()); let lexer = Lexer::new( @@ -41,7 +41,7 @@ pub fn parse_assets(code: &str) -> anyhow::Result> } struct AssetsFinder { - assets: Vec>, + assets: Vec, // The user will write code like: // let sql = wmill.datatable('main') @@ -58,7 +58,7 @@ impl Visit for AssetsFinder { fn visit_lit(&mut self, node: &swc_ecma_ast::Lit) { match node { swc_ecma_ast::Lit::Str(str) => { - if let Some((kind, path)) = parse_asset_syntax(str.value.as_str()) { + if let Some((kind, path)) = parse_asset_syntax(str.value.as_str(), false) { self.assets.push(ParseAssetsResult { kind, path: path.to_string(), @@ -84,19 +84,12 @@ impl Visit for AssetsFinder { // Visit children (this may add new identifiers) node.visit_children_with(self); - // If we find 'let sql = wmill.datatable(...)', - // but no sql`` tagged templates were used, we add - // the asset with unknown access type for var in self.var_identifiers.keys() { if saved_var_identifiers.contains_key(var) { continue; } let (kind, ref path) = self.var_identifiers[var]; - if self - .assets - .iter() - .any(|a| a.kind == kind && &a.path == path) - { + if asset_was_used(&self.assets, (kind, path)) { continue; } self.assets @@ -187,13 +180,22 @@ impl Visit for AssetsFinder { .iter() .map(|quasi| quasi.raw.as_str()) .collect::>() - .join(" "); + .join("$1"); // placeholder for expressions - // Determine access type based on SQL keywords - let access_type = detect_sql_access_type(&sql); + let duckdb_conn_prefix = match kind { + AssetKind::DataTable => "datatable", + AssetKind::Ducklake => "ducklake", + _ => return, + }; + let sql = format!("ATTACH '{duckdb_conn_prefix}://{asset_name}' AS dt; USE dt; {sql}"); - self.assets - .push(ParseAssetsResult { kind, path: asset_name, access_type }); + // We use the SQL parser to detect if it's a read or write query + match windmill_parser_sql::parse_assets(&sql) { + Ok(sql_assets) => { + self.assets.extend(sql_assets); + } + _ => {} + } } } @@ -221,7 +223,9 @@ impl AssetsFinder { match arg_value.map(|e| e.expr.as_ref()) { Some(Expr::Lit(Lit::Str(Str { value, .. }))) => { - let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); + let path = parse_asset_syntax(&value, false) + .map(|(_, p)| p) + .unwrap_or(&value); self.assets .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); } @@ -230,3 +234,164 @@ impl AssetsFinder { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ts_asset_parser_load_s3() { + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + wmill.loadS3File('s3:///test.csv') + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/test.csv".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_ts_asset_parser_unused_sql() { + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + let sql = wmill.datatable('dt') + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt".to_string(), + access_type: None + },]) + ); + } + + #[test] + fn test_ts_asset_parser_sql_read() { + let input = r#" + import * as wmill from "windmill-client" + export async function main(x: number) { + let sql = wmill.datatable('dt') + return await sql`SELECT * FROM friends WHERE age = ${x}`.fetch() + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/friends".to_string(), + access_type: Some(R) + },]) + ); + } + + #[test] + fn test_ts_asset_parser_sql_read_write() { + let input = r#" + import * as wmill from "windmill-client" + export async function main(x: number) { + let sql = wmill.datatable('dt') + await sql`UPDATE friends SET name = 'Pierre' WHERE age = ${x}`.fetch() + let pierre = await sql`SELECT * FROM friends WHERE age = ${x}`.fetchOne() + return await sql`SELECT * FROM analytics`.fetch() + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/analytics".to_string(), + access_type: Some(R) + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "dt/friends".to_string(), + access_type: Some(RW) + }, + ]) + ); + } + + #[test] + fn test_ts_asset_parser_multiple_sql_scopes() { + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + async function f(x: number) { + let sql = wmill.datatable() + return await sql`SELECT * FROM friends WHERE age = ${x}`.fetch() + } + let sql = wmill.datatable('another1') + return await sql`INSERT INTO customers VALUES (${0})`.fetch() + } + + function unused() { + let sql = wmill.ducklake('another2') + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "another1/customers".to_string(), + access_type: Some(W) + }, + ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "another2".to_string(), + access_type: None + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/friends".to_string(), + access_type: Some(R) + }, + ]) + ); + } + + #[test] + fn test_ts_asset_parser_overriden_var_identifier() { + let input = r#" + import * as wmill from "windmill-client" + export async function main() { + let sql = wmill.datatable('another1') + } + function g() { + let sql = wmill.ducklake() + } + "#; + let s = parse_assets(input); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "another1".to_string(), + access_type: None + }, + ParseAssetsResult { + kind: AssetKind::Ducklake, + path: "main".to_string(), + access_type: None + }, + ]) + ); + } +} diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index f6de0a7b0508a..61cc11d81d0e2 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -190,41 +190,36 @@ pub fn parse_ruby(code: &str) -> String { #[cfg(feature = "sql-parser")] #[wasm_bindgen] pub fn parse_assets_sql(code: &str) -> String { - if let Ok(r) = windmill_parser_sql::parse_assets(code) { - return serde_json::to_string(&r).unwrap(); - } else { - return "Invalid".to_string(); + match windmill_parser_sql::parse_assets(code) { + Ok(r) => serde_json::to_string(&r).unwrap(), + Err(err) => format!("err: {:?}", err), } } #[cfg(feature = "ts-parser")] #[wasm_bindgen] pub fn parse_assets_ts(code: &str) -> String { - if let Ok(r) = windmill_parser_ts::parse_assets(code) { - return serde_json::to_string(&r).unwrap(); - } else { - return "Invalid".to_string(); + match windmill_parser_ts::parse_assets(code) { + Ok(r) => serde_json::to_string(&r).unwrap(), + Err(err) => format!("err: {:?}", err), } } #[cfg(feature = "py-parser")] #[wasm_bindgen] pub fn parse_assets_py(code: &str) -> String { - if let Ok(r) = windmill_parser_py::parse_assets(code) { - return serde_json::to_string(&r).unwrap(); - } else { - return "Invalid".to_string(); + match windmill_parser_py::parse_assets(code) { + Ok(r) => serde_json::to_string(&r).unwrap(), + Err(err) => format!("err: {:?}", err), } } #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_assets_ansible(code: &str) -> String { - let o = windmill_parser_yaml::parse_assets(code); - if let Ok(r) = o { - return serde_json::to_string(&r).unwrap(); - } else { - return format!("err: {:?}", o.err().unwrap()); + match windmill_parser_yaml::parse_assets(code) { + Ok(r) => serde_json::to_string(&r).unwrap(), + Err(err) => format!("err: {:?}", err), } } diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs index 0b9ba0d38a89b..48f7984334ff6 100644 --- a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -1,10 +1,10 @@ use windmill_parser::asset_parser::{ - merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, - }; + merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsResult, +}; use crate::{parse_ansible_reqs, ResourceOrVariablePath}; -pub fn parse_assets(input: &str) -> anyhow::Result>> { +pub fn parse_assets(input: &str) -> anyhow::Result> { let mut assets = vec![]; if let (_, Some(ansible_reqs), _) = parse_ansible_reqs(input)? { if let Some(delegate_to_git_repo_details) = ansible_reqs.delegate_to_git_repo { @@ -38,4 +38,3 @@ pub fn parse_assets(input: &str) -> anyhow::Result Ok(merge_assets(assets)) } - diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 26a9203c8ef94..962fa627b4154 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,6 +1,6 @@ use serde::Serialize; -#[derive(Serialize, PartialEq, Clone, Copy)] +#[derive(Serialize, PartialEq, Clone, Copy, Debug)] #[serde(rename_all(serialize = "lowercase"))] pub enum AssetUsageAccessType { R, @@ -10,7 +10,7 @@ pub enum AssetUsageAccessType { use AssetUsageAccessType::*; -#[derive(Serialize, PartialEq, Clone, Copy)] +#[derive(Serialize, PartialEq, Clone, Copy, Debug)] #[serde(rename_all(serialize = "lowercase"))] pub enum AssetKind { S3Object, @@ -19,10 +19,10 @@ pub enum AssetKind { DataTable, } -#[derive(Serialize)] -pub struct ParseAssetsResult> { +#[derive(Serialize, Debug, PartialEq)] +pub struct ParseAssetsResult { pub kind: AssetKind, - pub path: S, + pub path: String, #[serde(skip_serializing_if = "Option::is_none")] pub access_type: Option, // None in case of ambiguity } @@ -34,13 +34,13 @@ pub struct DelegateToGitRepoDetails { pub commit: Option, } -pub fn merge_assets>(assets: Vec>) -> Vec> { - let mut arr: Vec> = vec![]; +pub fn merge_assets(assets: Vec) -> Vec { + let mut arr: Vec = vec![]; for asset in assets { // Remove duplicates if let Some(existing) = arr .iter_mut() - .find(|x| x.path.as_ref() == asset.path.as_ref() && x.kind == asset.kind) + .find(|x| x.path == asset.path && x.kind == asset.kind) { // merge access types existing.access_type = match (asset.access_type, existing.access_type) { @@ -54,12 +54,33 @@ pub fn merge_assets>(assets: Vec>) -> Vec Option<(AssetKind, &str)> { - if s.starts_with("s3://") { +// Will return false if the user assigned an asset to a variable like: +// let sql = wmill.datatable('main') +// But never used it. In that case we don't know which table is being used, +// but we still want to add the main datatable as an asset with unknown access type. +// +// This function takes care of the fact that assets can be suffixed (e.g. "main/users") +pub fn asset_was_used(assets: &Vec, (kind, path): (AssetKind, &String)) -> bool { + assets.iter().any(|a| { + let a_path = a.path.as_str(); + let has_same_path_base = a_path + .strip_prefix(path) + .map(|p| p.starts_with('/')) + .unwrap_or(false); + (has_same_path_base || a_path == path) && a.kind == kind + }) +} + +pub fn parse_asset_syntax(s: &str, enable_default_syntax: bool) -> Option<(AssetKind, &str)> { + if enable_default_syntax && s == "datatable" { + Some((AssetKind::DataTable, "main")) + } else if enable_default_syntax && s == "ducklake" { + Some((AssetKind::Ducklake, "main")) + } else if s.starts_with("s3://") { Some((AssetKind::S3Object, &s[5..])) } else if s.starts_with("res://") { Some((AssetKind::Resource, &s[6..])) @@ -68,41 +89,8 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { } else if s.starts_with("ducklake://") { Some((AssetKind::Ducklake, &s[11..])) } else if s.starts_with("datatable://") { - Some((AssetKind::DataTable, &s[11..])) + Some((AssetKind::DataTable, &s[12..])) } else { None } } - -pub fn detect_sql_access_type(sql: &str) -> Option { - let first_kw = sql - .trim() - .split_whitespace() - .next() - .unwrap_or("") - .to_lowercase(); - - // Check for write operations - let has_write = first_kw.starts_with("insert") - || first_kw.starts_with("update") - || first_kw.starts_with("delete") - || first_kw.starts_with("drop") - || first_kw.starts_with("create") - || first_kw.starts_with("alter") - || first_kw.starts_with("truncate") - || first_kw.starts_with("merge"); - - // Check for read operations - let has_read = first_kw.starts_with("select") - || first_kw.starts_with("with") // CTEs, usually for reads - || first_kw.starts_with("show") - || first_kw.starts_with("describe") - || first_kw.starts_with("explain"); - - match (has_read, has_write) { - (true, true) => Some(RW), - (true, false) => Some(R), - (false, true) => Some(W), - (false, false) => None, // Unknown - couldn't determine - } -} diff --git a/backend/windmill-api/src/assets.rs b/backend/windmill-api/src/assets.rs index 196ed0cfee46c..58633227fcd0b 100644 --- a/backend/windmill-api/src/assets.rs +++ b/backend/windmill-api/src/assets.rs @@ -39,7 +39,9 @@ async fn list_assets( ) )) as "list!: _" FROM asset - LEFT JOIN resource ON asset.kind = 'resource' AND asset.path = resource.path AND resource.workspace_id = $1 + LEFT JOIN resource ON asset.kind = 'resource' + AND array_to_string((string_to_array(asset.path, '/'))[1:3], '/') = resource.path -- With specific table, asset path can be e.g u/diego/pg_db/table_name + AND resource.workspace_id = $1 WHERE asset.workspace_id = $1 AND (asset.kind <> 'resource' OR resource.path IS NOT NULL) AND (asset.usage_kind <> 'flow' OR asset.usage_path = ANY(SELECT path FROM flow WHERE workspace_id = $1)) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 144c11647da66..0f85f1f8502af 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -79,11 +79,11 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.589.3", - "windmill-parser-wasm-regex": "1.589.3", + "windmill-parser-wasm-py": "1.590.0", + "windmill-parser-wasm-regex": "1.590.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.589.3", + "windmill-parser-wasm-ts": "1.590.0", "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", @@ -263,7 +263,6 @@ "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/helper-validator-identifier": "^7.27.1", "js-tokens": "^4.0.0", @@ -279,7 +278,6 @@ "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6.9.0" } @@ -1861,7 +1859,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -2371,6 +2368,7 @@ "integrity": "sha512-Jer+M7DgIwT5IHfTayb4Iw/fkkxWNmC/mqn/nMh9JrbPbkxmyabfLQnhJ+JDn5HK77f84j34lubO3iqFtYAfMg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@floating-ui/core": "^1.3.1", "@floating-ui/dom": "^1.4.5", @@ -2569,6 +2567,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -2914,6 +2913,7 @@ "integrity": "sha512-7TSvMrCdmig5TMyYDW876C5FljhA0wlGixtvASCiqUqtLfmyEEpaysXjC7GhR5mWcGRrCGF+L2Bl1eEaW1wTCA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@standard-schema/spec": "^1.0.0", "@sveltejs/acorn-typescript": "^1.0.5", @@ -2991,6 +2991,7 @@ "integrity": "sha512-YZs/OSKOQAQCnJvM/P+F1URotNnYNeU3P2s4oIpzm1uFaqUEqRxUB0g5ejMjEb5Gjb9/PiBI5Ktrq4rUUF8UVQ==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^5.0.0", "debug": "^4.4.1", @@ -3497,8 +3498,7 @@ "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/ms": { "version": "2.1.0", @@ -3511,8 +3511,7 @@ "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@types/semver": { "version": "7.7.1", @@ -3582,6 +3581,7 @@ "integrity": "sha512-VlJEV0fOQ7BExOsHYAGrgbEiZoi8D+Bl2+f6V2RrXerRSylnp+ZBHmPvaIa8cz0Ajx7WO7Z5RqfgYg7ED1nRhA==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "5.62.0", "@typescript-eslint/types": "5.62.0", @@ -3772,6 +3772,7 @@ "integrity": "sha512-94yVpDbb+ykiT7mK6ToonGnq2GIHEQGBTZTAzGxBGQXcVNCh54YKC2/WkfaDzxy0m6Kgw05kq3FYHKHu+wRdIA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/browser": "4.0.15", "@vitest/mocker": "4.0.15", @@ -4021,6 +4022,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4074,6 +4076,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -4219,7 +4222,6 @@ "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -4247,7 +4249,6 @@ "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -4340,8 +4341,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-2.0.0.tgz", "integrity": "sha512-1ugUSr8BHXRnK23KfuYS+gVMC3LB8QGH9W1iGtDPsNWoQbgtXSExkBu2aDR4epiGWZOjZsj6lDl/N/AqqTC3UA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/base64-js": { "version": "1.5.1", @@ -4468,6 +4468,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -4665,7 +4666,6 @@ "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "camelcase": "^6.3.0", "map-obj": "^4.1.0", @@ -4685,7 +4685,6 @@ "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4699,7 +4698,6 @@ "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -4713,7 +4711,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -4822,6 +4819,7 @@ "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.5.1.tgz", "integrity": "sha512-GIjfiT9dbmHRiYi6Nl2yFCq7kkwdkp1W/lp2J99rX0yo9tgJGn3lKQATztIjb5tVtevcBtIdICNWqlq5+E8/Pw==", "license": "MIT", + "peer": true, "dependencies": { "@kurkle/color": "^0.3.0" }, @@ -5062,7 +5060,6 @@ "integrity": "sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "import-fresh": "^3.3.0", "js-yaml": "^4.1.0", @@ -5127,7 +5124,6 @@ "integrity": "sha512-IQOkD3hbR5KrN93MtcYuad6YPuTSUhntLHDuLEbFWE+ff2/XSZNdZG+LcbbIW5AXKg/WFIfYItIzVoHngHXZzA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12 || >=16" } @@ -5393,6 +5389,7 @@ "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", "license": "ISC", + "peer": true, "engines": { "node": ">=12" } @@ -5446,6 +5443,7 @@ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==", "license": "MIT", + "peer": true, "dependencies": { "@babel/runtime": "^7.21.0" }, @@ -5486,7 +5484,6 @@ "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=10" }, @@ -5500,7 +5497,6 @@ "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "decamelize": "^1.1.0", "map-obj": "^1.0.0" @@ -5518,7 +5514,6 @@ "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5529,7 +5524,6 @@ "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -6015,7 +6009,6 @@ "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "is-arrayish": "^0.2.1" } @@ -6103,6 +6096,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6649,7 +6643,6 @@ "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">= 4.9.1" } @@ -7135,7 +7128,6 @@ "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "global-prefix": "^3.0.0" }, @@ -7149,7 +7141,6 @@ "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ini": "^1.3.5", "kind-of": "^6.0.2", @@ -7165,7 +7156,6 @@ "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -7215,8 +7205,7 @@ "resolved": "https://registry.npmjs.org/globjoin/-/globjoin-0.1.4.tgz", "integrity": "sha512-xYfnw62CKG8nLkZBfWbhWwDw02CHty86jfPcc2cr3ZfeuK9ysoVPPEUxf21bAD/rWAgk52SuBrLJlefNy8mvFg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/gopd": { "version": "1.2.0", @@ -7252,6 +7241,7 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", "integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==", "license": "MIT", + "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } @@ -7310,7 +7300,6 @@ "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=6" } @@ -7532,7 +7521,6 @@ "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -7546,7 +7534,6 @@ "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7559,8 +7546,7 @@ "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/html-tags": { "version": "3.3.1", @@ -7568,7 +7554,6 @@ "integrity": "sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -7652,7 +7637,6 @@ "integrity": "sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -7673,7 +7657,6 @@ "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -7745,8 +7728,7 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/is-binary-path": { "version": "2.1.0", @@ -7851,7 +7833,6 @@ "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7862,7 +7843,6 @@ "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7961,8 +7941,7 @@ "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -7998,8 +7977,7 @@ "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/json-refs": { "version": "3.0.15", @@ -8112,7 +8090,6 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8691,8 +8668,7 @@ "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/lodash.uniq": { "version": "4.5.0", @@ -8770,7 +8746,6 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" }, @@ -8821,7 +8796,6 @@ "integrity": "sha512-APMBEanjybaPzUrfqU0IMU5I0AswKMH7k8OTLs0vvV4KZpExkTkY87nR/zpbuTPj+gARop7aGUbl11pnDfW6xg==", "dev": true, "license": "MIT", - "peer": true, "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -9062,7 +9036,6 @@ "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/minimist": "^1.2.2", "camelcase-keys": "^7.0.0", @@ -9090,7 +9063,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -9784,7 +9756,6 @@ "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "arrify": "^1.0.1", "is-plain-obj": "^1.1.0", @@ -9863,6 +9834,7 @@ "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-editor-api/-/monaco-vscode-editor-api-21.6.0.tgz", "integrity": "sha512-YTxKRHe9d4TvyEzWIqLpJXLyZyO4xFlLgrkgHoWBpomm6gIuwaRJJRpapBZf24oG8AhkniZSPg2iv/84M+ho6g==", "license": "MIT", + "peer": true, "dependencies": { "@codingame/monaco-vscode-5452e2b7-9081-5f95-839b-4ab3544ce28f-common": "21.6.0", "@codingame/monaco-vscode-api": "21.6.0" @@ -10125,7 +10097,6 @@ "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, "license": "BSD-2-Clause", - "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -10477,7 +10448,6 @@ "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.0.0", "error-ex": "^1.3.1", @@ -10823,6 +10793,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", @@ -11011,6 +10982,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" @@ -11400,8 +11372,7 @@ "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.6.tgz", "integrity": "sha512-0sglIs9Wmkzbr8lQwEyIzlDOOC9bGmfVKcJTaxv3vMmd3uo4o4DerC3En0bnmgceeql9BfC8hRkp7cg0fjdVqw==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/postcss-safe-parser": { "version": "6.0.0", @@ -11576,6 +11547,7 @@ "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -11880,7 +11852,6 @@ "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/normalize-package-data": "^2.4.0", "normalize-package-data": "^3.0.2", @@ -11900,7 +11871,6 @@ "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "find-up": "^5.0.0", "read-pkg": "^6.0.0", @@ -11919,7 +11889,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11933,7 +11902,6 @@ "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", "dev": true, "license": "(MIT OR CC0-1.0)", - "peer": true, "engines": { "node": ">=10" }, @@ -11976,7 +11944,6 @@ "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "indent-string": "^5.0.0", "strip-indent": "^4.0.0" @@ -12572,7 +12539,6 @@ "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "ansi-styles": "^4.0.0", "astral-regex": "^2.0.0", @@ -12649,7 +12615,6 @@ "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-expression-parse": "^3.0.0", "spdx-license-ids": "^3.0.0" @@ -12660,8 +12625,7 @@ "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", "dev": true, - "license": "CC-BY-3.0", - "peer": true + "license": "CC-BY-3.0" }, "node_modules/spdx-expression-parse": { "version": "3.0.1", @@ -12669,7 +12633,6 @@ "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "spdx-exceptions": "^2.1.0", "spdx-license-ids": "^3.0.0" @@ -12680,8 +12643,7 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.22.tgz", "integrity": "sha512-4PRT4nh1EImPbt2jASOKHX7PB7I+e4IWNLvkKFDxNhJlfjbYlleYQh285Z/3mPTHSAK/AvdMmw5BNNuYH8ShgQ==", "dev": true, - "license": "CC0-1.0", - "peer": true + "license": "CC0-1.0" }, "node_modules/sprintf-js": { "version": "1.0.3", @@ -12775,7 +12737,6 @@ "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -12801,8 +12762,7 @@ "resolved": "https://registry.npmjs.org/style-search/-/style-search-0.1.0.tgz", "integrity": "sha512-Dj1Okke1C3uKKwQcetra4jSuk0DqbzbYtXipzFlFMZtowbF1x7BKJwB9AayVMyFARvU8EDrZdcax4At/452cAg==", "dev": true, - "license": "ISC", - "peer": true + "license": "ISC" }, "node_modules/style-to-object": { "version": "0.4.4", @@ -12851,7 +12811,6 @@ "integrity": "sha512-78O4c6IswZ9TzpcIiQJIN49K3qNoXTM8zEJzhaTE/xRTCZswaovSEVIa/uwbOltZrk16X4jAxjaOhzz/hTm1Kw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@csstools/css-parser-algorithms": "^2.3.1", "@csstools/css-tokenizer": "^2.2.0", @@ -12934,7 +12893,6 @@ } ], "license": "MIT-0", - "peer": true, "engines": { "node": "^14 || ^16 || >=18" }, @@ -12948,7 +12906,6 @@ "integrity": "sha512-TfW7/1iI4Cy7Y8L6iqNdZQVvdXn0f8B4QcIXmkIbtTIe/Okm/nSlHb4IwGzRVOd3WfSieCgvf5cMzEfySAIl0g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "flat-cache": "^3.2.0" }, @@ -12961,8 +12918,7 @@ "resolved": "https://registry.npmjs.org/known-css-properties/-/known-css-properties-0.29.0.tgz", "integrity": "sha512-Ne7wqW7/9Cz54PDt4I3tcV+hAyat8ypyOGzYRJQfdxnnjeWsTxt1cy8pjvvKeI5kfXuyvULyeeAvwvvtAX3ayQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/stylelint/node_modules/postcss-selector-parser": { "version": "6.1.2", @@ -12985,7 +12941,6 @@ "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=8" } @@ -13120,7 +13075,6 @@ "integrity": "sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "has-flag": "^4.0.0", "supports-color": "^7.0.0" @@ -13150,6 +13104,7 @@ "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.39.12.tgz", "integrity": "sha512-CEzwxFuEycokU8K8CE/OuwVbmei+ivu2HvBGYIdASfMa1hCRSNr4RRkzNSvbAvu6h+BOig2CsZTAEY+WKvwZpA==", "license": "MIT", + "peer": true, "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", @@ -13245,21 +13200,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -13462,8 +13402,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/svg-tags/-/svg-tags-1.0.0.tgz", "integrity": "sha512-ovssysQTa+luh7A5Weu3Rta6FJlFBBbInjOh722LIt6klpU2/HtdUbszju/G4devcvk8PGt7FCLv5wftu3THUA==", - "dev": true, - "peer": true + "dev": true }, "node_modules/svgo": { "version": "3.3.2", @@ -13514,7 +13453,6 @@ "integrity": "sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "ajv": "^8.0.1", "lodash.truncate": "^4.4.2", @@ -13542,6 +13480,7 @@ "integrity": "sha512-6A2rnmW5xZMdw11LYjhcI5846rt9pbLSabY5XPxo+XWdxwZaFEn47Go4NzFiHu9sNNmr/kXivP1vStfvMaK1GQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", @@ -13784,6 +13723,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13846,7 +13786,6 @@ "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13945,6 +13884,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14159,7 +14099,6 @@ "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", "dev": true, "license": "Apache-2.0", - "peer": true, "dependencies": { "spdx-correct": "^3.0.0", "spdx-expression-parse": "^3.0.0" @@ -14214,6 +14153,7 @@ "integrity": "sha512-8wKihlF6EDF8grimwd7GPOhLkQkSIgj6Hlcp0CXhtO3HAXeUUqhgZmJmn07OF8e4PbTusMX6Yxmy1BptVRZsdw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@oxc-project/runtime": "0.99.0", "fdir": "^6.5.0", @@ -14326,6 +14266,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -14359,6 +14300,7 @@ "integrity": "sha512-n1RxDp8UJm6N0IbJLQo+yzLZ2sQCDyl1o0LeugbPWf8+8Fttp29GghsQBjYJVmWq3gBFfe9Hs1spR44vovn2wA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.15", "@vitest/mocker": "4.0.15", @@ -14691,14 +14633,14 @@ "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.589.3", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.589.3.tgz", - "integrity": "sha512-ob1auYfqDyjaliXVluSKAuhfW09TefJ1rJjfDBnV5KCumkTBYGSchrGU+2AJYiuwCtljRE5CzTPBAo6JTaSKHA==" + "version": "1.590.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.590.0.tgz", + "integrity": "sha512-8TojFdxzoRYsx+UqYuhwE/rBZqH9OBs8FWe7A+GLaA3HtKe4NDb57mKpiXFPDnEYXaSF8gs8g+IvNrZJq4qhig==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.589.3", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.589.3.tgz", - "integrity": "sha512-yfYxHtrTuW+vqovPvu1h0t9XmC1cLzE6Quxb6fke3rxKYxQ3scO31zvIeZs2hGlvi/nzJjtCCgmZRyqD6RVQXQ==" + "version": "1.590.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.590.0.tgz", + "integrity": "sha512-M+KqE6x66Utjp/V8XbHHXHtjzXYpkLcd3jINi1hohhYhNru950c68psyu8X7WmXZBnv+AL0NvcOsrBDHW5q0iQ==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", @@ -14711,9 +14653,9 @@ "integrity": "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.589.3", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.589.3.tgz", - "integrity": "sha512-PNyzEWo3qJxlocz1qmeh4IOxm8TzCeyAqqXFO4oyz9Lu6WAyCaSfefhZfaZG01wz3J1hJTZxymB9dDiv0VxgzQ==" + "version": "1.590.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.590.0.tgz", + "integrity": "sha512-OHeIkuYd7RiC946fSzb9HnHSGJzcFbtz57LfRCnRuF/ky7rq26Csr+4uArpTmtBxiyHMVR3U22+qiNe2TjYxJQ==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.561.0", @@ -14864,7 +14806,6 @@ "integrity": "sha512-+QU2zd6OTD8XWIJCbffaiQeH9U73qIqafo1x6V1snCWYGJf6cVE0cDR4D8xRzcEnfI21IFrUPzPGtcPf8AC+Rw==", "dev": true, "license": "ISC", - "peer": true, "dependencies": { "imurmurhash": "^0.1.4", "signal-exit": "^4.0.1" @@ -14879,6 +14820,7 @@ "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "devOptional": true, "license": "MIT", + "peer": true, "engines": { "node": ">=10.0.0" }, @@ -15082,7 +15024,6 @@ "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", "dev": true, "license": "ISC", - "peer": true, "engines": { "node": ">=10" } @@ -15102,6 +15043,7 @@ "resolved": "https://registry.npmjs.org/yjs/-/yjs-13.6.27.tgz", "integrity": "sha512-OIDwaflOaq4wC6YlPBy2L6ceKeKuF7DeTxx+jPzv1FHn9tCZ0ZwSRnUBxD05E3yed46fv/FWJbvR+Ud7x0L7zw==", "license": "MIT", + "peer": true, "dependencies": { "lib0": "^0.2.99" }, @@ -15144,6 +15086,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/frontend/package.json b/frontend/package.json index f330aae681167..71c7d03e421aa 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -153,11 +153,11 @@ "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", - "windmill-parser-wasm-py": "1.589.3", - "windmill-parser-wasm-regex": "1.589.3", + "windmill-parser-wasm-py": "1.590.0", + "windmill-parser-wasm-regex": "1.590.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", - "windmill-parser-wasm-ts": "1.589.3", + "windmill-parser-wasm-ts": "1.590.0", "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", diff --git a/frontend/src/lib/assets/app.css b/frontend/src/lib/assets/app.css index cdccc1a865807..8aeccc37dbe68 100644 --- a/frontend/src/lib/assets/app.css +++ b/frontend/src/lib/assets/app.css @@ -208,4 +208,8 @@ svelte-virtual-list-contents > * + * { input[type="time"]::-webkit-calendar-picker-indicator { margin: 0; padding: 0; +} + +.svelte-flow__edges { + z-index: -10; } \ No newline at end of file diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index f8c4db4aaa052..ca6de47906185 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -22,6 +22,7 @@ dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps dbSchemaOps: IDbSchemaOps refresh?: () => void + initialTableKey?: string } let { dbType, @@ -30,7 +31,8 @@ dbSchemaOps, getColDefs, dbSupportsSchemas, - refresh + refresh, + initialTableKey }: Props = $props() let schemaKeys = $derived(Object.keys(dbSchema.schema ?? {})) @@ -42,10 +44,13 @@ $effect(() => { if (!selected.schemaKey && schemaKeys.length) { - selected = { - schemaKey: - 'public' in dbSchema.schema ? 'public' : 'dbo' in dbSchema.schema ? 'dbo' : schemaKeys[0] - } + let schemaKey = + 'public' in dbSchema.schema ? 'public' : 'dbo' in dbSchema.schema ? 'dbo' : schemaKeys[0] + let tableKey = + initialTableKey && dbSchema.schema?.[schemaKey]?.[initialTableKey] + ? initialTableKey + : undefined + selected = { schemaKey, tableKey } } }) @@ -122,7 +127,7 @@ onConfirm: async () => { askingForConfirmation && (askingForConfirmation.loading = true) try { - await dbSchemaOps.onDelete({ tableKey }) + await dbSchemaOps.onDelete({ tableKey, schema: selected.schemaKey }) refresh?.() sendUserToast(`Table '${tableKey}' deleted successfully`) } catch (e) { diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 2c79c73c6c6d8..bafcdace76ff0 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -194,6 +194,7 @@ input: _input, workspace: $workspaceStore })} + initialTableKey={input.specificTable} {dbType} {refresh} /> diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 9718e3f6526f9..54526592e4a9f 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -741,7 +741,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS } }} tooltip="Attach a Ducklake to your scripts. Ducklake allows you to manipulate large data on S3 blob files through a traditional SQL interface." - documentationLink="https://www.windmill.dev/docs/core_concepts/ducklake" + documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake" itemName="ducklake" loadItems={async () => (await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? 'NO_W' })).map( @@ -783,7 +783,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS } }} tooltip="Attach a datatable to your script." - documentationLink="https://www.windmill.dev/docs/core_concepts/data_tables" + documentationLink="https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables" itemName="data table" loadItems={async () => (await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map( @@ -836,7 +836,7 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS onSelectAndClose={(s3obj) => { let s = `'${formatS3Object(s3obj)}'` if (lang === 'duckdb') { - editor?.insertAtCursor(`SELECT * FROM ${s}`) + editor?.insertAtCursor(`SELECT * FROM ${s};`) } else if (lang === 'python3') { if (!editor?.getCode().includes('import wmill')) { editor?.insertAtBeginning('import wmill\n') diff --git a/frontend/src/lib/components/ExploreAssetButton.svelte b/frontend/src/lib/components/ExploreAssetButton.svelte index 6cbbb0439ea61..806cbd500c725 100644 --- a/frontend/src/lib/components/ExploreAssetButton.svelte +++ b/frontend/src/lib/components/ExploreAssetButton.svelte @@ -58,20 +58,28 @@ {btnClasses} on:click={async () => { if (asset.kind === 'resource' && isDbType(_resourceMetadata?.resource_type)) { + let resourcePath = asset.path.split('/').slice(0, 3).join('/') + let specificTable = asset.path.split('/')[3] as string | undefined dbManagerDrawer?.openDrawer({ type: 'database', resourceType: _resourceMetadata.resource_type, - resourcePath: asset.path + resourcePath, + specificTable }) } else if (asset.kind === 's3object' && isS3Uri(assetUri)) { s3FilePicker?.open(assetUri) } else if (asset.kind === 'ducklake') { - dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake: asset.path }) + let ducklake = asset.path.split('/')[0] + let specificTable = asset.path.split('/')[1] as string | undefined + dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificTable }) } else if (asset.kind === 'datatable') { + let datatable = asset.path.split('/')[0] + let specificTable = asset.path.split('/')[1] as string | undefined dbManagerDrawer?.openDrawer({ type: 'database', resourceType: 'postgresql', - resourcePath: `datatable://${asset.path}` + resourcePath: `datatable://${datatable}`, + specificTable }) } onClick?.() diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 1346502948af6..afeea09dd4a4c 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -58,6 +58,7 @@ import { copilotInfo } from '$lib/aiStore' import JsonInputs from '$lib/components/JsonInputs.svelte' import Toggle from './Toggle.svelte' + import { deepEqual } from 'fast-equals' interface Props { // Exported @@ -158,12 +159,14 @@ $effect(() => { ;[lang, code] untrack(() => { - inferAssets(lang, code).then((newAssets: AssetWithAltAccessType[]) => { + inferAssets(lang, code).then((inferAssetsResult) => { + if (inferAssetsResult.status === 'error') return + let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[] for (const asset of newAssets) { const old = assets?.find((a) => assetEq(a, asset)) if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type } - assets = newAssets + if (!deepEqual(assets, newAssets)) assets = newAssets }) if (lang === 'ansible') { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/queries/deleteTable.ts b/frontend/src/lib/components/apps/components/display/dbtable/queries/deleteTable.ts index 7cd694413cfcf..267d36f9a57c4 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/queries/deleteTable.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/queries/deleteTable.ts @@ -1,6 +1,11 @@ import type { DbType } from '$lib/components/dbTypes' -export function makeDeleteTableQuery(tableKey: string, resourceType: DbType): string { +export function makeDeleteTableQuery( + tableKey: string, + resourceType: DbType, + schema?: string +): string { + if (schema?.length) tableKey = `${schema}.${tableKey}` // same for all sql dbs return `DROP TABLE ${tableKey};` } diff --git a/frontend/src/lib/components/assets/AssetButtons.svelte b/frontend/src/lib/components/assets/AssetButtons.svelte index 57a8c837faf44..f322abed8af83 100644 --- a/frontend/src/lib/components/assets/AssetButtons.svelte +++ b/frontend/src/lib/components/assets/AssetButtons.svelte @@ -28,10 +28,15 @@ datatableNotFound = false, onClick }: Props = $props() + + let resourceDataCacheValue = $derived.by(() => { + let truncatedPath = asset.path.split('/').slice(0, 3).join('/') + return resourceDataCache[truncatedPath] + })
- {#if asset.kind === 'resource' && resourceDataCache[asset.path] !== undefined} + {#if asset.kind === 'resource' && resourceDataCacheValue !== undefined} - {:else if asset.kind === 'resource' && resourceDataCache[asset.path] === undefined} + {:else if asset.kind === 'resource' && resourceDataCacheValue === undefined} {/if} - {:else if assetCanBeExplored(asset, { resource_type: resourceDataCache[asset.path] })} + {:else if assetCanBeExplored(asset, { resource_type: resourceDataCacheValue })} onClick?.()} noText - _resourceMetadata={{ resource_type: resourceDataCache[asset.path] }} + _resourceMetadata={{ resource_type: resourceDataCacheValue }} /> {/if}
diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index e480a86ed100a..2a86595f7419e 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -84,10 +84,14 @@ } for (const asset of assets) { - if (asset.kind !== 'resource' || asset.path in resourceDataCache) continue - ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! }) - .then((resource) => (resourceDataCache[asset.path] = resource.resource_type)) - .catch((err) => (resourceDataCache[asset.path] = undefined)) + if (asset.kind == 'resource') { + let truncatedPath = asset.path.split('/').slice(0, 3).join('/') + if (truncatedPath in resourceDataCache) continue + resourceDataCache[truncatedPath] = undefined // avoid fetching multiple times because of async + ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! }) + .then((r) => (resourceDataCache[truncatedPath] = r.resource_type)) + .catch((err) => console.error("Couldn't fetch resource", truncatedPath, err)) + } } }) }) @@ -107,7 +111,7 @@ 'text-xs flex items-center gap-1.5 px-2 rounded-md relative', 'border', 'bg-surface hover:bg-surface-hover active:bg-surface', - 'transition-all hover:text-primary cursor-pointer' + 'transition-all hover:text-primary backdrop-blur-md cursor-pointer' )} >
name === asset.path)} + !ducklakes.current.find((name) => name === asset.path.split('/')[0])} {@const datatableNotFound = asset.kind === 'datatable' && datatables.current && - !datatables.current.find((name) => name === asset.path)} + !datatables.current.find((name) => name === asset.path.split('/')[0])}
  • onHoverLi?.(asset, 'enter')} @@ -146,9 +150,7 @@
    @@ -162,7 +164,11 @@ Please select manually
    - + {#snippet children({ item })} diff --git a/frontend/src/lib/components/assets/JobAssetsViewer.svelte b/frontend/src/lib/components/assets/JobAssetsViewer.svelte index 6375783595d0a..060803884b5e6 100644 --- a/frontend/src/lib/components/assets/JobAssetsViewer.svelte +++ b/frontend/src/lib/components/assets/JobAssetsViewer.svelte @@ -34,11 +34,11 @@ (x) => x.kind + x.path ) } + if (job.job_kind === 'script') { - return [ - ...(await inferAssets(job.language!, job.raw_code ?? '')), - ...parseInputArgsAssets(job.args ?? {}) - ] + let inferAssetsResult = await inferAssets(job.language!, job.raw_code ?? '') + let assets = inferAssetsResult.status === 'ok' ? inferAssetsResult.assets : [] + return [...assets, ...parseInputArgsAssets(job.args ?? {})] } return [] } @@ -53,10 +53,14 @@ let resourceDataCache: Record = $state({}) $effect(() => { for (const asset of assets.value ?? []) { - if (asset.kind !== 'resource' || asset.path in resourceDataCache) continue - ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! }) - .then((resource) => (resourceDataCache[asset.path] = resource.resource_type)) - .catch((err) => (resourceDataCache[asset.path] = undefined)) + if (asset.kind == 'resource') { + let truncatedPath = asset.path.split('/').slice(0, 3).join('/') + if (truncatedPath in resourceDataCache) continue + resourceDataCache[truncatedPath] = undefined // avoid fetching multiple times because of async + ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! }) + .then((r) => (resourceDataCache[truncatedPath] = r.resource_type)) + .catch((err) => console.error("Couldn't fetch resource", truncatedPath, err)) + } } }) diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index cb7b824a64d06..0e187e1578afe 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -28,6 +28,10 @@ export function formatAsset(asset: Asset): string { return 'unknown' } +export function formatShortAssetPath(asset: Asset): string { + return asset.path.split('/').pop() || asset.path +} + export function getAssetUsagePageUri(usage: ListAssetsResponse[number]['usages'][number]) { if (usage.kind === 'script') { return `/scripts/get/${usage.path}` @@ -93,8 +97,8 @@ export function formatAssetAccessType(accessType: AssetUsageAccessType | undefin } export function getAccessType(asset: AssetWithAltAccessType): AssetUsageAccessType | undefined { - if (asset.alt_access_type) return asset.alt_access_type if (asset.access_type) return asset.access_type + if (asset.alt_access_type) return asset.alt_access_type } export function getFlowModuleAssets( diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 0e0386c25f287..70da66b61a70c 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -112,7 +112,7 @@ export function dbTableOpsWithPreviewScripts({ } export type IDbSchemaOps = { - onDelete: (params: { tableKey: string }) => Promise + onDelete: (params: { tableKey: string; schema?: string }) => Promise onCreate: (params: { values: CreateTableValues; schema?: string }) => Promise previewCreateSql: (params: { values: CreateTableValues; schema?: string }) => string } @@ -128,8 +128,8 @@ export function dbSchemaOpsWithPreviewScripts({ const dbArg = getDatabaseArg(input) const language = getLanguageByResourceType(dbType) return { - onDelete: async ({ tableKey }) => { - let deleteQuery = makeDeleteTableQuery(tableKey, dbType) + onDelete: async ({ tableKey, schema }) => { + let deleteQuery = makeDeleteTableQuery(tableKey, dbType, schema) if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake) await runScriptAndPollResult({ workspace, diff --git a/frontend/src/lib/components/dbTypes.ts b/frontend/src/lib/components/dbTypes.ts index 6e002efa06b5a..3f4eed08760dd 100644 --- a/frontend/src/lib/components/dbTypes.ts +++ b/frontend/src/lib/components/dbTypes.ts @@ -3,8 +3,13 @@ export type DbInput = type: 'database' resourceType: DbType resourcePath: string + specificTable?: string + } + | { + type: 'ducklake' + ducklake: string + specificTable?: string } - | { type: 'ducklake'; ducklake: string } export type DbType = (typeof dbTypes)[number] export const dbTypes = [ diff --git a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte index cd4ebc98c49e3..5600adccfd1d1 100644 --- a/frontend/src/lib/components/flows/FlowAssetsHandler.svelte +++ b/frontend/src/lib/components/flows/FlowAssetsHandler.svelte @@ -73,13 +73,14 @@ (m) => getFlowModuleAssets(m, flowGraphAssetsCtx?.val.additionalAssetsMap) ?? [] ) ?? [] for (const asset of assets) { - if (asset.kind !== 'resource' || asset.path in resMetadataCache) continue - resMetadataCache[asset.path] = undefined // avoid fetching multiple times because of async - ResourceService.getResource({ path: asset.path, workspace: $workspaceStore! }) - .then((r) => (resMetadataCache[asset.path] = { resource_type: r.resource_type })) - .catch((err) => { - console.error("Couldn't fetch resource", asset.path, err) - }) + if (asset.kind == 'resource') { + let truncatedPath = asset.path.split('/').slice(0, 3).join('/') + if (truncatedPath in resMetadataCache) continue + resMetadataCache[truncatedPath] = undefined // avoid fetching multiple times because of async + ResourceService.getResource({ path: truncatedPath, workspace: $workspaceStore! }) + .then((r) => (resMetadataCache[truncatedPath] = { resource_type: r.resource_type })) + .catch((err) => console.error("Couldn't fetch resource", truncatedPath, err)) + } } }) @@ -119,14 +120,14 @@ }) async function parseAndUpdateRawScriptModule(v: RawScript) { - try { - let parsedAssets: AssetWithAltAccessType[] = await inferAssets(v.language, v.content) - for (const asset of parsedAssets) { - const old = v.assets?.find((a) => assetEq(a, asset)) - if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type - } - if (!deepEqual(v.assets, parsedAssets)) v.assets = parsedAssets - } catch (e) {} + let inferAssetsResult = await inferAssets(v.language, v.content) + if (inferAssetsResult.status === 'error') return + let newAssets = inferAssetsResult.assets as AssetWithAltAccessType[] + for (const asset of newAssets) { + const old = v.assets?.find((a) => assetEq(a, asset)) + if (old?.alt_access_type) asset.alt_access_type = old.alt_access_type + } + if (!deepEqual(v.assets, newAssets)) v.assets = newAssets } // Check for raw script modules whose assets were not parsed. Useful for flows created diff --git a/frontend/src/lib/components/flows/propPicker/OutputPicker.svelte b/frontend/src/lib/components/flows/propPicker/OutputPicker.svelte index 0328bc18aa332..14701b17995f5 100644 --- a/frontend/src/lib/components/flows/propPicker/OutputPicker.svelte +++ b/frontend/src/lib/components/flows/propPicker/OutputPicker.svelte @@ -122,7 +122,7 @@ }} > -
    +
    diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte index 9a8eda2a93206..f4ded7dfa2f37 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte @@ -220,6 +220,7 @@ import { assetEq, formatAssetKind, + formatShortAssetPath, getAccessType, type AssetWithAltAccessType } from '$lib/components/assets/lib' @@ -246,9 +247,11 @@ let { data }: Props = $props() const isSelected = $derived(assetEq(flowGraphAssetsCtx?.val.selectedAsset, data.asset)) - const cachedResourceMetadata = $derived( - flowGraphAssetsCtx?.val.resourceMetadataCache[data.asset.path] - ) + const cachedResourceMetadata = $derived.by(() => { + if (data.asset.kind !== 'resource') return undefined + let truncatedPath = data.asset.path.split('/').slice(0, 3).join('/') + return flowGraphAssetsCtx?.val.resourceMetadataCache[truncatedPath] + }) const usageCount = $derived(flowGraphAssetsCtx?.val.computeAssetsCount?.(data.asset)) const colors = $derived(getNodeColorClasses(undefined, isSelected)) @@ -271,14 +274,11 @@ > - {data.asset.path} + {formatShortAssetPath(data.asset)} {#if data.asset.kind === 'resource' && cachedResourceMetadata === undefined} diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte index 9c637dd819cf4..2353d37aa812f 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetsOverflowedNode.svelte @@ -9,6 +9,7 @@ import type { FlowGraphAssetContext } from '$lib/components/flows/types' import { getContext } from 'svelte' import { assetEq } from '$lib/components/assets/lib' + import { getNodeColorClasses } from '../../util' interface Props { data: AssetsOverflowedN['data'] @@ -33,6 +34,7 @@ wasOpenedBecauseOfExternalSelected = false } }) + const colors = $derived(getNodeColorClasses(undefined, includesSelected)) @@ -43,9 +45,11 @@ usePointerDownOutside bind:isOpen class={twMerge( - '!w-full text-2xs font-normal bg-surface h-6 pr-0.5 flex justify-center items-center rounded-sm text-primary border', - includesSelected ? 'bg-surface-secondary border-surface-inverse' : 'border-transparent', - 'hover:bg-surface-secondary hover:border-surface-inverse active:opacity-55' + '!w-full text-2xs font-normal h-6 pr-0.5 flex justify-center items-center rounded-md text-primary drop-shadow-base', + 'hover:bg-surface-hover active:bg-surface active:opacity-80', + colors.bg, + colors.outline, + colors.text )} placement="top" > diff --git a/frontend/src/lib/components/icons/AssetGenericIcon.svelte b/frontend/src/lib/components/icons/AssetGenericIcon.svelte index 2f0cb28a5cf80..669201d919e5e 100644 --- a/frontend/src/lib/components/icons/AssetGenericIcon.svelte +++ b/frontend/src/lib/components/icons/AssetGenericIcon.svelte @@ -1,29 +1,25 @@ {#if assetKind == 's3object'} - -{:else if assetKind == 'resource'} - + {:else if assetKind == 'datatable'} - + {:else if assetKind == 'ducklake'} - + +{:else if assetKind == 'resource'} + {:else} - + {/if} diff --git a/frontend/src/lib/components/icons/DucklakeIcon.svelte b/frontend/src/lib/components/icons/DucklakeIcon.svelte index 6bf68a339558c..488260ea1a3f6 100644 --- a/frontend/src/lib/components/icons/DucklakeIcon.svelte +++ b/frontend/src/lib/components/icons/DucklakeIcon.svelte @@ -1,14 +1,16 @@
    Ducklake
    - + Windmill has first class support for Ducklake. You can use and explore ducklakes like a normal SQL database, even though the data is actually stored in parquet files in S3 ! diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index ffe37a860f6b7..228e68d491d80 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -89,33 +89,43 @@ async function initWasmRuby() { await initRubyParser(wasmUrlRuby) } +type InferAssetsResult = + | { status: 'ok'; assets: AssetWithAccessType[] } + | { status: 'error'; error: string } + export async function inferAssets( language: SupportedLanguage | undefined, code: string -): Promise { +): Promise { + function wrap(raw_result: string): InferAssetsResult { + if (raw_result.startsWith('err:')) { + return { status: 'error', error: raw_result.slice(4).trim() } + } + return { status: 'ok', assets: JSON.parse(raw_result) as AssetWithAccessType[] } + } + try { if (language === 'duckdb') { await initWasmRegex() - let r = JSON.parse(parse_assets_sql(code)) - return r + return wrap(parse_assets_sql(code)) } if (language === 'deno' || language === 'nativets' || language === 'bun') { await initWasmTs() - return JSON.parse(parse_assets_ts(code)) + return wrap(parse_assets_ts(code)) } if (language === 'python3') { await initWasmPython() - return JSON.parse(parse_assets_py(code)) + return wrap(parse_assets_py(code)) } if (language === 'ansible') { await initWasmYaml() - return JSON.parse(parse_assets_ansible(code)) + return wrap(parse_assets_ansible(code)) } } catch (e) { - console.error('error parsing assets', e) - return [] + return { status: 'error', error: (e as Error)?.message || JSON.stringify(e) } } - return [] + + return { status: 'ok', assets: [] } } export async function inferAnsibleExecutionMode(code: string): Promise { diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 58bec208de794..f8fc95c8f382e 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -323,7 +323,7 @@ const DUCKDB_INIT_CODE = `-- result_collection=last_statement_all_rows -- SELECT * FROM db.public.friends; -- Click the +Ducklake button to use a ducklake --- https://www.windmill.dev/docs/core_concepts/ducklake +-- https://www.windmill.dev/docs/core_concepts/persistent_storage/ducklake -- -- ATTACH 'ducklake' AS dl; -- USE dl; diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/+page.svelte index 6f7aab03afd69..464da3cd04d2c 100644 --- a/frontend/src/routes/(root)/(logged)/assets/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/assets/+page.svelte @@ -76,12 +76,7 @@ - + {formatAssetKind(asset)}