diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83e958d98fd..bd623ca3ade 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,6 +226,7 @@ jobs: EXE_SUFFIX: "" TARGET_TRIPLE: x86_64-unknown-linux-gnu OPENSSL_CACHE_SUFFIX: "" + SMOKETEST_SUITE: standalone steps: &smoketest-build-steps - *find-git-ref @@ -256,7 +257,7 @@ jobs: - name: Build smoketest dependencies and archive test binaries shell: bash run: | - cargo ci smoketests archive --archive-file smoketest-nextest.tar.zst + cargo ci smoketests --suite "${SMOKETEST_SUITE}" archive --archive-file smoketest-nextest.tar.zst shopt -s nullglob precompiled_modules=(target/wasm32-unknown-unknown/release/smoketest_module_*.wasm) @@ -303,15 +304,16 @@ jobs: TARGET_TRIPLE: x86_64-pc-windows-msvc OPENSSL_CACHE_SUFFIX: -nasm OPENSSL_RUST_USE_NASM: "1" + SMOKETEST_SUITE: all steps: *smoketest-build-steps smoketest_partitions_linux: needs: [smoketest_build_linux] - name: Smoketests (Linux ${{ matrix.partition }}/2) + name: Smoketests (Linux ${{ matrix.partition }}/1) strategy: fail-fast: false matrix: - partition: [1, 2] + partition: [1] runs-on: spacetimedb-new-runner-2 timeout-minutes: 30 env: @@ -319,7 +321,7 @@ jobs: RUST_BACKTRACE: full SPACETIMEDB_CPP_DIR: ${{ github.workspace }}/crates/bindings-cpp ARTIFACT_SUFFIX: linux - PARTITION_COUNT: 2 + PARTITION_COUNT: 1 steps: &smoketest-partition-steps - *find-git-ref @@ -490,8 +492,7 @@ jobs: strategy: matrix: include: - - name: Linux 1/2 - - name: Linux 2/2 + - name: Linux 1/1 - name: Windows 1/4 - name: Windows 2/4 - name: Windows 3/4 @@ -1437,10 +1438,7 @@ jobs: permissions: contents: read env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target RUST_BACKTRACE: full - TARGET_TRIPLE: x86_64-unknown-linux-gnu - OPENSSL_CACHE_SUFFIX: "" steps: - name: Find Git ref env: @@ -1461,21 +1459,10 @@ jobs: - uses: dsherret/rust-toolchain-file@v1 - name: Set default rust toolchain run: rustup default $(rustup show active-toolchain | cut -d' ' -f1) - - *restore-rusty-v8-debug - - *restore-openssl - - *configure-cached-openssl - - name: Verify crates/smoketests/tests/smoketests/mod.rs lists all entries + - name: Verify smoketest module lists and suite constraints run: | - cargo ci smoketests check-mod-list - - - name: Upload timing reports - if: always() - uses: actions/upload-artifact@v4 - with: - name: cargo-timings-smoketests-mod-rs-complete - path: ${{ env.CARGO_TARGET_DIR }}/cargo-timings/ - retention-days: 14 + cargo run -p ci-smoketest-checks docs-build: needs: [merge_queue_noop] diff --git a/Cargo.lock b/Cargo.lock index 7e8ab82a95c..8b32f9444d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1003,6 +1003,10 @@ dependencies = [ "duct", ] +[[package]] +name = "ci-smoketest-checks" +version = "0.1.0" + [[package]] name = "ci-smoketests" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 1d84dc181e3..80cafd6f654 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ members = [ "tools/ci/commands/lint", "tools/ci/commands/wasm-bindings", "tools/ci/commands/smoketests", + "tools/ci/commands/smoketest-checks", "tools/ci/commands/keynote-bench", "tools/ci/commands/update-flow", "tools/ci/commands/cli-docs", diff --git a/crates/smoketests/src/lib.rs b/crates/smoketests/src/lib.rs index 5c90ae988d2..2450b8b9579 100644 --- a/crates/smoketests/src/lib.rs +++ b/crates/smoketests/src/lib.rs @@ -52,6 +52,7 @@ mod csharp; pub mod modules; +mod template_registry; use anyhow::{bail, Context, Result}; use regex::Regex; diff --git a/crates/smoketests/src/template_registry.rs b/crates/smoketests/src/template_registry.rs new file mode 100644 index 00000000000..63685373a8c --- /dev/null +++ b/crates/smoketests/src/template_registry.rs @@ -0,0 +1,32 @@ +#[doc(hidden)] +#[macro_export] +macro_rules! for_each_smoketest_template { + ($callback:ident) => { + $callback! { + test_template_angular_ts => "angular-ts", + test_template_astro_ts => "astro-ts", + test_template_basic_cpp => "basic-cpp", + test_template_basic_cs => "basic-cs", + test_template_basic_rs => "basic-rs", + test_template_basic_ts => "basic-ts", + test_template_browser_ts => "browser-ts", + test_template_bun_ts => "bun-ts", + test_template_chat_console_cs => "chat-console-cs", + test_template_chat_console_rs => "chat-console-rs", + test_template_chat_react_ts => "chat-react-ts", + test_template_deno_ts => "deno-ts", + test_template_hangman_react_ts => "hangman-react-ts", + test_template_llm_chat_ts => "llm-chat-ts", + test_template_money_exchange_react_ts => "money-exchange-react-ts", + test_template_nextjs_ts => "nextjs-ts", + test_template_nodejs_ts => "nodejs-ts", + test_template_nuxt_ts => "nuxt-ts", + test_template_react_ts => "react-ts", + test_template_remix_ts => "remix-ts", + test_template_solid_ts => "solid-ts", + test_template_svelte_ts => "svelte-ts", + test_template_tanstack_ts => "tanstack-ts", + test_template_vue_ts => "vue-ts", + } + }; +} diff --git a/crates/smoketests/tests/cluster.rs b/crates/smoketests/tests/cluster.rs new file mode 100644 index 00000000000..b4cb1865a82 --- /dev/null +++ b/crates/smoketests/tests/cluster.rs @@ -0,0 +1,41 @@ +// Cluster test binary entry point. These smoketests provide useful coverage +// against a cluster, but can also run against a local standalone server. +// +// We group the tests into one binary to avoid linking every source file as an +// independent integration test target. +mod cluster { + mod add_remove_index; + mod auto_inc; + mod auto_migration; + mod call; + mod cli; + mod client_connection_errors; + mod column_defaults; + mod confirmed_reads; + mod connect_disconnect_from_cli; + mod database_lock; + mod delete_database; + mod describe; + mod dml; + mod domains; + mod fail_initial_publish; + mod filtering; + mod http_egress; + mod http_routes; + mod logs_level_filter; + mod module_nested_op; + mod modules; + mod namespaces; + mod new_user_flow; + mod panic; + mod permissions; + mod publish_upgrade_prompt; + mod quickstart; + mod rls; + mod schedule_reducer; + mod sql; + mod sql_connect_hook; + mod templates; + mod timestamp_route; + mod views; +} diff --git a/crates/smoketests/tests/smoketests/add_remove_index.rs b/crates/smoketests/tests/cluster/add_remove_index.rs similarity index 100% rename from crates/smoketests/tests/smoketests/add_remove_index.rs rename to crates/smoketests/tests/cluster/add_remove_index.rs diff --git a/crates/smoketests/tests/smoketests/auto_inc.rs b/crates/smoketests/tests/cluster/auto_inc.rs similarity index 100% rename from crates/smoketests/tests/smoketests/auto_inc.rs rename to crates/smoketests/tests/cluster/auto_inc.rs diff --git a/crates/smoketests/tests/smoketests/auto_migration.rs b/crates/smoketests/tests/cluster/auto_migration.rs similarity index 77% rename from crates/smoketests/tests/smoketests/auto_migration.rs rename to crates/smoketests/tests/cluster/auto_migration.rs index fea22c94e63..679d6d11854 100644 --- a/crates/smoketests/tests/smoketests/auto_migration.rs +++ b/crates/smoketests/tests/cluster/auto_migration.rs @@ -1,4 +1,4 @@ -use spacetimedb_smoketests::{require_local_server, Smoketest}; +use spacetimedb_smoketests::Smoketest; /// Tests that a module with invalid schema changes cannot be published without -c or a migration. #[test] @@ -238,59 +238,3 @@ fn automigrate_reschema_event_table_arbitrarily() { .run() .expect("Changing schema of event table should succeed"); } - -/// Regression test: dropping an event table must not brick commitlog replay. -/// -/// Dropping an event table deletes its `st_table`, `st_column` and `st_event_table` rows -/// in a single transaction. Replay applies deletes in ascending table id order, -/// so the `st_table` row is already gone when the `st_column` deletes are replayed, -/// while the `st_event_table` row is still present. -/// Replay therefore treated the dropped table as a live event table -/// and tried to refresh its layout, failing with -/// `Table with ID ... not found in st_table` -/// and permanently preventing the database from starting. -#[test] -fn automigrate_drop_event_table_replays_after_restart() { - require_local_server!(); - let mut test = Smoketest::builder() - .precompiled_module("auto-migration-drop-event-table-before") - .build(); - - let identity = test - .database_identity - .clone() - .expect("database should be published after build"); - - // Write some history, including an event row. - test.call("add_person", &["Robert"]).unwrap(); - test.call("emit_event", &[]).unwrap(); - - // Drop the event table. - test.use_precompiled_module("auto-migration-drop-event-table-after"); - test.publish() - .name(&identity) - .break_clients(true) - .run() - .expect("Dropping the event table should succeed"); - - // Wait until data written after the drop is durable, - // which implies the drop itself is durable too. - test.call("add_person", &["Julie"]).unwrap(); - let output = test.sql_confirmed("SELECT * FROM person WHERE name = 'Julie'").unwrap(); - assert!(output.contains("Julie"), "Data not confirmed before restart: {output}"); - - // Restarting forces a commitlog replay, which must replay the event table drop. - test.restart_server(); - - let output = test.sql("SELECT name FROM person").unwrap(); - assert!(output.contains("Robert"), "Expected 'Robert' after restart: {output}"); - assert!(output.contains("Julie"), "Expected 'Julie' after restart: {output}"); - - // The database should still accept writes after replay. - test.call("add_person", &["Samantha"]).unwrap(); - let output = test.sql("SELECT name FROM person WHERE name = 'Samantha'").unwrap(); - assert!( - output.contains("Samantha"), - "Expected 'Samantha' after restart: {output}" - ); -} diff --git a/crates/smoketests/tests/smoketests/call.rs b/crates/smoketests/tests/cluster/call.rs similarity index 100% rename from crates/smoketests/tests/smoketests/call.rs rename to crates/smoketests/tests/cluster/call.rs diff --git a/crates/smoketests/tests/cluster/cli/mod.rs b/crates/smoketests/tests/cluster/cli/mod.rs new file mode 100644 index 00000000000..32a73205ae4 --- /dev/null +++ b/crates/smoketests/tests/cluster/cli/mod.rs @@ -0,0 +1,2 @@ +pub mod publish; +pub mod server; diff --git a/crates/smoketests/tests/smoketests/cli/publish.rs b/crates/smoketests/tests/cluster/cli/publish.rs similarity index 97% rename from crates/smoketests/tests/smoketests/cli/publish.rs rename to crates/smoketests/tests/cluster/cli/publish.rs index 7fe5fab5fc0..599a4669d8a 100644 --- a/crates/smoketests/tests/smoketests/cli/publish.rs +++ b/crates/smoketests/tests/cluster/cli/publish.rs @@ -1,6 +1,6 @@ //! CLI publish command tests -use spacetimedb_smoketests::{patch_module_cargo_to_local_bindings, require_local_server, Smoketest}; +use spacetimedb_smoketests::{patch_module_cargo_to_local_bindings, random_string, Smoketest}; #[test] fn cli_can_publish_spacetimedb_on_disk() { @@ -47,11 +47,10 @@ fn cli_can_publish_spacetimedb_on_disk() { // e.g. when providing --delete-data, or when there's a conflict and --delete-data=on-conflict is provided. fn migration_test(module_name: &str, republish_args: &[&str], expect_success: bool) { - // This only requires a local server because the module names are static - require_local_server!(); - let test = Smoketest::builder().autopublish(false).build(); + let module_name = format!("{module_name}-{}", random_string()); + let workspace_dir = cargo_metadata::MetadataCommand::new().exec().unwrap().workspace_root; let dir = workspace_dir.join("modules").join("module-test"); @@ -63,7 +62,8 @@ fn migration_test(module_name: &str, republish_args: &[&str], expect_success: bo &dir, "--server", &test.server_url, - module_name, + "--yes=remote", + &module_name, ]) .unwrap(); @@ -74,7 +74,8 @@ fn migration_test(module_name: &str, republish_args: &[&str], expect_success: bo &dir, "--server", &test.server_url, - module_name, + "--yes=remote", + &module_name, ]; args.extend(republish_args); let output = test.spacetime_cmd(&args); diff --git a/crates/smoketests/tests/cluster/cli/server.rs b/crates/smoketests/tests/cluster/cli/server.rs new file mode 100644 index 00000000000..ebd979c982a --- /dev/null +++ b/crates/smoketests/tests/cluster/cli/server.rs @@ -0,0 +1,23 @@ +//! CLI server command tests + +use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_smoketests::Smoketest; +use std::process::Command; + +fn cli_cmd() -> Command { + Command::new(ensure_binaries_built()) +} + +#[test] +fn cli_can_ping_spacetimedb_server() { + let spacetime = Smoketest::builder().autopublish(false).build(); + let output = cli_cmd() + .args(["server", "ping", &spacetime.server_url]) + .output() + .expect("failed to execute"); + assert!( + output.status.success(), + "ping failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/smoketests/tests/cluster/client_connection_errors.rs b/crates/smoketests/tests/cluster/client_connection_errors.rs new file mode 100644 index 00000000000..c937d750ce4 --- /dev/null +++ b/crates/smoketests/tests/cluster/client_connection_errors.rs @@ -0,0 +1,59 @@ +use spacetimedb_smoketests::Smoketest; + +/// Test that client_connected returning an error rejects the connection +#[test] +fn test_client_connected_error_rejects_connection() { + let test = Smoketest::builder() + .precompiled_module("client-connection-reject") + .build(); + + // Subscribe should fail because client_connected returns an error + let result = test.subscribe(&["SELECT * FROM all_u8s"]).expect_rows(0).run(); + assert!( + result.is_err(), + "Expected subscribe to fail when client_connected returns error" + ); + + let logs = test.logs(100).unwrap(); + assert!( + logs.iter().any(|l| l.contains("Rejecting connection from client")), + "Expected rejection message in logs: {:?}", + logs + ); + assert!( + !logs.iter().any(|l| l.contains("This should never be called")), + "client_disconnected should not have been called: {:?}", + logs + ); +} + +/// Test that client_disconnected panicking still cleans up the st_client row +#[test] +fn test_client_disconnected_error_still_deletes_st_client() { + let test = Smoketest::builder() + .precompiled_module("client-connection-disconnect-panic") + .build(); + + // Subscribe should succeed (client_connected returns Ok) + let result = test.subscribe(&["SELECT * FROM all_u8s"]).expect_rows(0).run(); + assert!(result.is_ok(), "Expected subscribe to succeed"); + + let logs = test.logs(100).unwrap(); + assert!( + logs.iter() + .any(|l| { l.contains("This should be called, but the `st_client` row should still be deleted") }), + "Expected disconnect panic message in logs: {:?}", + logs + ); + + // Verify the websocket's st_client row was deleted despite the panic. + // The SQL query itself creates a temporary connection, so we may see + // exactly one row (the SQL connection's own), but the websocket's row + // should be gone. + let sql_out = test.sql("SELECT * FROM st_client").unwrap(); + let row_count = sql_out.lines().filter(|l| l.contains("0x")).count(); + assert!( + row_count <= 1, + "Expected at most 1 st_client row (the SQL connection itself), got {row_count}: {sql_out}", + ); +} diff --git a/crates/smoketests/tests/smoketests/column_defaults.rs b/crates/smoketests/tests/cluster/column_defaults.rs similarity index 100% rename from crates/smoketests/tests/smoketests/column_defaults.rs rename to crates/smoketests/tests/cluster/column_defaults.rs diff --git a/crates/smoketests/tests/smoketests/confirmed_reads.rs b/crates/smoketests/tests/cluster/confirmed_reads.rs similarity index 100% rename from crates/smoketests/tests/smoketests/confirmed_reads.rs rename to crates/smoketests/tests/cluster/confirmed_reads.rs diff --git a/crates/smoketests/tests/smoketests/connect_disconnect_from_cli.rs b/crates/smoketests/tests/cluster/connect_disconnect_from_cli.rs similarity index 100% rename from crates/smoketests/tests/smoketests/connect_disconnect_from_cli.rs rename to crates/smoketests/tests/cluster/connect_disconnect_from_cli.rs diff --git a/crates/smoketests/tests/smoketests/database_lock.rs b/crates/smoketests/tests/cluster/database_lock.rs similarity index 100% rename from crates/smoketests/tests/smoketests/database_lock.rs rename to crates/smoketests/tests/cluster/database_lock.rs diff --git a/crates/smoketests/tests/smoketests/delete_database.rs b/crates/smoketests/tests/cluster/delete_database.rs similarity index 100% rename from crates/smoketests/tests/smoketests/delete_database.rs rename to crates/smoketests/tests/cluster/delete_database.rs diff --git a/crates/smoketests/tests/smoketests/describe.rs b/crates/smoketests/tests/cluster/describe.rs similarity index 100% rename from crates/smoketests/tests/smoketests/describe.rs rename to crates/smoketests/tests/cluster/describe.rs diff --git a/crates/smoketests/tests/smoketests/dml.rs b/crates/smoketests/tests/cluster/dml.rs similarity index 100% rename from crates/smoketests/tests/smoketests/dml.rs rename to crates/smoketests/tests/cluster/dml.rs diff --git a/crates/smoketests/tests/smoketests/domains.rs b/crates/smoketests/tests/cluster/domains.rs similarity index 100% rename from crates/smoketests/tests/smoketests/domains.rs rename to crates/smoketests/tests/cluster/domains.rs diff --git a/crates/smoketests/tests/smoketests/fail_initial_publish.rs b/crates/smoketests/tests/cluster/fail_initial_publish.rs similarity index 100% rename from crates/smoketests/tests/smoketests/fail_initial_publish.rs rename to crates/smoketests/tests/cluster/fail_initial_publish.rs diff --git a/crates/smoketests/tests/smoketests/filtering.rs b/crates/smoketests/tests/cluster/filtering.rs similarity index 100% rename from crates/smoketests/tests/smoketests/filtering.rs rename to crates/smoketests/tests/cluster/filtering.rs diff --git a/crates/smoketests/tests/cluster/http_egress.rs b/crates/smoketests/tests/cluster/http_egress.rs new file mode 100644 index 00000000000..e326830f753 --- /dev/null +++ b/crates/smoketests/tests/cluster/http_egress.rs @@ -0,0 +1,16 @@ +use spacetimedb_smoketests::Smoketest; + +#[test] +fn test_http_disallowed_ip_is_blocked() { + let test = Smoketest::builder().precompiled_module("http-egress").build(); + + let output = test.call_output("request_disallowed_ip", &["http://10.0.0.1:80/"]); + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "Expected request_disallowed_ip to succeed after observing blocked egress error.\nstdout:\n{}\nstderr:\n{}", + stdout, + stderr + ); +} diff --git a/crates/smoketests/tests/smoketests/http_routes.rs b/crates/smoketests/tests/cluster/http_routes.rs similarity index 100% rename from crates/smoketests/tests/smoketests/http_routes.rs rename to crates/smoketests/tests/cluster/http_routes.rs diff --git a/crates/smoketests/tests/smoketests/logs_level_filter.rs b/crates/smoketests/tests/cluster/logs_level_filter.rs similarity index 100% rename from crates/smoketests/tests/smoketests/logs_level_filter.rs rename to crates/smoketests/tests/cluster/logs_level_filter.rs diff --git a/crates/smoketests/tests/smoketests/module_nested_op.rs b/crates/smoketests/tests/cluster/module_nested_op.rs similarity index 100% rename from crates/smoketests/tests/smoketests/module_nested_op.rs rename to crates/smoketests/tests/cluster/module_nested_op.rs diff --git a/crates/smoketests/tests/smoketests/modules.rs b/crates/smoketests/tests/cluster/modules.rs similarity index 100% rename from crates/smoketests/tests/smoketests/modules.rs rename to crates/smoketests/tests/cluster/modules.rs diff --git a/crates/smoketests/tests/smoketests/namespaces.rs b/crates/smoketests/tests/cluster/namespaces.rs similarity index 100% rename from crates/smoketests/tests/smoketests/namespaces.rs rename to crates/smoketests/tests/cluster/namespaces.rs diff --git a/crates/smoketests/tests/smoketests/new_user_flow.rs b/crates/smoketests/tests/cluster/new_user_flow.rs similarity index 100% rename from crates/smoketests/tests/smoketests/new_user_flow.rs rename to crates/smoketests/tests/cluster/new_user_flow.rs diff --git a/crates/smoketests/tests/smoketests/panic.rs b/crates/smoketests/tests/cluster/panic.rs similarity index 100% rename from crates/smoketests/tests/smoketests/panic.rs rename to crates/smoketests/tests/cluster/panic.rs diff --git a/crates/smoketests/tests/smoketests/permissions.rs b/crates/smoketests/tests/cluster/permissions.rs similarity index 100% rename from crates/smoketests/tests/smoketests/permissions.rs rename to crates/smoketests/tests/cluster/permissions.rs diff --git a/crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs b/crates/smoketests/tests/cluster/publish_upgrade_prompt.rs similarity index 100% rename from crates/smoketests/tests/smoketests/publish_upgrade_prompt.rs rename to crates/smoketests/tests/cluster/publish_upgrade_prompt.rs diff --git a/crates/smoketests/tests/smoketests/quickstart.rs b/crates/smoketests/tests/cluster/quickstart.rs similarity index 100% rename from crates/smoketests/tests/smoketests/quickstart.rs rename to crates/smoketests/tests/cluster/quickstart.rs diff --git a/crates/smoketests/tests/smoketests/rls.rs b/crates/smoketests/tests/cluster/rls.rs similarity index 100% rename from crates/smoketests/tests/smoketests/rls.rs rename to crates/smoketests/tests/cluster/rls.rs diff --git a/crates/smoketests/tests/smoketests/schedule_reducer.rs b/crates/smoketests/tests/cluster/schedule_reducer.rs similarity index 100% rename from crates/smoketests/tests/smoketests/schedule_reducer.rs rename to crates/smoketests/tests/cluster/schedule_reducer.rs diff --git a/crates/smoketests/tests/smoketests/sql.rs b/crates/smoketests/tests/cluster/sql.rs similarity index 100% rename from crates/smoketests/tests/smoketests/sql.rs rename to crates/smoketests/tests/cluster/sql.rs diff --git a/crates/smoketests/tests/smoketests/sql_connect_hook.rs b/crates/smoketests/tests/cluster/sql_connect_hook.rs similarity index 100% rename from crates/smoketests/tests/smoketests/sql_connect_hook.rs rename to crates/smoketests/tests/cluster/sql_connect_hook.rs diff --git a/crates/smoketests/tests/smoketests/templates.rs b/crates/smoketests/tests/cluster/templates.rs similarity index 73% rename from crates/smoketests/tests/smoketests/templates.rs rename to crates/smoketests/tests/cluster/templates.rs index 5623e3baab4..6dbb10bf789 100644 --- a/crates/smoketests/tests/smoketests/templates.rs +++ b/crates/smoketests/tests/cluster/templates.rs @@ -12,7 +12,6 @@ use anyhow::{bail, Context, Result}; use regex::Regex; use serde_json::Value; -use spacetimedb_guard::ensure_binaries_built; use spacetimedb_smoketests::{pnpm, random_string, workspace_root, Smoketest}; use std::env; use std::fs; @@ -132,132 +131,6 @@ fn init_template_with_dotnet_version( Ok((tmpdir, project_path)) } -fn fake_dotnet_path(dir: &Path, sdk_list_output: &str) -> Result { - let executable_name = if cfg!(windows) { "dotnet.exe" } else { "dotnet" }; - let dotnet_path = dir.join(executable_name); - let echo_lines = sdk_list_output - .lines() - .map(|line| format!("echo {line}")) - .collect::>() - .join(if cfg!(windows) { "\r\n" } else { "\n" }); - - if cfg!(windows) { - let source_path = dir.join("fake_dotnet.rs"); - fs::write( - &source_path, - format!( - r#"fn main() {{ - if std::env::args().nth(1).as_deref() == Some("--list-sdks") {{ - print!("{{}}", {sdk_list_output:?}); - return; - }} - - std::process::exit(1); -}} -"# - ), - ) - .with_context(|| format!("Failed to write fake dotnet source {:?}", source_path))?; - - let output = Command::new("rustc") - .arg(&source_path) - .arg("-o") - .arg(&dotnet_path) - .output() - .context("Failed to spawn rustc for fake dotnet")?; - if !output.status.success() { - bail!( - "rustc failed to compile fake dotnet:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - } else { - fs::write( - &dotnet_path, - format!("#!/usr/bin/env sh\nif [ \"$1\" = \"--list-sdks\" ]; then\n{echo_lines}\nexit 0\nfi\nexit 1\n"), - ) - .with_context(|| format!("Failed to write fake dotnet executable {:?}", dotnet_path))?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut permissions = fs::metadata(&dotnet_path)?.permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&dotnet_path, permissions)?; - } - } - - Ok(dotnet_path) -} - -fn init_basic_cs_with_fake_dotnet(sdk_list_output: &str) -> Result<(TempDir, PathBuf)> { - let tmpdir = tempfile::tempdir().context("Failed to create temp dir")?; - let fake_bin = tmpdir.path().join("bin"); - fs::create_dir(&fake_bin).context("Failed to create fake dotnet bin dir")?; - fake_dotnet_path(&fake_bin, sdk_list_output)?; - - let current_path = env::var_os("PATH").unwrap_or_default(); - let test_path = env::join_paths(std::iter::once(fake_bin).chain(env::split_paths(¤t_path))) - .context("Failed to build test PATH")?; - - let project_name = "test-basic-cs-default-dotnet"; - let project_path = tmpdir.path().join(project_name); - let config_path = tmpdir.path().join("config.toml"); - let output = Command::new(ensure_binaries_built()) - .arg("--config-path") - .arg(&config_path) - .args([ - "init", - "--template", - "basic-cs", - "--project-path", - project_path.to_str().unwrap(), - "--non-interactive", - project_name, - ]) - .env("PATH", test_path) - .current_dir(tmpdir.path()) - .output() - .context("Failed to execute spacetime init")?; - - if !output.status.success() { - bail!( - "spacetime init with fake dotnet failed:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - - Ok((tmpdir, project_path)) -} - -fn assert_basic_cs_default_dotnet(sdk_list_output: &str, expected_major: u8) -> Result<()> { - let (_tmpdir, project_path) = init_basic_cs_with_fake_dotnet(sdk_list_output)?; - let server_path = project_path.join("spacetimedb"); - - let global_json = fs::read_to_string(server_path.join("global.json")).context("Failed to read global.json")?; - assert!( - global_json.contains(&format!("\"version\": \"{expected_major}.0.100\"")), - "global.json did not target .NET {expected_major}:\n{global_json}" - ); - - let csproj_path = find_csproj(&server_path)?; - let csproj = fs::read_to_string(&csproj_path).with_context(|| format!("Failed to read {:?}", csproj_path))?; - assert!( - csproj.contains(&format!("net{expected_major}.0")), - "{:?} did not target net{expected_major}.0:\n{csproj}", - csproj_path - ); - assert!( - !csproj.contains(""), - "{:?} should use a single TargetFramework after init:\n{csproj}", - csproj_path - ); - - Ok(()) -} - /// Updates a `[dependencies]` entry in a `Cargo.toml` to use a local path. fn update_cargo_toml_dependency(cargo_toml_path: &Path, package_name: &str, local_path: &Path) -> Result<()> { if !cargo_toml_path.exists() { @@ -314,97 +187,6 @@ fn update_package_json_dependency(package_json_path: &Path, package_name: &str, Ok(()) } -fn assert_major_minor_version(actual: &str, context: impl std::fmt::Display) -> Result<()> { - let re = Regex::new(r"^\d+\.\d+$").unwrap(); - if !re.is_match(actual) { - bail!("{context}: expected MAJOR.MINOR, got {actual}"); - } - Ok(()) -} - -fn assert_major_minor_patch_wildcard(actual: &str, context: impl std::fmt::Display) -> Result<()> { - let re = Regex::new(r"^\d+\.\d+\.\*$").unwrap(); - if !re.is_match(actual) { - bail!("{context}: expected MAJOR.MINOR.*, got {actual}"); - } - Ok(()) -} - -fn read_cargo_dependency_version(cargo_toml_path: &Path, package_name: &str) -> Result { - let content = - fs::read_to_string(cargo_toml_path).with_context(|| format!("Failed to read {:?}", cargo_toml_path))?; - let data: toml::Value = content - .parse() - .with_context(|| format!("Failed to parse {:?}", cargo_toml_path))?; - let dep = data - .get("dependencies") - .and_then(|deps| deps.get(package_name)) - .with_context(|| format!("No dependency `{package_name}` found in {:?}", cargo_toml_path))?; - match dep { - toml::Value::String(version) => Ok(version.clone()), - toml::Value::Table(table) => table - .get("version") - .and_then(|v| v.as_str()) - .map(String::from) - .with_context(|| format!("Dependency `{package_name}` in {:?} has no version", cargo_toml_path)), - _ => bail!( - "Unsupported dependency `{package_name}` format in {:?}", - cargo_toml_path - ), - } -} - -fn read_package_json_dependency_version(package_json_path: &Path, package_name: &str) -> Result { - let content = - fs::read_to_string(package_json_path).with_context(|| format!("Failed to read {:?}", package_json_path))?; - let data: Value = - serde_json::from_str(&content).with_context(|| format!("Failed to parse {:?}", package_json_path))?; - data.get("dependencies") - .and_then(|deps| deps.get(package_name)) - .and_then(|v| v.as_str()) - .map(String::from) - .with_context(|| format!("No dependency `{package_name}` found in {:?}", package_json_path)) -} - -fn read_csproj_package_reference_version(csproj_path: &Path, package_name: &str) -> Result { - let content = fs::read_to_string(csproj_path).with_context(|| format!("Failed to read {:?}", csproj_path))?; - let root = xmltree::Element::parse(content.as_bytes()) - .with_context(|| format!("Failed to parse XML {:?}", csproj_path))?; - - root.children - .iter() - .filter_map(|node| match node { - xmltree::XMLNode::Element(element) if element.name == "ItemGroup" => Some(element), - _ => None, - }) - .flat_map(|item_group| item_group.children.iter()) - .filter_map(|node| match node { - xmltree::XMLNode::Element(element) if element.name == "PackageReference" => Some(element), - _ => None, - }) - .find(|package_ref| package_ref.attributes.get("Include").map(String::as_str) == Some(package_name)) - .and_then(|package_ref| package_ref.attributes.get("Version").cloned()) - .with_context(|| format!("No PackageReference `{package_name}` found in {:?}", csproj_path)) -} - -fn find_csproj(dir: &Path) -> Result { - fs::read_dir(dir) - .with_context(|| format!("Failed to read {:?}", dir))? - .flatten() - .map(|entry| entry.path()) - .find(|path| path.extension().is_some_and(|ext| ext == "csproj")) - .with_context(|| format!("No .csproj found in {:?}", dir)) -} - -fn read_spacetimedb_cpp_version(cmake_path: &Path) -> Result { - let content = fs::read_to_string(cmake_path).with_context(|| format!("Failed to read {:?}", cmake_path))?; - let re = Regex::new(r#"set\(SPACETIMEDB_CPP_VERSION\s+"([^"]+)""#).unwrap(); - let caps = re - .captures(&content) - .with_context(|| format!("No SPACETIMEDB_CPP_VERSION found in {:?}", cmake_path))?; - Ok(caps.get(1).unwrap().as_str().to_string()) -} - /// Runs pnpm with the given arguments in the given working directory. fn run_pnpm(args: &[&str], cwd: &Path) -> Result<()> { pnpm(args, cwd)?; @@ -1130,54 +912,6 @@ fn test_template(test: &Smoketest, template: &Template) -> Result<()> { // Test entry point // ============================================================================ -#[test] -fn test_basic_template_dependency_versions() -> Result<()> { - let test = Smoketest::builder().autopublish(false).build(); - - let (_basic_cpp_tmpdir, basic_cpp_path) = init_template(&test, "basic-cpp")?; - let cpp_server_version = read_spacetimedb_cpp_version(&basic_cpp_path.join("spacetimedb").join("CMakeLists.txt"))?; - assert_major_minor_version(&cpp_server_version, "basic-cpp C++ server SPACETIMEDB_CPP_VERSION")?; - // The current basic C++ template still uses a Rust client; we do not have a C++ client yet. - let cpp_client_manifest = basic_cpp_path.join("Cargo.toml"); - if !cpp_client_manifest.exists() { - bail!("basic-cpp expected Rust client manifest at {:?}", cpp_client_manifest); - } - - let (_basic_rs_tmpdir, basic_rs_path) = init_template(&test, "basic-rs")?; - let rs_server_version = - read_cargo_dependency_version(&basic_rs_path.join("spacetimedb").join("Cargo.toml"), "spacetimedb")?; - assert_major_minor_patch_wildcard(&rs_server_version, "basic-rs Rust server spacetimedb")?; - let rs_client_version = read_cargo_dependency_version(&basic_rs_path.join("Cargo.toml"), "spacetimedb-sdk")?; - assert_major_minor_patch_wildcard(&rs_client_version, "basic-rs Rust client spacetimedb-sdk")?; - - let (_basic_ts_tmpdir, basic_ts_path) = init_template(&test, "basic-ts")?; - let ts_server_version = - read_package_json_dependency_version(&basic_ts_path.join("spacetimedb").join("package.json"), "spacetimedb")?; - assert_major_minor_patch_wildcard(&ts_server_version, "basic-ts TypeScript server spacetimedb")?; - let ts_client_version = read_package_json_dependency_version(&basic_ts_path.join("package.json"), "spacetimedb")?; - assert_major_minor_patch_wildcard(&ts_client_version, "basic-ts TypeScript client spacetimedb")?; - - let (_basic_cs_tmpdir, basic_cs_path) = init_template(&test, "basic-cs")?; - let cs_server_project = find_csproj(&basic_cs_path.join("spacetimedb"))?; - let cs_server_version = read_csproj_package_reference_version(&cs_server_project, "SpacetimeDB.Runtime")?; - assert_major_minor_patch_wildcard(&cs_server_version, "basic-cs C# server SpacetimeDB.Runtime")?; - let cs_client_version = - read_csproj_package_reference_version(&basic_cs_path.join("client.csproj"), "SpacetimeDB.ClientSDK")?; - assert_major_minor_patch_wildcard(&cs_client_version, "basic-cs C# client SpacetimeDB.ClientSDK")?; - - Ok(()) -} - -#[test] -fn test_basic_cs_init_default_dotnet_selection() -> Result<()> { - assert_basic_cs_default_dotnet("8.0.416 [/usr/share/dotnet/sdk]", 8)?; - assert_basic_cs_default_dotnet("10.0.100 [/usr/share/dotnet/sdk]", 10)?; - assert_basic_cs_default_dotnet("8.0.416 [/usr/share/dotnet/sdk]\n10.0.100 [/usr/share/dotnet/sdk]", 10)?; - assert_basic_cs_default_dotnet("", 10)?; - - Ok(()) -} - /// Runs the init + publish + client-test cycle for one registered template. /// /// Each template is a separate Rust test so nextest can hash it independently @@ -1205,8 +939,6 @@ fn run_registered_template(template_id: &str) { macro_rules! template_tests { ($($test_name:ident => $template_id:literal),+ $(,)?) => { - const REGISTERED_TEMPLATE_IDS: &[&str] = &[$($template_id),+]; - $( #[test] fn $test_name() { @@ -1216,46 +948,4 @@ macro_rules! template_tests { }; } -template_tests! { - test_template_angular_ts => "angular-ts", - test_template_astro_ts => "astro-ts", - test_template_basic_cpp => "basic-cpp", - test_template_basic_cs => "basic-cs", - test_template_basic_rs => "basic-rs", - test_template_basic_ts => "basic-ts", - test_template_browser_ts => "browser-ts", - test_template_bun_ts => "bun-ts", - test_template_chat_console_cs => "chat-console-cs", - test_template_chat_console_rs => "chat-console-rs", - test_template_chat_react_ts => "chat-react-ts", - test_template_deno_ts => "deno-ts", - test_template_hangman_react_ts => "hangman-react-ts", - test_template_llm_chat_ts => "llm-chat-ts", - test_template_money_exchange_react_ts => "money-exchange-react-ts", - test_template_nextjs_ts => "nextjs-ts", - test_template_nodejs_ts => "nodejs-ts", - test_template_nuxt_ts => "nuxt-ts", - test_template_react_ts => "react-ts", - test_template_remix_ts => "remix-ts", - test_template_solid_ts => "solid-ts", - test_template_svelte_ts => "svelte-ts", - test_template_tanstack_ts => "tanstack-ts", - test_template_vue_ts => "vue-ts", -} - -#[test] -fn test_template_registry_matches_discovered_templates() { - let discovered = get_templates() - .into_iter() - .map(|template| template.id) - .collect::>(); - let registered = REGISTERED_TEMPLATE_IDS - .iter() - .map(|template_id| (*template_id).to_owned()) - .collect::>(); - - assert_eq!( - discovered, registered, - "Every discovered template must have its own nextest-visible test" - ); -} +spacetimedb_smoketests::for_each_smoketest_template!(template_tests); diff --git a/crates/smoketests/tests/smoketests/timestamp_route.rs b/crates/smoketests/tests/cluster/timestamp_route.rs similarity index 100% rename from crates/smoketests/tests/smoketests/timestamp_route.rs rename to crates/smoketests/tests/cluster/timestamp_route.rs diff --git a/crates/smoketests/tests/smoketests/views.rs b/crates/smoketests/tests/cluster/views.rs similarity index 92% rename from crates/smoketests/tests/smoketests/views.rs rename to crates/smoketests/tests/cluster/views.rs index d4c4b1cf394..c592cce3070 100644 --- a/crates/smoketests/tests/smoketests/views.rs +++ b/crates/smoketests/tests/cluster/views.rs @@ -1,12 +1,5 @@ -use std::path::PathBuf; - use serde_json::{json, Value}; -use spacetimedb_smoketests::{ - random_string, require_dotnet, require_local_server, require_pnpm, workspace_root, ModuleLanguage, Smoketest, -}; - -const STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY: &str = - "c200f6ec405075e508c2ed6474019332d6a2a46c69614306cc4bd980e0b8b767"; +use spacetimedb_smoketests::{random_string, require_dotnet, require_pnpm, ModuleLanguage, Smoketest}; const TS_VIEWS_SUBSCRIBE_MODULE: &str = r#"import { schema, t, table } from "spacetimedb/server"; @@ -225,27 +218,6 @@ fn project_fields(events: Vec, view_name: &str, projected_fields: &[&str] .collect() } -fn stale_view_backing_table_fixture() -> PathBuf { - workspace_root() - .join("crates") - .join("smoketests") - .join("fixtures") - .join("stale-view-backing-table-v2.6.0") -} - -fn stale_view_backing_table_test() -> Smoketest { - let test = Smoketest::builder() - .data_dir_fixture( - stale_view_backing_table_fixture(), - STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY, - ) - .autopublish(false) - .build(); - - test.new_identity().unwrap(); - test -} - fn assert_count_view_refresh_behavior(test: &Smoketest, view_name: &str, id: &str, value: &str, updated_value: &str) { let query = format!("select * from {view_name}"); let sub = test.subscribe(&[&query]).expect_rows(2).background().unwrap(); @@ -648,78 +620,6 @@ fn test_view_primary_key_auto_migration_disconnects_clients() { ); } -#[test] -fn test_repair_stale_sender_scoped_view_backing_table_on_startup() { - require_local_server!(); - - let test = stale_view_backing_table_test(); - - let sender_view_sub = test - .subscribe(&["select * from player"]) - .expect_rows(2) - .background() - .unwrap(); - - test.call("set_player_state", &["42", "1"]).unwrap(); - test.call("set_player_state", &["42", "2"]).unwrap(); - - let sender_view_events = sender_view_sub.collect().unwrap(); - let sender_view_projection = project_fields(sender_view_events, "player", &["id", "level"]); - assert_eq!( - serde_json::json!(sender_view_projection), - json!([ - { - "player": { - "deletes": [], - "inserts": [{ "id": 42, "level": 1 }] - } - }, - { - "player": { - "deletes": [{ "id": 42, "level": 1 }], - "inserts": [{ "id": 42, "level": 2 }] - } - } - ]) - ); -} - -#[test] -fn test_repair_stale_anonymous_view_backing_table_on_startup() { - require_local_server!(); - - let test = stale_view_backing_table_test(); - - let anonymous_view_sub = test - .subscribe(&["select * from player_and_level"]) - .expect_rows(2) - .background() - .unwrap(); - - test.call("add_player_level", &["1", "2"]).unwrap(); - test.call("add_player_level", &["2", "2"]).unwrap(); - - let anonymous_view_events = anonymous_view_sub.collect().unwrap(); - let anonymous_view_projection = project_fields(anonymous_view_events, "player_and_level", &["id", "level"]); - assert_eq!( - serde_json::json!(anonymous_view_projection), - json!([ - { - "player_and_level": { - "deletes": [], - "inserts": [{ "id": 1, "level": 2 }] - } - }, - { - "player_and_level": { - "deletes": [], - "inserts": [{ "id": 2, "level": 2 }] - } - } - ]) - ); -} - #[test] fn test_view_accessibility() { let test = Smoketest::builder().precompiled_module("views-callable").build(); diff --git a/crates/smoketests/tests/integration.rs b/crates/smoketests/tests/integration.rs deleted file mode 100644 index 298954754fa..00000000000 --- a/crates/smoketests/tests/integration.rs +++ /dev/null @@ -1,6 +0,0 @@ -// Single test binary entry point - includes all smoketests -// We put the tests in a single submodule because if they are at the toplevel then -// they all build and link independently, which takes a lot of linker time. -// This has the unfortunate side effect of requiring that they are all listed in a mod.rs, -// but what can you do ¯\_(ツ)_/¯. -mod smoketests; diff --git a/crates/smoketests/tests/smoketests/cli/mod.rs b/crates/smoketests/tests/smoketests/cli/mod.rs deleted file mode 100644 index 60ece75d377..00000000000 --- a/crates/smoketests/tests/smoketests/cli/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod auth; -pub mod dev; -pub mod generate; -pub mod list; -pub mod publish; -pub mod server; diff --git a/crates/smoketests/tests/smoketests/mod.rs b/crates/smoketests/tests/smoketests/mod.rs deleted file mode 100644 index 5831b4813fa..00000000000 --- a/crates/smoketests/tests/smoketests/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -// All smoketest modules -mod add_remove_index; -mod auto_inc; -mod auto_migration; -mod call; -mod change_host_type; -mod cli; -mod client_connection_errors; -mod column_defaults; -mod confirmed_reads; -mod connect_disconnect_from_cli; -mod create_project; -mod csharp_aot_module; -mod csharp_module; -mod database_lock; -mod default_module_clippy; -mod delete_database; -mod describe; -mod detect_wasm_bindgen; -mod dml; -mod domains; -mod fail_initial_publish; -mod filtering; -mod http_egress; -mod http_routes; -mod logs_level_filter; -mod module_nested_op; -mod modules; -mod namespaces; -mod new_user_flow; -mod panic; -mod permissions; -mod pg_wire; -mod publish_upgrade_prompt; -mod quickstart; -mod restart; -mod rls; -mod schedule_reducer; -mod servers; -mod sql; -mod sql_connect_hook; -mod templates; -mod timestamp_route; -mod typescript_index_source_name; -mod views; diff --git a/crates/smoketests/tests/standalone.rs b/crates/smoketests/tests/standalone.rs new file mode 100644 index 00000000000..8c7ff66f80c --- /dev/null +++ b/crates/smoketests/tests/standalone.rs @@ -0,0 +1,24 @@ +// Standalone test binary entry point. +// +// These smoketests are assigned to standalone coverage. Some require control +// of a local SpacetimeDB server; others simply provide no additional value when +// repeated against a cluster. Tests that require local server control keep +// `require_local_server!()` as a defensive check. +mod standalone { + mod auto_migration; + mod change_host_type; + mod cli; + mod client_connection_errors; + mod create_project; + mod csharp_aot_module; + mod csharp_module; + mod default_module_clippy; + mod detect_wasm_bindgen; + mod http_egress; + mod pg_wire; + mod restart; + mod servers; + mod templates; + mod typescript_index_source_name; + mod views; +} diff --git a/crates/smoketests/tests/standalone/auto_migration.rs b/crates/smoketests/tests/standalone/auto_migration.rs new file mode 100644 index 00000000000..f00a49e75bf --- /dev/null +++ b/crates/smoketests/tests/standalone/auto_migration.rs @@ -0,0 +1,57 @@ +use spacetimedb_smoketests::{require_local_server, Smoketest}; + +/// Regression test: dropping an event table must not brick commitlog replay. +/// +/// Dropping an event table deletes its `st_table`, `st_column` and `st_event_table` rows +/// in a single transaction. Replay applies deletes in ascending table id order, +/// so the `st_table` row is already gone when the `st_column` deletes are replayed, +/// while the `st_event_table` row is still present. +/// Replay therefore treated the dropped table as a live event table +/// and tried to refresh its layout, failing with +/// `Table with ID ... not found in st_table` +/// and permanently preventing the database from starting. +#[test] +fn automigrate_drop_event_table_replays_after_restart() { + require_local_server!(); + let mut test = Smoketest::builder() + .precompiled_module("auto-migration-drop-event-table-before") + .build(); + + let identity = test + .database_identity + .clone() + .expect("database should be published after build"); + + // Write some history, including an event row. + test.call("add_person", &["Robert"]).unwrap(); + test.call("emit_event", &[]).unwrap(); + + // Drop the event table. + test.use_precompiled_module("auto-migration-drop-event-table-after"); + test.publish() + .name(&identity) + .break_clients(true) + .run() + .expect("Dropping the event table should succeed"); + + // Wait until data written after the drop is durable, + // which implies the drop itself is durable too. + test.call("add_person", &["Julie"]).unwrap(); + let output = test.sql_confirmed("SELECT * FROM person WHERE name = 'Julie'").unwrap(); + assert!(output.contains("Julie"), "Data not confirmed before restart: {output}"); + + // Restarting forces a commitlog replay, which must replay the event table drop. + test.restart_server(); + + let output = test.sql("SELECT name FROM person").unwrap(); + assert!(output.contains("Robert"), "Expected 'Robert' after restart: {output}"); + assert!(output.contains("Julie"), "Expected 'Julie' after restart: {output}"); + + // The database should still accept writes after replay. + test.call("add_person", &["Samantha"]).unwrap(); + let output = test.sql("SELECT name FROM person WHERE name = 'Samantha'").unwrap(); + assert!( + output.contains("Samantha"), + "Expected 'Samantha' after restart: {output}" + ); +} diff --git a/crates/smoketests/tests/smoketests/change_host_type.rs b/crates/smoketests/tests/standalone/change_host_type.rs similarity index 100% rename from crates/smoketests/tests/smoketests/change_host_type.rs rename to crates/smoketests/tests/standalone/change_host_type.rs diff --git a/crates/smoketests/tests/smoketests/cli/auth.rs b/crates/smoketests/tests/standalone/cli/auth.rs similarity index 96% rename from crates/smoketests/tests/smoketests/cli/auth.rs rename to crates/smoketests/tests/standalone/cli/auth.rs index 73e9477d51f..f26c7ef60db 100644 --- a/crates/smoketests/tests/smoketests/cli/auth.rs +++ b/crates/smoketests/tests/standalone/cli/auth.rs @@ -1,6 +1,6 @@ //! CLI auth command tests (`login` / `logout`) -use spacetimedb_smoketests::{require_local_server, Smoketest}; +use spacetimedb_smoketests::{require_server_issued_login, Smoketest}; use std::fs; use std::process::Output; use std::time::{Duration, Instant}; @@ -34,7 +34,7 @@ fn write_config(test: &Smoketest, config: &toml::Table) { #[test] fn cli_logout_removes_cached_tokens() { - require_local_server!(); + require_server_issued_login!(); let test = Smoketest::builder().autopublish(false).build(); let login = test.spacetime_cmd(&["login", "--server-issued-login", &test.server_url]); @@ -72,7 +72,7 @@ fn cli_logout_removes_cached_tokens() { #[test] // Even if there's no web session, logout still removes the SpacetimeDB token fn cli_logout_removes_cached_tokens_without_web_token() { - require_local_server!(); + require_server_issued_login!(); let test = Smoketest::builder().autopublish(false).build(); let login = test.spacetime_cmd(&["login", "--server-issued-login", &test.server_url]); @@ -101,7 +101,7 @@ fn cli_logout_removes_cached_tokens_without_web_token() { #[test] fn cli_logout_is_idempotent() { - require_local_server!(); + require_server_issued_login!(); let test = Smoketest::builder().autopublish(false).build(); let login = test.spacetime_cmd(&["login", "--server-issued-login", &test.server_url]); @@ -126,7 +126,7 @@ fn cli_logout_is_idempotent() { #[test] fn cli_direct_login_works_and_shows_core_messages() { - require_local_server!(); + require_server_issued_login!(); let test = Smoketest::builder().autopublish(false).build(); let login = test.spacetime_cmd(&["login", "--server-issued-login", &test.server_url]); @@ -150,7 +150,7 @@ fn cli_direct_login_works_and_shows_core_messages() { #[test] fn cli_logging_in_twice_works() { - require_local_server!(); + require_server_issued_login!(); let test = Smoketest::builder().autopublish(false).build(); let first = test.spacetime_cmd(&["login", "--server-issued-login", &test.server_url]); diff --git a/crates/smoketests/tests/smoketests/cli/dev.rs b/crates/smoketests/tests/standalone/cli/dev.rs similarity index 90% rename from crates/smoketests/tests/smoketests/cli/dev.rs rename to crates/smoketests/tests/standalone/cli/dev.rs index c2e81bf37fc..fd04c7f1866 100644 --- a/crates/smoketests/tests/smoketests/cli/dev.rs +++ b/crates/smoketests/tests/standalone/cli/dev.rs @@ -8,6 +8,7 @@ fn cli_cmd() -> Command { Command::new(ensure_binaries_built()) } +/// Standalone-only: this checks local CLI help output, so a remote cluster adds no coverage. #[test] fn cli_dev_help_shows_template_option() { let output = cli_cmd().args(["dev", "--help"]).output().expect("failed to execute"); @@ -21,6 +22,7 @@ fn cli_dev_help_shows_template_option() { assert!(predicate::str::contains("-t").eval(&stdout), "stdout should contain -t"); } +/// Standalone-only: this checks local argument parsing before any server request is made. #[test] fn cli_dev_accepts_template_flag() { // Running with an invalid server should fail, but not because of the template flag @@ -38,6 +40,7 @@ fn cli_dev_accepts_template_flag() { ); } +/// Standalone-only: this checks local argument parsing before any server request is made. #[test] fn cli_dev_accepts_short_template_flag() { let output = cli_cmd() @@ -54,6 +57,7 @@ fn cli_dev_accepts_short_template_flag() { ); } +/// Standalone-only: this initializes and inspects a project entirely on the local filesystem. #[test] fn cli_init_with_template_creates_project() { let temp_dir = tempfile::tempdir().expect("failed to create temp dir"); @@ -86,6 +90,7 @@ fn cli_init_with_template_creates_project() { assert!(project_dir.join("src").exists(), "src directory should exist"); } +/// Standalone-only: this validates local config parsing before the CLI contacts a server. #[test] fn config_with_invalid_field_shows_error() { // Test that using invalid field names shows a helpful error message diff --git a/crates/smoketests/tests/smoketests/cli/generate.rs b/crates/smoketests/tests/standalone/cli/generate.rs similarity index 96% rename from crates/smoketests/tests/smoketests/cli/generate.rs rename to crates/smoketests/tests/standalone/cli/generate.rs index 307d66c9e95..39d3c5a95a8 100644 --- a/crates/smoketests/tests/smoketests/cli/generate.rs +++ b/crates/smoketests/tests/standalone/cli/generate.rs @@ -7,6 +7,7 @@ fn cli_cmd() -> Command { Command::new(ensure_binaries_built()) } +/// Standalone-only: this builds a local module and generates bindings without contacting a server. #[test] fn cli_generate_with_config_but_no_match_uses_cli_args() { // Test that when config exists but doesn't match CLI args, we use CLI args diff --git a/crates/smoketests/tests/smoketests/cli/list.rs b/crates/smoketests/tests/standalone/cli/list.rs similarity index 100% rename from crates/smoketests/tests/smoketests/cli/list.rs rename to crates/smoketests/tests/standalone/cli/list.rs diff --git a/crates/smoketests/tests/standalone/cli/mod.rs b/crates/smoketests/tests/standalone/cli/mod.rs new file mode 100644 index 00000000000..8c2b6481318 --- /dev/null +++ b/crates/smoketests/tests/standalone/cli/mod.rs @@ -0,0 +1,5 @@ +mod auth; +mod dev; +mod generate; +mod list; +mod server; diff --git a/crates/smoketests/tests/smoketests/cli/server.rs b/crates/smoketests/tests/standalone/cli/server.rs similarity index 89% rename from crates/smoketests/tests/smoketests/cli/server.rs rename to crates/smoketests/tests/standalone/cli/server.rs index d821a4d50e0..e2fb35ce991 100644 --- a/crates/smoketests/tests/smoketests/cli/server.rs +++ b/crates/smoketests/tests/standalone/cli/server.rs @@ -1,7 +1,7 @@ -//! CLI server command tests +//! Standalone-only CLI server command tests. use spacetimedb_guard::ensure_binaries_built; -use spacetimedb_smoketests::{require_local_server, Smoketest}; +use spacetimedb_smoketests::require_local_server; use std::fs; use std::io::Read; use std::net::TcpListener; @@ -98,20 +98,6 @@ fn stop_child(mut child: Child) { let _ = child.wait(); } -#[test] -fn cli_can_ping_spacetimedb_server() { - let spacetime = Smoketest::builder().autopublish(false).build(); - let output = cli_cmd() - .args(["server", "ping", &spacetime.server_url]) - .output() - .expect("failed to execute"); - assert!( - output.status.success(), - "ping failed: {}", - String::from_utf8_lossy(&output.stderr) - ); -} - #[test] fn cli_start_uses_listen_addr_from_cli_toml() { require_local_server!(); diff --git a/crates/smoketests/tests/smoketests/client_connection_errors.rs b/crates/smoketests/tests/standalone/client_connection_errors.rs similarity index 65% rename from crates/smoketests/tests/smoketests/client_connection_errors.rs rename to crates/smoketests/tests/standalone/client_connection_errors.rs index ec1fb641cb6..7375e0b6848 100644 --- a/crates/smoketests/tests/smoketests/client_connection_errors.rs +++ b/crates/smoketests/tests/standalone/client_connection_errors.rs @@ -5,64 +5,6 @@ use std::net::TcpStream; use std::thread; use std::time::{Duration, Instant}; -/// Test that client_connected returning an error rejects the connection -#[test] -fn test_client_connected_error_rejects_connection() { - let test = Smoketest::builder() - .precompiled_module("client-connection-reject") - .build(); - - // Subscribe should fail because client_connected returns an error - let result = test.subscribe(&["SELECT * FROM all_u8s"]).expect_rows(0).run(); - assert!( - result.is_err(), - "Expected subscribe to fail when client_connected returns error" - ); - - let logs = test.logs(100).unwrap(); - assert!( - logs.iter().any(|l| l.contains("Rejecting connection from client")), - "Expected rejection message in logs: {:?}", - logs - ); - assert!( - !logs.iter().any(|l| l.contains("This should never be called")), - "client_disconnected should not have been called: {:?}", - logs - ); -} - -/// Test that client_disconnected panicking still cleans up the st_client row -#[test] -fn test_client_disconnected_error_still_deletes_st_client() { - let test = Smoketest::builder() - .precompiled_module("client-connection-disconnect-panic") - .build(); - - // Subscribe should succeed (client_connected returns Ok) - let result = test.subscribe(&["SELECT * FROM all_u8s"]).expect_rows(0).run(); - assert!(result.is_ok(), "Expected subscribe to succeed"); - - let logs = test.logs(100).unwrap(); - assert!( - logs.iter() - .any(|l| { l.contains("This should be called, but the `st_client` row should still be deleted") }), - "Expected disconnect panic message in logs: {:?}", - logs - ); - - // Verify the websocket's st_client row was deleted despite the panic. - // The SQL query itself creates a temporary connection, so we may see - // exactly one row (the SQL connection's own), but the websocket's row - // should be gone. - let sql_out = test.sql("SELECT * FROM st_client").unwrap(); - let row_count = sql_out.lines().filter(|l| l.contains("0x")).count(); - assert!( - row_count <= 1, - "Expected at most 1 st_client row (the SQL connection itself), got {row_count}: {sql_out}", - ); -} - #[test] fn test_http_reducer_call_cancel_still_deletes_st_client() { require_local_server!(); diff --git a/crates/smoketests/tests/smoketests/create_project.rs b/crates/smoketests/tests/standalone/create_project.rs similarity index 96% rename from crates/smoketests/tests/smoketests/create_project.rs rename to crates/smoketests/tests/standalone/create_project.rs index 1c77559661e..5d9fd160bec 100644 --- a/crates/smoketests/tests/smoketests/create_project.rs +++ b/crates/smoketests/tests/standalone/create_project.rs @@ -4,6 +4,7 @@ use tempfile::tempdir; /// Ensure that the CLI is able to create a local project. /// This test does not depend on a running spacetimedb instance. +/// Standalone-only: it exercises only local CLI validation and filesystem changes. #[test] fn test_create_project() { let cli_path = ensure_binaries_built(); diff --git a/crates/smoketests/tests/smoketests/csharp_aot_module.rs b/crates/smoketests/tests/standalone/csharp_aot_module.rs similarity index 97% rename from crates/smoketests/tests/smoketests/csharp_aot_module.rs rename to crates/smoketests/tests/standalone/csharp_aot_module.rs index 3824043bf1a..cf81726d5fb 100644 --- a/crates/smoketests/tests/smoketests/csharp_aot_module.rs +++ b/crates/smoketests/tests/standalone/csharp_aot_module.rs @@ -26,6 +26,8 @@ fn dotnet_major_version() -> Option { /// NativeAOT-LLVM targets WASI and uses WASI SDK (clang), not the wasi-experimental /// workload or emscripten. WASI SDK is auto-downloaded by SpacetimeDB.Runtime.targets. /// The user must set EXPERIMENTAL_WASM_AOT=1 to enable the AOT build path. +/// +/// Standalone-only: this invokes the local .NET toolchain and never contacts a server. #[test] fn test_build_csharp_module_aot() { require_dotnet!(); diff --git a/crates/smoketests/tests/smoketests/csharp_module.rs b/crates/smoketests/tests/standalone/csharp_module.rs similarity index 98% rename from crates/smoketests/tests/smoketests/csharp_module.rs rename to crates/smoketests/tests/standalone/csharp_module.rs index b9fa3f1c663..0975e78da99 100644 --- a/crates/smoketests/tests/smoketests/csharp_module.rs +++ b/crates/smoketests/tests/standalone/csharp_module.rs @@ -7,6 +7,7 @@ use std::process::Command; /// Ensure that the CLI is able to create and compile a C# project. /// This test does not depend on a running SpacetimeDB instance. /// Skips if dotnet 8.0+ is not available. +/// Standalone-only: it exercises only local project generation and compilation. #[test] fn test_build_csharp_module() { require_dotnet!(); diff --git a/crates/smoketests/tests/smoketests/default_module_clippy.rs b/crates/smoketests/tests/standalone/default_module_clippy.rs similarity index 89% rename from crates/smoketests/tests/smoketests/default_module_clippy.rs rename to crates/smoketests/tests/standalone/default_module_clippy.rs index 73e76f40357..e098cc01448 100644 --- a/crates/smoketests/tests/smoketests/default_module_clippy.rs +++ b/crates/smoketests/tests/standalone/default_module_clippy.rs @@ -30,12 +30,14 @@ fn check_template_clippy(template_name: &str) { } /// Ensure that the basic-rs template module has no clippy errors or warnings +/// Standalone-only: this runs `cargo clippy` locally and never contacts a server. #[test] fn test_basic_rs_template_clippy() { check_template_clippy("basic-rs"); } /// Ensure that the chat-console-rs template module has no clippy errors or warnings +/// Standalone-only: this runs `cargo clippy` locally and never contacts a server. #[test] fn test_chat_console_rs_template_clippy() { check_template_clippy("chat-console-rs"); diff --git a/crates/smoketests/tests/smoketests/detect_wasm_bindgen.rs b/crates/smoketests/tests/standalone/detect_wasm_bindgen.rs similarity index 90% rename from crates/smoketests/tests/smoketests/detect_wasm_bindgen.rs rename to crates/smoketests/tests/standalone/detect_wasm_bindgen.rs index 8ff2224cdab..baf95009eb4 100644 --- a/crates/smoketests/tests/smoketests/detect_wasm_bindgen.rs +++ b/crates/smoketests/tests/standalone/detect_wasm_bindgen.rs @@ -26,6 +26,7 @@ pub fn test(_ctx: &ReducerContext) { "#; /// Ensure that spacetime build properly catches wasm_bindgen imports +/// Standalone-only: this validates local CLI build diagnostics without publishing a module. #[test] fn test_detect_wasm_bindgen() { let test = Smoketest::builder() @@ -46,6 +47,7 @@ fn test_detect_wasm_bindgen() { } /// Ensure that spacetime build properly catches getrandom usage +/// Standalone-only: this validates local CLI build diagnostics without publishing a module. #[test] fn test_detect_getrandom() { let test = Smoketest::builder() diff --git a/crates/smoketests/tests/smoketests/http_egress.rs b/crates/smoketests/tests/standalone/http_egress.rs similarity index 82% rename from crates/smoketests/tests/smoketests/http_egress.rs rename to crates/smoketests/tests/standalone/http_egress.rs index 847d03134bb..644b3c3153e 100644 --- a/crates/smoketests/tests/smoketests/http_egress.rs +++ b/crates/smoketests/tests/standalone/http_egress.rs @@ -41,21 +41,6 @@ fn spawn_redirect_server(location: &str) -> (u16, JoinHandle (port, handle) } -#[test] -fn test_http_disallowed_ip_is_blocked() { - let test = Smoketest::builder().precompiled_module("http-egress").build(); - - let output = test.call_output("request_disallowed_ip", &["http://10.0.0.1:80/"]); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - assert!( - output.status.success(), - "Expected request_disallowed_ip to succeed after observing blocked egress error.\nstdout:\n{}\nstderr:\n{}", - stdout, - stderr - ); -} - #[test] fn test_http_redirect_to_disallowed_ip_is_blocked() { require_local_server!(); diff --git a/crates/smoketests/tests/smoketests/pg_wire.rs b/crates/smoketests/tests/standalone/pg_wire.rs similarity index 100% rename from crates/smoketests/tests/smoketests/pg_wire.rs rename to crates/smoketests/tests/standalone/pg_wire.rs diff --git a/crates/smoketests/tests/smoketests/restart.rs b/crates/smoketests/tests/standalone/restart.rs similarity index 100% rename from crates/smoketests/tests/smoketests/restart.rs rename to crates/smoketests/tests/standalone/restart.rs diff --git a/crates/smoketests/tests/smoketests/servers.rs b/crates/smoketests/tests/standalone/servers.rs similarity index 92% rename from crates/smoketests/tests/smoketests/servers.rs rename to crates/smoketests/tests/standalone/servers.rs index c275721103f..aaacdbea4c4 100644 --- a/crates/smoketests/tests/smoketests/servers.rs +++ b/crates/smoketests/tests/standalone/servers.rs @@ -1,7 +1,7 @@ use regex::Regex; use spacetimedb_smoketests::{require_local_server, Smoketest}; -/// Verify that we can add and list server configurations +/// Verify that we can add and list server configurations. #[test] fn test_servers() { // This only covers local CLI config behavior, so it is not valuable to run @@ -66,7 +66,8 @@ fn test_servers() { ); } -/// Verify that we can edit server configurations +/// Standalone-only: this edits and reads local CLI configuration, so a remote +/// cluster adds no coverage. #[test] fn test_edit_server() { let test = Smoketest::builder().autopublish(false).build(); @@ -94,7 +95,7 @@ fn test_edit_server() { let edited_re = Regex::new(r"(?m)^\s*edited-testnet\.spacetimedb\.com\s+https\s+edited-testnet\s*$").unwrap(); assert!( edited_re.is_match(&servers), - "Expected edited server in list: {}", + "Expected edited server in server list: {}", servers ); } diff --git a/crates/smoketests/tests/standalone/templates.rs b/crates/smoketests/tests/standalone/templates.rs new file mode 100644 index 00000000000..774de920e44 --- /dev/null +++ b/crates/smoketests/tests/standalone/templates.rs @@ -0,0 +1,337 @@ +use anyhow::{bail, Context, Result}; +use regex::Regex; +use serde_json::Value; +use spacetimedb_guard::ensure_binaries_built; +use spacetimedb_smoketests::workspace_root; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use tempfile::TempDir; + +macro_rules! registered_template_ids { + ($($test_name:ident => $template_id:literal),+ $(,)?) => { + const REGISTERED_TEMPLATE_IDS: &[&str] = &[$($template_id),+]; + }; +} + +spacetimedb_smoketests::for_each_smoketest_template!(registered_template_ids); + +fn init_template(template_id: &str) -> Result<(TempDir, PathBuf)> { + let tmpdir = tempfile::tempdir().context("Failed to create temp dir")?; + let project_name = format!("test-{template_id}"); + let project_path = tmpdir.path().join(&project_name); + let config_path = tmpdir.path().join("config.toml"); + let output = Command::new(ensure_binaries_built()) + .arg("--config-path") + .arg(&config_path) + .args([ + "init", + "--template", + template_id, + "--project-path", + project_path.to_str().unwrap(), + "--non-interactive", + &project_name, + ]) + .current_dir(tmpdir.path()) + .output() + .with_context(|| format!("Failed to execute spacetime init --template {template_id}"))?; + + if !output.status.success() { + bail!( + "spacetime init --template {template_id} failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + Ok((tmpdir, project_path)) +} + +fn fake_dotnet_path(dir: &Path, sdk_list_output: &str) -> Result { + let executable_name = if cfg!(windows) { "dotnet.exe" } else { "dotnet" }; + let dotnet_path = dir.join(executable_name); + let echo_lines = sdk_list_output + .lines() + .map(|line| format!("echo {line}")) + .collect::>() + .join(if cfg!(windows) { "\r\n" } else { "\n" }); + + if cfg!(windows) { + let source_path = dir.join("fake_dotnet.rs"); + fs::write( + &source_path, + format!( + r#"fn main() {{ + if std::env::args().nth(1).as_deref() == Some("--list-sdks") {{ + print!("{{}}", {sdk_list_output:?}); + return; + }} + + std::process::exit(1); +}} +"# + ), + ) + .with_context(|| format!("Failed to write fake dotnet source {source_path:?}"))?; + + let output = Command::new("rustc") + .arg(&source_path) + .arg("-o") + .arg(&dotnet_path) + .output() + .context("Failed to spawn rustc for fake dotnet")?; + if !output.status.success() { + bail!( + "rustc failed to compile fake dotnet:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + } else { + fs::write( + &dotnet_path, + format!("#!/usr/bin/env sh\nif [ \"$1\" = \"--list-sdks\" ]; then\n{echo_lines}\nexit 0\nfi\nexit 1\n"), + ) + .with_context(|| format!("Failed to write fake dotnet executable {dotnet_path:?}"))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut permissions = fs::metadata(&dotnet_path)?.permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&dotnet_path, permissions)?; + } + } + + Ok(dotnet_path) +} + +fn init_basic_cs_with_fake_dotnet(sdk_list_output: &str) -> Result<(TempDir, PathBuf)> { + let tmpdir = tempfile::tempdir().context("Failed to create temp dir")?; + let fake_bin = tmpdir.path().join("bin"); + fs::create_dir(&fake_bin).context("Failed to create fake dotnet bin dir")?; + fake_dotnet_path(&fake_bin, sdk_list_output)?; + + let current_path = env::var_os("PATH").unwrap_or_default(); + let test_path = env::join_paths(std::iter::once(fake_bin).chain(env::split_paths(¤t_path))) + .context("Failed to build test PATH")?; + + let project_name = "test-basic-cs-default-dotnet"; + let project_path = tmpdir.path().join(project_name); + let config_path = tmpdir.path().join("config.toml"); + let output = Command::new(ensure_binaries_built()) + .arg("--config-path") + .arg(&config_path) + .args([ + "init", + "--template", + "basic-cs", + "--project-path", + project_path.to_str().unwrap(), + "--non-interactive", + project_name, + ]) + .env("PATH", test_path) + .current_dir(tmpdir.path()) + .output() + .context("Failed to execute spacetime init")?; + + if !output.status.success() { + bail!( + "spacetime init with fake dotnet failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + + Ok((tmpdir, project_path)) +} + +fn assert_basic_cs_default_dotnet(sdk_list_output: &str, expected_major: u8) -> Result<()> { + let (_tmpdir, project_path) = init_basic_cs_with_fake_dotnet(sdk_list_output)?; + let server_path = project_path.join("spacetimedb"); + + let global_json = fs::read_to_string(server_path.join("global.json")).context("Failed to read global.json")?; + assert!( + global_json.contains(&format!("\"version\": \"{expected_major}.0.100\"")), + "global.json did not target .NET {expected_major}:\n{global_json}" + ); + + let csproj_path = find_csproj(&server_path)?; + let csproj = fs::read_to_string(&csproj_path).with_context(|| format!("Failed to read {csproj_path:?}"))?; + assert!( + csproj.contains(&format!("net{expected_major}.0")), + "{csproj_path:?} did not target net{expected_major}.0:\n{csproj}" + ); + assert!( + !csproj.contains(""), + "{csproj_path:?} should use a single TargetFramework after init:\n{csproj}" + ); + + Ok(()) +} + +fn assert_major_minor_version(actual: &str, context: impl std::fmt::Display) -> Result<()> { + let re = Regex::new(r"^\d+\.\d+$").unwrap(); + if !re.is_match(actual) { + bail!("{context}: expected MAJOR.MINOR, got {actual}"); + } + Ok(()) +} + +fn assert_major_minor_patch_wildcard(actual: &str, context: impl std::fmt::Display) -> Result<()> { + let re = Regex::new(r"^\d+\.\d+\.\*$").unwrap(); + if !re.is_match(actual) { + bail!("{context}: expected MAJOR.MINOR.*, got {actual}"); + } + Ok(()) +} + +fn read_cargo_dependency_version(cargo_toml_path: &Path, package_name: &str) -> Result { + let content = fs::read_to_string(cargo_toml_path).with_context(|| format!("Failed to read {cargo_toml_path:?}"))?; + let data: toml::Value = content + .parse() + .with_context(|| format!("Failed to parse {cargo_toml_path:?}"))?; + let dep = data + .get("dependencies") + .and_then(|deps| deps.get(package_name)) + .with_context(|| format!("No dependency `{package_name}` found in {cargo_toml_path:?}"))?; + match dep { + toml::Value::String(version) => Ok(version.clone()), + toml::Value::Table(table) => table + .get("version") + .and_then(|value| value.as_str()) + .map(String::from) + .with_context(|| format!("Dependency `{package_name}` in {cargo_toml_path:?} has no version")), + _ => bail!("Unsupported dependency `{package_name}` format in {cargo_toml_path:?}"), + } +} + +fn read_package_json_dependency_version(package_json_path: &Path, package_name: &str) -> Result { + let content = + fs::read_to_string(package_json_path).with_context(|| format!("Failed to read {package_json_path:?}"))?; + let data: Value = + serde_json::from_str(&content).with_context(|| format!("Failed to parse {package_json_path:?}"))?; + data.get("dependencies") + .and_then(|deps| deps.get(package_name)) + .and_then(|value| value.as_str()) + .map(String::from) + .with_context(|| format!("No dependency `{package_name}` found in {package_json_path:?}")) +} + +fn read_csproj_package_reference_version(csproj_path: &Path, package_name: &str) -> Result { + let content = fs::read_to_string(csproj_path).with_context(|| format!("Failed to read {csproj_path:?}"))?; + let root = + xmltree::Element::parse(content.as_bytes()).with_context(|| format!("Failed to parse XML {csproj_path:?}"))?; + + root.children + .iter() + .filter_map(|node| match node { + xmltree::XMLNode::Element(element) if element.name == "ItemGroup" => Some(element), + _ => None, + }) + .flat_map(|item_group| item_group.children.iter()) + .filter_map(|node| match node { + xmltree::XMLNode::Element(element) if element.name == "PackageReference" => Some(element), + _ => None, + }) + .find(|package_ref| package_ref.attributes.get("Include").map(String::as_str) == Some(package_name)) + .and_then(|package_ref| package_ref.attributes.get("Version").cloned()) + .with_context(|| format!("No PackageReference `{package_name}` found in {csproj_path:?}")) +} + +fn find_csproj(dir: &Path) -> Result { + fs::read_dir(dir) + .with_context(|| format!("Failed to read {dir:?}"))? + .flatten() + .map(|entry| entry.path()) + .find(|path| path.extension().is_some_and(|extension| extension == "csproj")) + .with_context(|| format!("No .csproj found in {dir:?}")) +} + +fn read_spacetimedb_cpp_version(cmake_path: &Path) -> Result { + let content = fs::read_to_string(cmake_path).with_context(|| format!("Failed to read {cmake_path:?}"))?; + let re = Regex::new(r#"set\(SPACETIMEDB_CPP_VERSION\s+"([^"]+)""#).unwrap(); + let captures = re + .captures(&content) + .with_context(|| format!("No SPACETIMEDB_CPP_VERSION found in {cmake_path:?}"))?; + Ok(captures.get(1).unwrap().as_str().to_string()) +} + +fn discovered_template_ids() -> Vec { + let templates_dir = workspace_root().join("templates"); + let mut templates = fs::read_dir(&templates_dir) + .unwrap_or_else(|error| panic!("Failed to read template directory {templates_dir:?}: {error}")) + .flatten() + .filter(|entry| entry.file_type().is_ok_and(|file_type| file_type.is_dir())) + .filter(|entry| entry.path().join(".template.json").exists()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect::>(); + templates.sort(); + templates +} + +/// Standalone-only: this initializes templates and inspects local dependency +/// manifests; a remote cluster is never contacted. +#[test] +fn test_basic_template_dependency_versions() -> Result<()> { + let (_basic_cpp_tmpdir, basic_cpp_path) = init_template("basic-cpp")?; + let cpp_server_version = read_spacetimedb_cpp_version(&basic_cpp_path.join("spacetimedb/CMakeLists.txt"))?; + assert_major_minor_version(&cpp_server_version, "basic-cpp C++ server SPACETIMEDB_CPP_VERSION")?; + let cpp_client_manifest = basic_cpp_path.join("Cargo.toml"); + if !cpp_client_manifest.exists() { + bail!("basic-cpp expected Rust client manifest at {cpp_client_manifest:?}"); + } + + let (_basic_rs_tmpdir, basic_rs_path) = init_template("basic-rs")?; + let rs_server_version = + read_cargo_dependency_version(&basic_rs_path.join("spacetimedb/Cargo.toml"), "spacetimedb")?; + assert_major_minor_patch_wildcard(&rs_server_version, "basic-rs Rust server spacetimedb")?; + let rs_client_version = read_cargo_dependency_version(&basic_rs_path.join("Cargo.toml"), "spacetimedb-sdk")?; + assert_major_minor_patch_wildcard(&rs_client_version, "basic-rs Rust client spacetimedb-sdk")?; + + let (_basic_ts_tmpdir, basic_ts_path) = init_template("basic-ts")?; + let ts_server_version = + read_package_json_dependency_version(&basic_ts_path.join("spacetimedb/package.json"), "spacetimedb")?; + assert_major_minor_patch_wildcard(&ts_server_version, "basic-ts TypeScript server spacetimedb")?; + let ts_client_version = read_package_json_dependency_version(&basic_ts_path.join("package.json"), "spacetimedb")?; + assert_major_minor_patch_wildcard(&ts_client_version, "basic-ts TypeScript client spacetimedb")?; + + let (_basic_cs_tmpdir, basic_cs_path) = init_template("basic-cs")?; + let cs_server_project = find_csproj(&basic_cs_path.join("spacetimedb"))?; + let cs_server_version = read_csproj_package_reference_version(&cs_server_project, "SpacetimeDB.Runtime")?; + assert_major_minor_patch_wildcard(&cs_server_version, "basic-cs C# server SpacetimeDB.Runtime")?; + let cs_client_version = + read_csproj_package_reference_version(&basic_cs_path.join("client.csproj"), "SpacetimeDB.ClientSDK")?; + assert_major_minor_patch_wildcard(&cs_client_version, "basic-cs C# client SpacetimeDB.ClientSDK")?; + + Ok(()) +} + +/// Standalone-only: this uses a fake local `dotnet` executable to validate CLI +/// project generation and never contacts a server. +#[test] +fn test_basic_cs_init_default_dotnet_selection() -> Result<()> { + assert_basic_cs_default_dotnet("8.0.416 [/usr/share/dotnet/sdk]", 8)?; + assert_basic_cs_default_dotnet("10.0.100 [/usr/share/dotnet/sdk]", 10)?; + assert_basic_cs_default_dotnet("8.0.416 [/usr/share/dotnet/sdk]\n10.0.100 [/usr/share/dotnet/sdk]", 10)?; + assert_basic_cs_default_dotnet("", 10)?; + Ok(()) +} + +/// Standalone-only: this compares local template metadata with the test +/// registry and never contacts a server. +#[test] +fn test_template_registry_matches_discovered_templates() { + let registered = REGISTERED_TEMPLATE_IDS + .iter() + .map(|template_id| (*template_id).to_owned()) + .collect::>(); + assert_eq!( + discovered_template_ids(), + registered, + "Every discovered template must have its own nextest-visible test" + ); +} diff --git a/crates/smoketests/tests/smoketests/typescript_index_source_name.rs b/crates/smoketests/tests/standalone/typescript_index_source_name.rs similarity index 100% rename from crates/smoketests/tests/smoketests/typescript_index_source_name.rs rename to crates/smoketests/tests/standalone/typescript_index_source_name.rs diff --git a/crates/smoketests/tests/standalone/views.rs b/crates/smoketests/tests/standalone/views.rs new file mode 100644 index 00000000000..b65f2ecb058 --- /dev/null +++ b/crates/smoketests/tests/standalone/views.rs @@ -0,0 +1,138 @@ +use std::path::PathBuf; + +use serde_json::{json, Value}; +use spacetimedb_smoketests::{require_local_server, workspace_root, Smoketest}; + +const STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY: &str = + "c200f6ec405075e508c2ed6474019332d6a2a46c69614306cc4bd980e0b8b767"; + +fn project_fields(events: Vec, view_name: &str, projected_fields: &[&str]) -> Vec { + let project_row = |row: &Value| { + if projected_fields.is_empty() { + row.clone() + } else { + let mut projected = serde_json::Map::new(); + for field in projected_fields { + if let Some(value) = row.get(*field) { + projected.insert((*field).to_string(), value.clone()); + } + } + Value::Object(projected) + } + }; + + events + .into_iter() + .map(|event| { + json!({ + view_name: { + "deletes": event[view_name]["deletes"] + .as_array() + .unwrap() + .iter() + .map(&project_row) + .collect::>(), + "inserts": event[view_name]["inserts"] + .as_array() + .unwrap() + .iter() + .map(&project_row) + .collect::>() + } + }) + }) + .collect() +} + +fn stale_view_backing_table_fixture() -> PathBuf { + workspace_root() + .join("crates") + .join("smoketests") + .join("fixtures") + .join("stale-view-backing-table-v2.6.0") +} + +fn stale_view_backing_table_test() -> Smoketest { + let test = Smoketest::builder() + .data_dir_fixture( + stale_view_backing_table_fixture(), + STALE_VIEW_BACKING_TABLE_FIXTURE_IDENTITY, + ) + .autopublish(false) + .build(); + + test.new_identity().unwrap(); + test +} + +#[test] +fn test_repair_stale_sender_scoped_view_backing_table_on_startup() { + require_local_server!(); + + let test = stale_view_backing_table_test(); + + let sender_view_sub = test + .subscribe(&["select * from player"]) + .expect_rows(2) + .background() + .unwrap(); + + test.call("set_player_state", &["42", "1"]).unwrap(); + test.call("set_player_state", &["42", "2"]).unwrap(); + + let sender_view_events = sender_view_sub.collect().unwrap(); + let sender_view_projection = project_fields(sender_view_events, "player", &["id", "level"]); + assert_eq!( + serde_json::json!(sender_view_projection), + json!([ + { + "player": { + "deletes": [], + "inserts": [{ "id": 42, "level": 1 }] + } + }, + { + "player": { + "deletes": [{ "id": 42, "level": 1 }], + "inserts": [{ "id": 42, "level": 2 }] + } + } + ]) + ); +} + +#[test] +fn test_repair_stale_anonymous_view_backing_table_on_startup() { + require_local_server!(); + + let test = stale_view_backing_table_test(); + + let anonymous_view_sub = test + .subscribe(&["select * from player_and_level"]) + .expect_rows(2) + .background() + .unwrap(); + + test.call("add_player_level", &["1", "2"]).unwrap(); + test.call("add_player_level", &["2", "2"]).unwrap(); + + let anonymous_view_events = anonymous_view_sub.collect().unwrap(); + let anonymous_view_projection = project_fields(anonymous_view_events, "player_and_level", &["id", "level"]); + assert_eq!( + serde_json::json!(anonymous_view_projection), + json!([ + { + "player_and_level": { + "deletes": [], + "inserts": [{ "id": 1, "level": 2 }] + } + }, + { + "player_and_level": { + "deletes": [], + "inserts": [{ "id": 2, "level": 2 }] + } + } + ]) + ); +} diff --git a/tools/ci/commands/smoketest-checks/Cargo.toml b/tools/ci/commands/smoketest-checks/Cargo.toml new file mode 100644 index 00000000000..6ae0e8f47c9 --- /dev/null +++ b/tools/ci/commands/smoketest-checks/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "ci-smoketest-checks" +version = "0.1.0" +edition.workspace = true + +[lints] +workspace = true diff --git a/tools/ci/commands/smoketest-checks/src/main.rs b/tools/ci/commands/smoketest-checks/src/main.rs new file mode 100644 index 00000000000..ead3e44ddcf --- /dev/null +++ b/tools/ci/commands/smoketest-checks/src/main.rs @@ -0,0 +1,108 @@ +#![allow(clippy::disallowed_macros)] + +use std::ffi::OsStr; +use std::fs; +use std::io::{Error, Result}; +use std::path::{Path, PathBuf}; + +fn main() -> Result<()> { + if !Path::new("Cargo.toml").exists() { + return Err(Error::other( + "You must execute this command from the SpacetimeDB repository root", + )); + } + check_smoketest_module_lists_complete()?; + check_no_require_local_server_cluster_tests()?; + eprintln!("smoketest module lists and suite constraints are up to date."); + Ok(()) +} + +fn check_smoketest_module_lists_complete() -> Result<()> { + let tests_dir = Path::new("crates/smoketests/tests"); + for suite in ["cluster", "standalone"] { + let suite_dir = tests_dir.join(suite); + let suite_root = tests_dir.join(format!("{suite}.rs")); + for source in collect_rust_sources(&suite_dir)? { + let source_dir = source + .parent() + .ok_or_else(|| Error::other("smoketest source has no parent"))?; + let (module_dir, module_name) = if source.file_name() == Some(OsStr::new("mod.rs")) { + ( + source_dir + .parent() + .ok_or_else(|| Error::other("nested module has no parent"))?, + source_dir + .file_name() + .ok_or_else(|| Error::other("nested module has no name"))? + .to_string_lossy() + .into_owned(), + ) + } else { + ( + source_dir, + source + .file_stem() + .ok_or_else(|| Error::other("smoketest source has no module name"))? + .to_string_lossy() + .into_owned(), + ) + }; + let module_file = if module_dir == suite_dir { + suite_root.clone() + } else { + module_dir.join("mod.rs") + }; + let module_list = fs::read_to_string(&module_file)?; + let private = format!("mod {module_name};"); + let public = format!("pub mod {module_name};"); + if !module_list + .lines() + .any(|line| matches!(line.trim(), value if value == private || value == public)) + { + return Err(Error::other(format!( + "{} does not declare module {module_name} from {}", + module_file.display(), + source.display(), + ))); + } + } + } + Ok(()) +} + +fn check_no_require_local_server_cluster_tests() -> Result<()> { + let mut misplaced_guards = Vec::new(); + let mut cluster_sources = collect_rust_sources(Path::new("crates/smoketests/tests/cluster"))?; + cluster_sources.push(PathBuf::from("crates/smoketests/tests/cluster.rs")); + for source in cluster_sources { + if fs::read_to_string(&source)?.contains("require_local_server!") { + misplaced_guards.push(source); + } + } + + if !misplaced_guards.is_empty() { + return Err(Error::other(format!( + "require_local_server!() may not be used in cluster smoketests:\n{}", + misplaced_guards + .iter() + .map(|path| format!("- {}", path.display())) + .collect::>() + .join("\n") + ))); + } + Ok(()) +} + +fn collect_rust_sources(dir: &Path) -> Result> { + let mut sources = Vec::new(); + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if entry.file_type()?.is_dir() { + sources.extend(collect_rust_sources(&path)?); + } else if path.extension() == Some(OsStr::new("rs")) { + sources.push(path); + } + } + Ok(sources) +} diff --git a/tools/ci/commands/smoketests/src/main.rs b/tools/ci/commands/smoketests/src/main.rs index b8d249972d6..52e38682601 100644 --- a/tools/ci/commands/smoketests/src/main.rs +++ b/tools/ci/commands/smoketests/src/main.rs @@ -1,13 +1,11 @@ #![allow(clippy::disallowed_macros)] use anyhow::{bail, ensure, Context, Result}; -use ci_common::ensure_repo_root; -use clap::{Parser, Subcommand}; +use clap::{Parser, Subcommand, ValueEnum}; use duct::cmd; use spacetimedb_guard::ensure_binaries_built; -use std::ffi::OsStr; +use std::env; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::{env, fs}; use tempfile::TempDir; #[derive(Parser)] @@ -43,18 +41,42 @@ struct SmoketestsArgs { #[arg(long, default_value_t = true, action = clap::ArgAction::Set)] dotnet: bool, + /// Select which smoketest suite to run or archive. + #[arg(long, value_enum, default_value_t = SmoketestSuite::All)] + suite: SmoketestSuite, + /// Additional arguments to pass to the test runner #[arg(trailing_var_arg = true)] args: Vec, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, ValueEnum)] +enum SmoketestSuite { + /// Run both cluster and standalone smoketests. + #[default] + All, + /// Run tests that provide useful coverage against a cluster. + Cluster, + /// Run tests assigned to standalone coverage. + Standalone, +} + +impl SmoketestSuite { + fn cargo_args(self) -> &'static [&'static str] { + match self { + Self::All => &[], + Self::Cluster => &["--test", "cluster"], + Self::Standalone => &["--test", "standalone"], + } + } +} + #[derive(Subcommand)] enum SmoketestCmd { /// Local helper: only build binaries without running tests. /// /// Use this before running `cargo test --all` to ensure binaries are built. Prepare, - CheckModList, /// CI build job: build dependencies and archive the smoketest binaries. Archive { @@ -84,14 +106,15 @@ fn main() -> Result<()> { eprintln!("Binaries ready. You can now run `cargo test --all`."); Ok(()) } - Some(SmoketestCmd::CheckModList) => { - check_smoketests_mod_rs_complete()?; - eprintln!("smoketests/mod.rs is up to date."); - Ok(()) - } - Some(SmoketestCmd::Archive { archive_file }) => archive_smoketests(&archive_file), + Some(SmoketestCmd::Archive { archive_file }) => archive_smoketests(&archive_file, args.suite), Some(SmoketestCmd::RunArchive { archive_file, args }) => run_smoketest_archive(&archive_file, args), - None => run_smoketest(args.server, args.dotnet, args.auth_host.as_deref(), args.args), + None => run_smoketest( + args.server, + args.dotnet, + args.auth_host.as_deref(), + args.suite, + args.args, + ), } } @@ -167,7 +190,7 @@ fn build_precompiled_modules() -> Result<()> { Ok(()) } -fn archive_smoketests(archive_file: &Path) -> Result<()> { +fn archive_smoketests(archive_file: &Path, suite: SmoketestSuite) -> Result<()> { build_precompiled_modules()?; let status = Command::new("cargo") @@ -178,8 +201,9 @@ fn archive_smoketests(archive_file: &Path) -> Result<()> { "--timings", "-p", "spacetimedb-smoketests", - "--archive-file", ]) + .args(suite.cargo_args()) + .arg("--archive-file") .arg(archive_file) .status()?; ensure!(status.success(), "Failed to archive smoketests"); @@ -223,7 +247,17 @@ fn run_smoketest_archive(archive_file: &Path, args: Vec) -> Result<()> { /// 16 was found to be optimal - higher values cause OS scheduler overhead. const DEFAULT_PARALLELISM: &str = "16"; -fn run_smoketest(server: Option, dotnet: bool, auth_host: Option<&str>, args: Vec) -> Result<()> { +fn run_smoketest( + server: Option, + dotnet: bool, + auth_host: Option<&str>, + suite: SmoketestSuite, + args: Vec, +) -> Result<()> { + if server.is_some() && suite == SmoketestSuite::Standalone { + bail!("the standalone smoketest suite cannot be run with --server"); + } + // 1. Build binaries first (single process, no race). Remote tests only need the CLI; // local tests also need standalone to spawn their test servers. build_cli()?; @@ -258,14 +292,9 @@ fn run_smoketest(server: Option, dotnet: bool, auth_host: Option<&str>, eprintln!("Running smoketests with cargo nextest...\n"); let mut cmd = Command::new("cargo"); set_env(&mut cmd, server, dotnet, auth_host.is_some(), &base_config_path); - cmd.args([ - "nextest", - "run", - "--release", - "-p", - "spacetimedb-smoketests", - "--no-fail-fast", - ]); + cmd.args(["nextest", "run", "--release", "-p", "spacetimedb-smoketests"]) + .args(suite.cargo_args()) + .arg("--no-fail-fast"); // Set default parallelism if user didn't specify -j if !args @@ -280,7 +309,8 @@ fn run_smoketest(server: Option, dotnet: bool, auth_host: Option<&str>, eprintln!("Running smoketests with cargo test...\n"); let mut cmd = Command::new("cargo"); set_env(&mut cmd, server, dotnet, auth_host.is_some(), &base_config_path); - cmd.args(["test", "--release", "-p", "spacetimedb-smoketests"]); + cmd.args(["test", "--release", "-p", "spacetimedb-smoketests"]) + .args(suite.cargo_args()); cmd }; let status = cmd.args(&args).status()?; @@ -367,65 +397,3 @@ fn set_env(cmd: &mut Command, server: Option, dotnet: bool, auth_host: b cmd.env("SPACETIME_USE_AUTH_HOST", if auth_host { "1" } else { "0" }); cmd.env("SMOKETESTS_DOTNET", if dotnet { "1" } else { "0" }); } - -fn check_smoketests_mod_rs_complete() -> Result<()> { - ensure_repo_root()?; - - let expected_dir = Path::new("crates/smoketests/tests/smoketests"); - let mut expected = std::collections::BTreeSet::::new(); - for entry in fs::read_dir(expected_dir)? { - let entry = entry?; - let path = entry.path(); - let name = entry.file_name(); - let name = name.to_string_lossy(); - if name == "mod.rs" { - continue; - } - if name.starts_with('.') { - continue; - } - - let ft = entry.file_type()?; - if ft.is_dir() { - expected.insert(name.to_string()); - } else if ft.is_file() - && path.extension() == Some(OsStr::new("rs")) - && let Some(stem) = path.file_stem() - { - expected.insert(stem.to_string_lossy().to_string()); - } - } - - let out = cmd!("cargo", "test", "-p", "spacetimedb-smoketests", "--", "--list",).read()?; - - let mut present = std::collections::BTreeSet::::new(); - for line in out.lines() { - let line = line.trim(); - let parts: Vec<&str> = line.split("::").collect(); - if parts.len() < 2 { - continue; - } - if parts[0] != "smoketests" { - continue; - } - present.insert(parts[1].to_string()); - } - - let missing = expected - .into_iter() - .filter(|m| !present.contains(m)) - .collect::>(); - - if !missing.is_empty() { - bail!( - "crates/smoketests/tests/smoketests/mod.rs appears incomplete; missing modules (not present in `cargo test -- --list`):\n{}", - missing - .iter() - .map(|m| format!("- mod {m};")) - .collect::>() - .join("\n") - ); - } - - Ok(()) -} diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 0b113fe7996..bb0878277d9 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -28,6 +28,10 @@ const COMMANDS: &[Command] = &[ path: &["smoketests"], package: "ci-smoketests", }, + Command { + path: &["smoketests", "check-mod-list"], + package: "ci-smoketest-checks", + }, Command { path: &["keynote-bench"], package: "ci-keynote-bench",