diff --git a/.batchfile b/.batchfile index 41cda54b..f7118959 100644 --- a/.batchfile +++ b/.batchfile @@ -1,4 +1,6 @@ # Usage: ;branch;TargetFolder https://github.com/opencloud-eu/web;design-system-docs;static/design-system +https://github.com/opencloud-eu/web;web-testing-docs;docs/dev/web/testing/running-tests +https://github.com/opencloud-eu/opencloud;server-testing-docs;docs/dev/server/testing https://github.com/opencloud-eu/cdperf;cdperf-docs;static/cdperf diff --git a/.gitignore b/.gitignore index f2295dff..d4163084 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,10 @@ /static/cdperf /static/design-system +# docs pulled from other repos at build time (git-batch) +/docs/dev/web/testing/running-tests +/docs/dev/server/testing + # Generated files .docusaurus .cache-loader diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index b20a5d91..2e551fca 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -10,6 +10,8 @@ "CHANGELOG.md", "CONTRIBUTING.md", "README.md", - "**/_static/env-vars" + "**/_static/env-vars", + "docs/dev/web/testing/running-tests", + "docs/dev/server/testing" ] } \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index 57b26e85..05f26d32 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,4 +3,6 @@ static/** README.md CHANGELOG.md CONTRIBUTING.md -**/_static/env-vars \ No newline at end of file +**/_static/env-vars +docs/dev/web/testing/running-tests/** +docs/dev/server/testing/** \ No newline at end of file diff --git a/docs/dev/web/testing/e2e-testing-standards.md b/docs/dev/web/testing/e2e-testing-standards.md index 49cd8b2e..40d6099f 100644 --- a/docs/dev/web/testing/e2e-testing-standards.md +++ b/docs/dev/web/testing/e2e-testing-standards.md @@ -18,40 +18,40 @@ Here are the test standards and guidelines we adhere to when creating Playwright ## Folder Structure -- `tests/:` - - `e2e/`: Main folder containing all (end-to-end) E2E test-related files. - - `cucumber/`: Main folder containing all Cucumber(BDD) test-related files. - - `features/`: Contains Gherkin feature files. - - `/`: Collection house for "**related"** feature files. - - `.feature`: A feature file. +- `tests/` + - `e2e/`: Main folder containing all end-to-end (E2E) test-related files. + - `features/`: Contains Gherkin feature files. + - `/`: Collection house for **related** feature files. + - `.feature`: A feature file. - - `steps/`: Holds the step definition files for mapping Gherkin steps to code. - - `.ts`: Step definitions for each feature. + - `steps/`: Step definitions that map Gherkin steps to code. They are compiled into Playwright specs by [playwright-bdd](https://github.com/vitalets/playwright-bdd). + - `.ts`: Step definitions. + - `ui/`: UI-related step definitions. - - `hooks/`: Cucumber hooks for setting up and tearing down test environments. - - `hooks.ts`: Contains `Before`, `After`, and other lifecycle hooks. + - `environment/`: playwright-bdd test setup. + - `fixtures.ts`: Custom fixtures and the `createBdd()` instance exporting `Given`, `When`, `Then`, `Before`, `After`. + - `world.ts`: The custom `World` shared across the steps of a scenario. + - `hooks.ts`: `Before`/`After` lifecycle hooks. - - `support/`: Playwright (Test implementation) - - `api/`: Contains API-related test files and configurations. - - `/`: Specific API tests for a particular service. + - `support/`: Test implementation and helpers. + - `api/`: API clients used to arrange state (e.g. LibreGraph, WebDAV, shares, Keycloak). + - `/`: Client for a particular service. - `objects/`: Contains the Page Object classes. - `/`: Collection house for related page objects for each webpage or component. - `.ts`: Page Object for each webpage or component. - - `utils/`: Utility functions and common helpers. - - `helpers.ts`: Common utility functions (e.g., date formatting, data generation). + - `utils/`: Utility functions and common helpers (e.g. accessibility checks, locator and date helpers). - - `test-data/`: Static test data files or folders for upload. - - `filesForUpload/`: Static test data files for upload. + - `filesForUpload/`: Static test data files for upload. - - `config/`: Configuration files for Playwright and other tools. - - `playwright.config.ts`: Playwright configuration. + - `playwright.config.ts`: Playwright configuration (browser projects, timeouts, `appConfig`). - - `reports/`: Generated test reports (e.g., HTML, JSON). - - `screenshots/`: Captured screenshots during test execution. + - `.features-gen/`: Playwright specs generated from the feature files by playwright-bdd (gitignored). - - `videos/`: Recorded videos of test runs. + - `playwright-report/`: HTML report of the last test run. + + - `test-results/`: Traces, screenshots and videos recorded from (failed) runs. ## Test Structure - Arrange, Act, Assert @@ -84,28 +84,27 @@ All assertions should be in your test, no assertion in the POM DO 👍 ```typescript -// POM file './pageOobjects/foo/' -// add all locators and functions related to the page. -// allowing all tests to reuse +// POM file './support/objects/foo/index.ts' +// add all locators and functions related to the page, +// allowing all tests to reuse them -import { expect, Locator, Page } from '@playwright/test'; +import { Locator, Page } from '@playwright/test'; export class FooPage { - readonly errorMessage: Locator; + #page: Page; - constructor(page: Page) { - this.page = page; - this.errorMessage = page.locator('.error-message'); + constructor({ page }: { page: Page }) { + this.#page = page; } } -// test file './steps/foo.ts' -import { FooPage } from './pageObjects/foo'; +// step definition file './steps/ui/foo.ts' +import { expect } from '@playwright/test'; +import { Then } from '../../environment/fixtures'; +import { FooPage } from '../../support/objects/foo'; -let fooPage: FooPage; - -Then('error message should be visible', async function ({ page }) { - const fooPage = new FooPage({ page }); +Then('the error message should be visible', async ({ page }) => { + const fooPage = new FooPage(page); await expect(fooPage.errorMessage).toBeVisible(); }); ``` @@ -113,11 +112,12 @@ Then('error message should be visible', async function ({ page }) { DO NOT ⚔️ ```typescript -// test file './steps/foo.ts' -// include locators directly in test -import { Locator, Page } from '@playwright/test'; +// step definition file './steps/ui/foo.ts' +// include locators directly in the step +import { expect } from '@playwright/test'; +import { Then } from '../../environment/fixtures'; -Then('error message should be visible', async function ({ page }) { +Then('the error message should be visible', async ({ page }) => { await expect(page.locator('.error-message')).toBeVisible(); }); ``` diff --git a/docs/dev/web/testing/running-tests.md b/docs/dev/web/testing/running-tests.md deleted file mode 100644 index ce557c9f..00000000 --- a/docs/dev/web/testing/running-tests.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -title: 'Running tests' -sidebar_position: 1 -id: running-tests ---- - -## Introduction - -In order to allow us to make changes quickly, often and with a high level of confidence, we heavily rely on tests within the `web` repository. - -This section assumes you have the Web development stack up and running. Please check out the [development setup docs](../development/tooling#development-setup) if you haven't. - -### Unit Tests - -We have a steadily growing coverage of unit tests. You can run them locally via - -```bash -pnpm test:unit -``` - -You can also specify which tests to run by giving a path param, like so: `pnpm test:unit packages//tests/unit/path/to/test.spec.ts`. - -#### Unit Test File Structure - -Our unit tests spec files follow a simple structure: - -- fixtures and mocks at the top -- helper functions at the bottom -- tests in between - -We usually organize tests with nested `describe` blocks. If you would like to get feedback from the core team about -the structure, scope and goals of your unit tests before actually writing some, we invite you to make a pull request -with only `describe` blocks and nested `it.todo("put your test description here")` lines. - -### E2E Tests (Playwright) - -Our end-to-end test suite is built upon the [Playwright Framework](https://github.com/microsoft/playwright), -which makes it easy to write tests, debug them and have them run cross-browser with minimal overhead. - -#### Bundling Web - -Make sure the Web frontend has been bundled with the following command since the dev server won't work: - -```bash -pnpm build:w -``` - -#### Run E2E Tests - -The following command will run all available e2e tests: - -```bash -pnpm test:e2e:cucumber 'tests/e2e/cucumber/**/*.feature' -``` - -#### Options - -To run a particular test, simply add the feature file and line number to the test command, e.g. `pnpm test:e2e:cucumber tests/e2e/cucumber/features/smoke/admin-settings/users.feature:84` - -Various options are available via ENV variables, e.g. - -- `OC_BASE_URL` # use your OpenCloud URL. Default value: host.docker.internal:9200 -- `BASIC_AUTH=true` use basic authorization for api requests. -- `RETRY=n` to retry failures `n` times -- `SLOW_MO=n` to slow the execution time by `n` milliseconds -- `TIMEOUT=n` to set tests to timeout after `n` milliseconds -- `HEADLESS=bool` to open the browser while the tests run (defaults to true => headless mode) -- `BROWSER=name` to run tests against a specific browser. Defaults to chromium, available are chromium, firefox, webkit, chromium -- `ADMIN_PASSWORD` to set administrator password. By default, the `admin` password is used in the test -- `PARALLEL` for parallel test execution - -For debugging reasons, you may want to record a video or traces of your test run. -Again, you can use the following ENV variables in your command: - -- `REPORT_DIR=another/path` to set a directory for your recorded files (defaults to "reports") -- `REPORT_VIDEO=true` to record a video of the test run -- `REPORT_HAR=true` to save request information from the test run -- `REPORT_TRACING=true` to record traces from the test run - -To then open e.g. the tracing from the `REPORT_DIR`, run - -```bash -npx playwright show-trace path/to/file.zip -``` - -#### Lint E2E Test Code - -Run the following command to find out the lint issues early in the test codes: - -```bash -pnpm lint -``` - -And to fix the lint problems run the following command: - -```bash -pnpm lint --fix -``` - -If the lint problems are not fixed by `--fix` option, we have to manually fix the code. - -### Analyze the Test Report - -The cucumber library is used as the test runner for e2e tests. The report generator script lives inside the `tests/e2e/cucumber/report` folder. If you want to create a report after the tests are done, run the command: - -```bash -node tests/e2e/cucumber/report --report-input=tests/e2e/cucumber/report/report.json -``` - -By default, the report gets generated to reports/e2e/cucumber/releaseReport/cucumber_report.html. -The location can be changed by adding the `--report-location` flag. - -To see all available options run - -```bash -node tests/e2e/cucumber/report --help -``` - -### E2E Tests on OpenCloud With Keycloak - -We can run some of the e2e tests on OpenCloud setup with Keycloak as an external idp. To run tests against locally, please follow the steps below: - -#### Run OpenCloud With Keycloak - -There's a documentation to serve [OpenCloud with Keycloak](../../../admin/configuration/authentication-and-user-management/keycloak.md). Please follow each step to run **OpenCloud with Keycloak**. - -#### Options - -Following environment variables come in use while running e2e tests on OpenCloud with Keycloak: - -- `KEYCLOAK=true` runs the tests with Keycloak. Default value: false -- `KEYCLOAK_HOST=your_keycloak_host` sets Keycloak url. Default value: keycloak.opencloud.test -- `KEYCLOAK_ADMIN_USER=admin` keycloak admin username -- `KEYCLOAK_ADMIN_PASSWORD=password` keycloak admin password -- `KEYCLOAK_REALM=your_realm` keycloak realm name. Default value: openCloud - -#### Run E2E Tests - -```bash -KEYCLOAK=true \ -KEYCLOAK_HOST=keycloak.opencloud.test \ -KEYCLOAK_ADMIN_USER=kcadmin \ -KEYCLOAK_ADMIN_PASSWORD=admin \ -OC_BASE_URL=cloud.opencloud.test \ -pnpm run test:e2e:cucumber tests/e2e/cucumber/features/keycloak -```