EDU-18161: make documentation page sampling deterministic

.github/workflows/cypress-extensive.yml

@@ -6,19 +6,18 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v1
-      - name: Fetch navigation.json
-        uses: wei/curl@v1
-        with:
-          args: -o src/tests/cypress/fixtures/navigation.json https://developers.vtex.com/navigation.json
+      # Keep extensive runs on the branch's checked-in public/navigation.json so
+      # manual reruns stay reproducible for the same workflow invocation.
       - name: Cypress run
         uses: cypress-io/github-action@v5
         env:
           CYPRESS_baseUrl: https://developers.vtex.com
           CYPRESS_testProbability: 1.0
+          SAMPLE_SEED: ${{ github.run_id }}
       - name: Summary report
         if: always()
         run: |
-          if [[ -f "cypress.log" ]]; then
+          if [[ -f "cypress-failures.jsonl" ]]; then
             node src/tests/utils/summary-report.js > $GITHUB_STEP_SUMMARY
-            rm cypress.log
-          fi
+          fi
+          rm -f cypress-failures.jsonl cypress-sample.json

.github/workflows/cypress.yml

@@ -13,22 +13,33 @@ jobs:
           cache: 'yarn'
       - name: Install dependencies
         run: yarn install --frozen-lockfile
-      
+
       # Run component tests first since they don't need the preview deployment
       - name: Cypress component tests
+        id: cypress-component
+        timeout-minutes: 10
         uses: cypress-io/github-action@v5
         with:
           browser: chrome
           component: true
-
-      # Ensure component test results are included in the log file
+
+      - name: Cypress component tests (retry)
+        id: cypress-component-retry
+        if: steps.cypress-component.outcome == 'failure'
+        timeout-minutes: 10
+        uses: cypress-io/github-action@v5
+        with:
+          browser: chrome
+          component: true
+
+      # Preserve component test failures before integration tests overwrite the file
       - name: Preserve component test results
         if: always()
         run: |
-          if [[ -f "./cypress.log" ]]; then
-            cp cypress.log component-tests.log
+          if [[ -f "./cypress-failures.jsonl" ]]; then
+            cp cypress-failures.jsonl component-failures.jsonl
           fi
-      
+
       # Then run integration tests after preview is ready
       - name: Waiting for pages to change
         uses: fountainhead/action-wait-for-check@v1.1.0
@@ -38,7 +49,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
           checkName: 'Pages changed - elated-hoover-5c29bf'
           timeoutSeconds: 3600
-      
+
       - name: Waiting for 200 from the Netlify Preview
         if: steps.wait-for-pages-changed.outputs.conclusion == 'neutral'
         uses: jakepartusch/wait-for-netlify-action@v1.4
@@ -48,36 +59,40 @@ jobs:
           max_timeout: 3600
         env:
           NETLIFY_TOKEN: ${{ secrets.NETLIFY_TOKEN }}
-
-      - name: Fetch navigation.json
-        uses: wei/curl@v1
-        with:
-          args: -o src/tests/cypress/fixtures/navigation.json https://developers.vtex.com/navigation.json
-
+
+      # Sample pages from the branch's checked-in public/navigation.json so the
+      # PR comment reflects the same navigation data shipped by this preview.
       - name: Cypress integration tests
         uses: cypress-io/github-action@v5
+        timeout-minutes: 60
         with:
           browser: chrome
           headed: true
         env:
           CYPRESS_baseUrl: ${{ steps.wait-for-netflify-preview.outputs.url }}
           CYPRESS_testProbability: 0.1
-
+          SAMPLE_SEED: ${{ github.event.pull_request.head.sha || github.sha }}
+
       # Merge component and integration test results
       - name: Generate summary report
         if: always()
         run: |
           touch src/tests/summary.md
-          # Merge component test results if they exist
-          if [[ -f "./component-tests.log" ]]; then
-            cat component-tests.log >> cypress.log
+          # Merge component test failures if they exist
+          if [[ -f "./component-failures.jsonl" ]]; then
+            cat component-failures.jsonl >> cypress-failures.jsonl
           fi
-          if [[ -f "./cypress.log" ]]; then
+          # Only run the report if there are recorded failures OR the component
+          # tests actually completed (success means no failures, not a crash).
+          # Without this guard, a Cypress crash produces no JSONL file and the
+          # report would incorrectly say "All tests were successful!".
+          if [[ -f "./cypress-failures.jsonl" ]] || [[ "${{ steps.cypress-component.outcome }}" == "success" ]] || [[ "${{ steps.cypress-component-retry.outcome }}" == "success" ]]; then
             node src/tests/utils/summary-report.js > src/tests/summary.md
-            rm cypress.log
-            rm -f component-tests.log
+          else
+            printf "# End-to-end tests\n\nCypress failed to run — no test results available.\n" > src/tests/summary.md
           fi
-
+          rm -f cypress-failures.jsonl component-failures.jsonl cypress-sample.json
+
       - name: Comment PR with summary report
         if: always()
         uses: thollander/actions-comment-pull-request@v2

.gitignore

@@ -13,6 +13,9 @@ sitemap.xml
 sitemap-0.xml
 robots.txt
 
+# Local configs
+.claude/settings.local.json
+
 # Local Scraping
 /docsearch-scraper
 
@@ -56,3 +59,5 @@ cypress.log
 src/tests/cypress/screenshots
 src/tests/cypress/videos
 RapiDoc
+
+plans/

cypress.config.ts

@@ -16,6 +16,9 @@ export default defineConfig({
   chromeWebSecurity: false,
   numTestsKeptInMemory: 10,
   experimentalMemoryManagement: true,
+  env: {
+    sampleSeed: process.env.SAMPLE_SEED,
+  },
   component: {
     devServer: {
       framework: 'next',

src/tests/cypress/component/DropdownMenu.cy.tsx

@@ -10,15 +10,14 @@ import {
 import { writeLog } from '../support/functions'
 
 describe('DropdownMenu Component', () => {
-  before(() => {
-    cy.writeFile('cypress.log', `#Component Tests - DropdownMenu#\n`, {
-      flag: 'a+',
-    })
-  })
-
   afterEach(function () {
     if (this.currentTest.state === 'failed') {
-      writeLog(this.currentTest.title)
+      writeLog({
+        spec: Cypress.spec.name,
+        title: this.currentTest.title,
+        attempt: this.currentTest.currentRetry(),
+        type: 'dom',
+      })
     }
   })
 

src/tests/cypress/component/markdown-renderer/index.cy.tsx

@@ -55,15 +55,14 @@ const TestMarkdownRenderer = ({
 }
 
 describe('MarkdownRenderer Component', () => {
-  before(() => {
-    cy.writeFile('cypress.log', `#Component Tests - MarkdownRenderer#\n`, {
-      flag: 'a+',
-    })
-  })
-
   afterEach(function () {
     if (this.currentTest.state === 'failed') {
-      writeLog(this.currentTest.title)
+      writeLog({
+        spec: Cypress.spec.name,
+        title: this.currentTest.title,
+        attempt: this.currentTest.currentRetry(),
+        type: 'dom',
+      })
     }
   })
 

src/tests/cypress/integration/api-guides.cy.js

@@ -6,9 +6,6 @@ const messages = getMessages()
 describe('API guides documentation page', () => {
   before(() => {
     cy.task('setUrl', '/docs/guides')
-    cy.writeFile('cypress.log', `#API guides documentation page#\n`, {
-      flag: 'a+',
-    })
   })
 
   beforeEach(() => {
@@ -20,7 +17,13 @@ describe('API guides documentation page', () => {
   afterEach(function () {
     if (this.currentTest.state === 'failed') {
       cy.task('getUrl').then((url) => {
-        writeLog(`${this.currentTest.title} (${url})`)
+        writeLog({
+          spec: Cypress.spec.name,
+          title: this.currentTest.title,
+          attempt: this.currentTest.currentRetry(),
+          type: 'dom',
+          message: url,
+        })
       })
     }
   })

src/tests/cypress/integration/api-reference.cy.js

@@ -5,9 +5,6 @@ import { writeLog } from '../support/functions'
 describe('API reference documentation page', () => {
   before(() => {
     cy.task('setUrl', '/docs/api-reference')
-    cy.writeFile('cypress.log', `#API reference documentation page#\n`, {
-      flag: 'a+',
-    })
   })
 
   beforeEach(() => {
@@ -19,7 +16,13 @@ describe('API reference documentation page', () => {
   afterEach(function () {
     if (this.currentTest.state === 'failed') {
       cy.task('getUrl').then((url) => {
-        writeLog(`${this.currentTest.title} (${url})`)
+        writeLog({
+          spec: Cypress.spec.name,
+          title: this.currentTest.title,
+          attempt: this.currentTest.currentRetry(),
+          type: 'dom',
+          message: url,
+        })
       })
     }
   })

src/tests/cypress/integration/documentation-pages-status.cy.js

@@ -1,13 +1,58 @@
 /// <reference types="cypress" />
 
 import { writeLog } from '../support/functions'
-import { selectRandomPages } from '../../utils/select-pages.js'
+import { getPageSample, NAVIGATION_SOURCE } from '../../utils/select-pages.js'
+
+const MAX_INFRA_REQUEST_RETRIES = 3
+
+const { pages, seed, seedLabel } = getPageSample({
+  prob: Cypress.env('testProbability') || 1.0,
+  seed: Cypress.env('sampleSeed'),
+})
+
+const isRetryableInfraStatus = (status) =>
+  status >= 500 || status === 408 || status === 429
+
+const requestPage = (page, attempt = 0) =>
+  cy
+    .request({
+      failOnStatusCode: false,
+      retryOnNetworkFailure: true,
+      url: Cypress.config().baseUrl + page,
+    })
+    .then((response) => {
+      if (
+        isRetryableInfraStatus(response.status) &&
+        attempt < MAX_INFRA_REQUEST_RETRIES
+      ) {
+        cy.log(
+          `Retrying infrastructure failure for ${page} (${response.status})`
+        )
+
+        return requestPage(page, attempt + 1)
+      }
+
+      return response
+    })
+
+// Tracks the failure classification for the current test, consumed by afterEach.
+let failureType = 'dom'
+let failureMessage = ''
 
 describe('Status of documentation pages', () => {
   before(() => {
-    cy.writeFile('cypress.log', `#Status of documentation pages#\n`, {
-      flag: 'a+',
-    })
+    cy.writeFile(
+      'cypress-sample.json',
+      {
+        seed,
+        seedLabel,
+        navigationSource: NAVIGATION_SOURCE,
+        pages,
+      },
+      {
+        log: false,
+      }
+    )
 
     // Handle React hydration errors and other expected errors
     Cypress.on('uncaught:exception', (err) => {
@@ -22,44 +67,51 @@ describe('Status of documentation pages', () => {
     })
   })
 
+  beforeEach(() => {
+    failureType = 'dom'
+    failureMessage = ''
+  })
+
   afterEach(function () {
     if (this.currentTest.state === 'failed') {
-      writeLog(this.currentTest.title)
+      writeLog({
+        spec: Cypress.spec.name,
+        title: this.currentTest.title,
+        attempt: this.currentTest.currentRetry(),
+        type: failureType,
+        message: failureMessage,
+      })
     }
   })
 
-  const pages = selectRandomPages({
-    prob: Cypress.env('testProbability') || 1.0,
-  })
-
   pages.forEach((page) =>
-    it(
-      `Checks page ${page}`,
-      {
-        retries: {
-          runMode: 3,
-          openMode: 3,
-        },
-      },
-      () => {
-        // Handle PDF content-type gracefully
-        cy.request(Cypress.config().baseUrl + page).then((response) => {
-          // If it's a PDF, consider it a valid response and skip further checks
-          if (response.headers['content-type']?.includes('application/pdf')) {
-            return
-          }
+    it(`Checks page ${page}`, () => {
+      requestPage(page).then((response) => {
+        if (response.status < 200 || response.status > 399) {
+          failureType = 'http'
+          failureMessage = `HTTP ${response.status}`
+          throw new Error(`HTTP ${response.status} for ${page}`)
+        }
 
-          // For HTML content, proceed with page navigation and checks
-          cy.visit(page)
-          // Wait for page to be fully hydrated
-          cy.wait(2000) // Give time for initial hydration
-          cy.get('body').should('be.visible')
-          // Wait for any dynamic content to load
-          cy.get('[data-cy="sidebar-section"]', { timeout: 10000 }).should(
-            'exist'
-          )
+        // If it's a PDF, consider it a valid response and skip further checks
+        if (response.headers['content-type']?.includes('application/pdf')) {
+          return
+        }
+
+        // Set type to load_timeout before cy.visit; reset to dom once visit
+        // succeeds so that a subsequent sidebar failure is correctly classified.
+        failureType = 'load_timeout'
+        cy.visit(page, {
+          retryOnNetworkFailure: true,
+          retryOnStatusCodeFailure: true,
         })
-      }
-    )
+        cy.then(() => {
+          failureType = 'dom'
+        })
+        cy.get('[data-cy="sidebar-section"]', { timeout: 10000 }).should(
+          'be.visible'
+        )
+      })
+    })
   )
 })