Skip to content
Open
173 changes: 173 additions & 0 deletions __tests__/helpers/validate-python-output.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""
Validates a generated Python pytest zip produced by PythonAppiumScriptGenerator.

Usage: python3 validate-python-output.py <path-to-zip>
Exit 0 = all checks passed. Exit 1 = at least one check failed (details on stdout).

Checks performed:
1. ast.parse() succeeds on every .py file (catches ALL syntax/indentation errors)
2. test_app.py imports AppiumBy and By (locator code needs both at module scope)
3. config.py uses Python True/False (not JS true/false)
4. config.py has @classmethod at 4-space indent (inside class body, not at col 0)
5. test_suite.py: finally at 4-space indent (peer of try/except, not nested inside except)
6. test_suite.py: every def test_* at module level (no leading spaces, drift check)
"""

import ast
import re
import sys
import zipfile


def validate(zip_path):
errors = []

with zipfile.ZipFile(zip_path) as zf:
py_files = {
name: zf.read(name).decode('utf-8')
for name in zf.namelist()
if name.endswith('.py')
}

if not py_files:
errors.append('zip contains no .py files')
return errors

# ------------------------------------------------------------------
# 1. Syntax check — ast.parse catches IndentationError, SyntaxError,
# wrong boolean literals, and broken string literals.
# ------------------------------------------------------------------
for name, src in sorted(py_files.items()):
try:
ast.parse(src)
except SyntaxError as exc:
errors.append(f'{name}: SyntaxError line {exc.lineno}: {exc.msg}')

# ------------------------------------------------------------------
# 2. test_app.py must have AppiumBy and By imported at module scope.
# Generated locator code uses these names; without the imports the
# script raises NameError at runtime before any test runs.
# ------------------------------------------------------------------
test_app = py_files.get('test_app.py', '')
if 'from appium.webdriver.common.appiumby import AppiumBy' not in test_app:
errors.append(
'test_app.py: missing import '
'"from appium.webdriver.common.appiumby import AppiumBy"'
)
if 'from selenium.webdriver.common.by import By' not in test_app:
errors.append(
'test_app.py: missing import '
'"from selenium.webdriver.common.by import By"'
)

# ------------------------------------------------------------------
# 2b. test_base.py must import AppiumOptions from its real module path.
# In Appium-Python-Client 3.x/4.x, AppiumOptions lives at
# appium.options.common.base, NOT appium.options — the shorter
# path raises ImportError at collection time.
# ------------------------------------------------------------------
test_base = py_files.get('test_base.py', '')
if 'from appium.options.common.base import AppiumOptions' not in test_base:
errors.append(
'test_base.py: missing or wrong AppiumOptions import — '
'must be "from appium.options.common.base import AppiumOptions"'
Comment on lines +65 to +74
Copy link

Copilot AI Apr 30, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This check still requires test_base.py to import AppiumOptions from appium.options.common.base, but the template now imports UiAutomator2Options. As written, the integration test will fail even when generation is correct. Update the assertion to match the intended options import(s) (and consider covering both Android and iOS options if supported).

Suggested change
# 2b. test_base.py must import AppiumOptions from its real module path.
# In Appium-Python-Client 3.x/4.x, AppiumOptions lives at
# appium.options.common.base, NOT appium.options — the shorter
# path raises ImportError at collection time.
# ------------------------------------------------------------------
test_base = py_files.get('test_base.py', '')
if 'from appium.options.common.base import AppiumOptions' not in test_base:
errors.append(
'test_base.py: missing or wrong AppiumOptions import — '
'must be "from appium.options.common.base import AppiumOptions"'
# 2b. test_base.py must import a supported Appium options class.
# Current templates use platform-specific options such as
# UiAutomator2Options (Android) and XCUITestOptions (iOS).
# Keep AppiumOptions accepted as a legacy fallback so validation
# does not fail for otherwise-correct generated output.
# ------------------------------------------------------------------
test_base = py_files.get('test_base.py', '')
valid_options_imports = (
'from appium.options.android import UiAutomator2Options',
'from appium.options.ios import XCUITestOptions',
'from appium.options.common.base import AppiumOptions',
)
if not any(import_line in test_base for import_line in valid_options_imports):
errors.append(
'test_base.py: missing or wrong Appium options import — '
'must import one of '
'"from appium.options.android import UiAutomator2Options", '
'"from appium.options.ios import XCUITestOptions", or '
'"from appium.options.common.base import AppiumOptions"'

Copilot uses AI. Check for mistakes.
)

# ------------------------------------------------------------------
# 2c. test_base.py find_online_device must accept the newer
# /v1/devices response shape. test-green returns
# privateDevices/favoriteDevices/cloudDevices/etc, NOT
# deviceListData — checking only the legacy key produces
# false-negative "device not available" retries every run.
# ------------------------------------------------------------------
if "'privateDevices'" not in test_base and 'privateDevices' not in test_base:
errors.append(
'test_base.py: find_online_device does not recognize the newer '
'Kobiton /v1/devices response shape (privateDevices/cloudDevices/...). '
'Must union all device category keys, not only deviceListData'
)

# ------------------------------------------------------------------
# 2d. proxy_server.py must strip the client Host header before
# forwarding to Kobiton. Without this, the upstream sees
# Host: localhost:<port> and responds 404 to every request.
# ------------------------------------------------------------------
proxy = py_files.get('proxy_server.py', '')
if "'host'" not in proxy.lower():
errors.append(
'proxy_server.py: does not strip the Host header before forwarding. '
'Upstream Kobiton routes by Host and returns 404 for localhost. '
'Must filter hop-by-hop/routing headers (including Host) from the forwarded request'
)

# ------------------------------------------------------------------
# 3. config.py: generated boolean capabilities must use Python
# True/False, not JavaScript true/false.
# Pattern: a dict-value position — '...': true or '...': false
# ------------------------------------------------------------------
config = py_files.get('config.py', '')
js_bool_re = re.compile(r"^\s+'[^']+': (true|false)[,\s]*$")
for lineno, line in enumerate(config.splitlines(), 1):
if js_bool_re.match(line):
errors.append(
f'config.py line {lineno}: JS boolean literal '
f'(must be True/False): {line.rstrip()}'
)

# ------------------------------------------------------------------
# 4. config.py: @classmethod must be indented exactly 4 spaces.
# When the first-line absorber was missing, @classmethod landed
# at column 0, placing it outside the class body.
# ------------------------------------------------------------------
if ' @classmethod\n' not in config and not config.endswith(' @classmethod'):
errors.append(
'config.py: no @classmethod found at 4-space indent — '
'generated methods are outside the class body'
)

# ------------------------------------------------------------------
# 5. test_suite.py: finally must be at 4-space indent (same level
# as try/except). The original bug nested it at 8 spaces inside
# the except block, causing an immediate SyntaxError.
# ------------------------------------------------------------------
test_suite = py_files.get('test_suite.py', '')
for lineno, line in enumerate(test_suite.splitlines(), 1):
if line.startswith(' finally:'):
errors.append(
f'test_suite.py line {lineno}: "finally:" is at 8-space indent '
f'(nested inside except block — must be at 4-space, peer of try/except)'
)

# ------------------------------------------------------------------
# 6. test_suite.py: every def test_* must start at column 0.
# When per-device desiredCaps methods leaked +1 indent per device,
# the second (and later) test functions shifted right and became
# nested, making pytest unable to discover them.
# ------------------------------------------------------------------
for lineno, line in enumerate(test_suite.splitlines(), 1):
stripped = line.lstrip()
if stripped.startswith('def test_') and line != stripped:
errors.append(
f'test_suite.py line {lineno}: test function not at module level '
f'(leading whitespace = {len(line) - len(stripped)} spaces): '
f'{line.rstrip()}'
)

return errors


if __name__ == '__main__':
if len(sys.argv) != 2:
print('Usage: validate-python-output.py <zip_path>', file=sys.stderr)
sys.exit(2)

found_errors = validate(sys.argv[1])

if found_errors:
for err in found_errors:
print(f'FAIL: {err}')
sys.exit(1)

print(f'OK: all checks passed ({sys.argv[1]})')
sys.exit(0)
37 changes: 37 additions & 0 deletions __tests__/integration/python-pytest.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import path from 'path'
import fs from 'fs'
import os from 'os'
import {execFileSync} from 'child_process'
import PythonAppiumScriptGenerator from '../../src/services/python'
import {removeDir} from '../../src/utils/fs-wrapper'

const INPUT_FILE = path.resolve(__dirname, '../resource/python-pytest-input.json')
const VALIDATOR = path.resolve(__dirname, '../helpers/validate-python-output.py')

describe('PythonAppiumScriptGenerator', () => {
it('generates a valid Python pytest script from a 2-device session fixture', async () => {
const input = JSON.parse(fs.readFileSync(INPUT_FILE, 'utf-8'))
const workingDir = path.join(os.tmpdir(), `python-pytest-test-${Date.now()}`)

const generator = new PythonAppiumScriptGenerator({})
const {outputFile} = await generator.run({...input, workingDir})

expect(outputFile).toBeTruthy()
expect(fs.existsSync(outputFile)).toBe(true)

try {
execFileSync('python3', [VALIDATOR, outputFile], {
encoding: 'utf8',
stdio: 'pipe'
})
}
catch (err) {
throw new Error(
`Python validation failed:\n${(err.stdout || '').trim()}\n${(err.stderr || '').trim()}`
)
}
finally {
await removeDir(workingDir)
}
}, 30000)
})
180 changes: 180 additions & 0 deletions __tests__/resource/python-pytest-input.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
{
"sentAt": {
"low": 1105187238,
"high": 395,
"unsigned": true
},
"serverInfo": {
"apiUrl": "https://api.kobiton.com",
"portalUrl": "https://portal.kobiton.com",
"username": "kobitonadmin+appiumScriptGen",
"apiKey": "your_api_key"
},
"isManualSession": true,
"manualSessionId": "605",
"devices": [
{
"id": 1881,
"name": "A5 2020",
"capabilities": {
"platformName": "Android",
"platformVersion": "9",
"resolution": {
"width": 720,
"height": 1600,
"scale": 1
}
}
},
{
"id": 1882,
"name": "Galaxy S22",
"capabilities": {
"platformName": "Android",
"platformVersion": "16",
"resolution": {
"width": 1080,
"height": 2340,
"scale": 1
}
}
}
],
"testSteps": [
{
"id": "17000",
"context": "NATIVE",
"selectorConfigurations": [
{
"device": {
"deviceName": "A5 2020",
"platformVersion": "9"
},
"selectors": [
{
"type": "xpath",
"value": "//*[@resource-id='com.android.chrome:id/url_bar']"
}
]
},
{
"selectors": [
{
"type": "xpath",
"value": "//android.widget.EditText[@resource-id='com.android.chrome:id/url_bar']"
}
]
}
],
"actionJson": "{\"command\":\"touchOnElement\",\"x\":\"0.387\",\"y\":\"0.430\"}",
"findingElementTimeout": 7129,
"isOnKeyboard": false
},
{
"id": "17001",
"context": "NATIVE",
"actionJson": "{\"command\":\"sendKeys\",\"value\":\"kobiton \"}",
"findingElementTimeout": 4014,
"isOnKeyboard": false
},
{
"id": "17002",
"context": "NATIVE",
"actionJson": "{\"command\":\"press\",\"value\":\"ENTER\"}",
"findingElementTimeout": 1612,
"isOnKeyboard": false
},
{
"id": "17003",
"context": "WEB",
"selectorConfigurations": [
{
"selectors": [
{
"type": "css",
"value": "a[href='https://kobiton.com/']"
}
]
}
],
"actionJson": "{\"command\":\"touchOnElement\",\"x\":\"0.359\",\"y\":\"0.300\"}",
"findingElementTimeout": 8965,
"isOnKeyboard": false
},
{
"id": "17004",
"context": "NATIVE",
"actionJson": "{\"command\":\"rotate\",\"orientation\":\"LANDSCAPE\"}",
"findingElementTimeout": 14108,
"isOnKeyboard": false
},
{
"id": "17005",
"context": "NATIVE",
"selectorConfigurations": [
{
"selectors": [
{
"type": "xpath",
"value": "(//android.webkit.WebView)[1]"
}
]
}
],
"actionJson": "{\"command\":\"swipeFromElement\",\"x1\":\"0.506\",\"y1\":\"0.815\",\"x2\":\"0.574\",\"y2\":\"0.251\",\"duration\":637}",
"findingElementTimeout": 13081,
"isOnKeyboard": false
},
{
"id": "17006",
"context": "NATIVE",
"actionJson": "{\"command\":\"press\",\"value\":\"BACK\"}",
"findingElementTimeout": 6526,
"isOnKeyboard": false
}
],
"appUnderTest": {
"id": "com.android.chrome"
},
"desiredCapabilitiesOfDevices": [
{
"deviceId": 1881,
"desiredCapabilities": [
{"key": "sessionName", "value": "Automation on A5 2020", "type": "string"},
{"key": "sessionDescription", "value": "", "type": "string"},
{"key": "deviceOrientation", "value": "portrait", "type": "string"},
{"key": "noReset", "value": "false", "type": "bool"},
{"key": "fullReset", "value": "true", "type": "bool"},
{"key": "captureScreenshots", "value": "true", "type": "bool"},
{"key": "newCommandTimeout", "value": "900", "type": "int"},
{"key": "keepScreenOn", "value": "true", "type": "bool"},
{"key": "deviceGroup", "value": "KOBITON", "type": "string"},
{"key": "deviceName", "value": "A5 2020", "type": "string"},
{"key": "platformVersion", "value": "9", "type": "string"},
{"key": "platformName", "value": "Android", "type": "string"}
]
},
{
"deviceId": 1882,
"desiredCapabilities": [
{"key": "sessionName", "value": "Automation on Galaxy S22", "type": "string"},
{"key": "sessionDescription", "value": "", "type": "string"},
{"key": "deviceOrientation", "value": "portrait", "type": "string"},
{"key": "noReset", "value": "false", "type": "bool"},
{"key": "fullReset", "value": "false", "type": "bool"},
{"key": "captureScreenshots", "value": "true", "type": "bool"},
{"key": "newCommandTimeout", "value": "900", "type": "int"},
{"key": "keepScreenOn", "value": "true", "type": "bool"},
{"key": "deviceGroup", "value": "KOBITON", "type": "string"},
{"key": "deviceName", "value": "Galaxy S22", "type": "string"},
{"key": "platformVersion", "value": "16", "type": "string"},
{"key": "platformName", "value": "Android", "type": "string"}
]
}
],
"requestScript": {
"name": "kobiton-appium-script-s605-python-pytest",
"language": "python",
"testingFramework": "pytest"
}
}
Loading