diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 1e747a61..de3e7604 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -72,4 +72,5 @@ jobs: - uses: actions/setup-go@v5 with: go-version: "${{ env.GOVERSION }}" + cache-dependency-path: go/sdl/go.sum - run: make test-sdl-parity diff --git a/docs/sdl-parity.md b/docs/sdl-parity.md new file mode 100644 index 00000000..00c8a058 --- /dev/null +++ b/docs/sdl-parity.md @@ -0,0 +1,66 @@ +# ADR: SDL Parity Between Go and TypeScript + +## Context + +Akash Network maintains two independent SDL (Stack Definition Language) parsers: + +- **Go** (`go/sdl/`) — used by the blockchain node and provider software +- **TypeScript** (`ts/src/sdl/`) — used by Console and other web-based tools + +Both must produce identical outputs from the same SDL input. Without enforced parity, the two implementations can silently diverge, causing deployment failures or inconsistent behavior between frontend and backend. + +## Decision + +### 1. Single Schema as Source of Truth + +The file `go/sdl/sdl-input.schema.yaml` is the **sole canonical definition** of valid SDL input for both languages. + +- All input validation rules (types, enums, patterns, constraints) live in this schema +- The Go parser embeds this schema via `//go:embed` +- The TS validator is generated from this schema via `compile-json-schema-to-ts.ts` +- No hand-maintained validation logic should duplicate what the schema already enforces + +### 2. Shared Test Fixtures + +Test fixtures under `testdata/sdl/` are the parity harness: + +``` +testdata/sdl/ +├── input/ # SDL input YAML files +│ ├── v2.0/ # Valid v2.0 fixtures (each dir: input.yaml) +│ ├── v2.1/ # Valid v2.1 fixtures +│ ├── invalid/ # Rejected by both schema and parsers +│ ├── schema-only-invalid/ # Schema rejects, Go parser accepts +│ └── semantic-only-invalid/ # Schema accepts, parser rejects +└── output-fixtures/ # Expected outputs (generated from Go parser) + ├── v2.0/ # manifest.json + group-specs.json per fixture + └── v2.1/ +``` + +- **Inputs** are written by hand to cover SDL features and edge cases +- **Outputs** are generated by the Go parser (`make generate-sdl-fixtures`) and committed +- Both Go and TS parity tests compare their generated output against these committed fixtures + +### 3. Parity Test Requirements + +Any SDL-related PR **must**: + +1. Pass `make test-sdl-parity` (runs both Go and TS parity tests) +2. Update fixtures if parser output changes (`make generate-sdl-fixtures`) +3. Add new fixtures for new SDL features or edge cases +4. Keep the TS-generated validator in sync with the schema (CI drift check) + +### 4. Strict vs Lenient Parsing + +- `ReadFile` / `Read` (Go) — lenient, does not enforce schema (backward compatible) +- `ReadFileStrict` / `ReadStrict` (Go) — strict, validates schema first, then semantic rules +- `generateManifest` (TS) — always validates schema (via `validateSDL`) + +New code should prefer the strict APIs. The lenient APIs exist for backward compatibility with existing consumers. + +## Consequences + +- Schema changes require updating both Go and TS test suites +- TS validator drift is caught by CI +- Parity regressions are caught before merge +- Edge cases are documented as test fixtures, not just code comments diff --git a/go/sdl/parity_test.go b/go/sdl/parity_test.go index aaa9761f..b4e760d7 100644 --- a/go/sdl/parity_test.go +++ b/go/sdl/parity_test.go @@ -27,26 +27,87 @@ package sdl import ( "encoding/json" + "fmt" "os" "path/filepath" "testing" "github.com/stretchr/testify/require" + "github.com/xeipuuv/gojsonschema" + "gopkg.in/yaml.v3" ) -const fixturesInputRoot = "../../testdata/sdl/input" -const fixturesOutputRoot = "../../testdata/sdl/output-fixtures" +type parityTestSuite struct { + fixturesInputRoot string + fixturesOutputRoot string + schemasRoot string + manifestSchema *gojsonschema.Schema + groupsSchema *gojsonschema.Schema +} + +// newParityTestSuite initializes a test suite with compiled output schemas. +func newParityTestSuite(t *testing.T) *parityTestSuite { + t.Helper() + + s := &parityTestSuite{ + fixturesInputRoot: "../../testdata/sdl/input", + fixturesOutputRoot: "../../testdata/sdl/output-fixtures", + schemasRoot: "../../specs/sdl", + } + + var err error + s.manifestSchema, err = compileSchemaFromPath(filepath.Join(s.schemasRoot, "manifest-output.schema.yaml")) + require.NoError(t, err, "Failed to compile manifest output schema") + + s.groupsSchema, err = compileSchemaFromPath(filepath.Join(s.schemasRoot, "groups-output.schema.yaml")) + require.NoError(t, err, "Failed to compile groups output schema") + + return s +} + +// validateInputSchema validates raw YAML bytes against the embedded SDL input schema. +func (s *parityTestSuite) validateInputSchema(t *testing.T, inputBytes []byte) { + t.Helper() + err := validateInputAgainstSchema(inputBytes) + require.NoError(t, err, "Input schema validation failed") +} + +// validateOutputAgainstSchema validates manifest and group-specs JSON against output schemas. +func (s *parityTestSuite) validateOutputAgainstSchema(t *testing.T, manifestBytes []byte, groupSpecsBytes []byte) { + t.Helper() + + err := validateDataAgainstCompiledSchema(manifestBytes, s.manifestSchema) + require.NoError(t, err, "Manifest schema validation failed") + + err = validateDataAgainstCompiledSchema(groupSpecsBytes, s.groupsSchema) + require.NoError(t, err, "Groups schema validation failed") +} + +// validateFixtureBytes compares generated JSON output against a committed fixture file. +func (s *parityTestSuite) validateFixtureBytes(t *testing.T, expectedPath string, actualBytes []byte, name string) { + t.Helper() + expectedBytes, err := os.ReadFile(expectedPath) + require.NoError(t, err, "Failed to read expected %s", name) + + require.JSONEq(t, string(expectedBytes), string(actualBytes), "%s does not match expected output", name) +} + +// TestParityV2_0 runs parity tests for SDL v2.0 fixtures. func TestParityV2_0(t *testing.T) { - testParity(t, "v2.0") + s := newParityTestSuite(t) + s.testParity(t, "v2.0") } +// TestParityV2_1 runs parity tests for SDL v2.1 fixtures. func TestParityV2_1(t *testing.T) { - testParity(t, "v2.1") + s := newParityTestSuite(t) + s.testParity(t, "v2.1") } -func testParity(t *testing.T, version string) { - inputDir := filepath.Join(fixturesInputRoot, version) +// testParity validates all fixtures for a given SDL version against Go parser output. +func (s *parityTestSuite) testParity(t *testing.T, version string) { + inputDir := filepath.Join(s.fixturesInputRoot, version) entries, err := os.ReadDir(inputDir) require.NoError(t, err) @@ -58,8 +119,8 @@ func testParity(t *testing.T, version string) { fixtureName := entry.Name() inputPath := filepath.Join(inputDir, fixtureName, "input.yaml") - manifestPath := filepath.Join(fixturesOutputRoot, version, fixtureName, "manifest.json") - groupSpecsPath := filepath.Join(fixturesOutputRoot, version, fixtureName, "group-specs.json") + manifestPath := filepath.Join(s.fixturesOutputRoot, version, fixtureName, "manifest.json") + groupSpecsPath := filepath.Join(s.fixturesOutputRoot, version, fixtureName, "group-specs.json") if _, err := os.Stat(manifestPath); os.IsNotExist(err) { t.Fatalf("manifest.json not generated for %s (run: make generate-sdl-fixtures)", fixtureName) @@ -73,7 +134,7 @@ func testParity(t *testing.T, version string) { inputBytes, err := os.ReadFile(inputPath) require.NoError(t, err) - validateInputSchema(t, inputBytes) + s.validateInputSchema(t, inputBytes) sdl, err := ReadFile(inputPath) require.NoError(t, err) @@ -90,26 +151,18 @@ func testParity(t *testing.T, version string) { groupSpecsBytes, err := json.Marshal(groupSpecs) require.NoError(t, err) - validateFixtureBytes(t, manifestPath, manifestBytes, "Manifest") - validateFixtureBytes(t, groupSpecsPath, groupSpecsBytes, "GroupSpecs") + s.validateOutputAgainstSchema(t, manifestBytes, groupSpecsBytes) + + s.validateFixtureBytes(t, manifestPath, manifestBytes, "Manifest") + s.validateFixtureBytes(t, groupSpecsPath, groupSpecsBytes, "GroupSpecs") }) } } -func validateInputSchema(t *testing.T, inputBytes []byte) { - err := validateInputAgainstSchema(inputBytes) - require.NoError(t, err, "Input schema validation failed") -} - -func validateFixtureBytes(t *testing.T, expectedPath string, actualBytes []byte, name string) { - expectedBytes, err := os.ReadFile(expectedPath) - require.NoError(t, err, "Failed to read expected %s", name) - - require.JSONEq(t, string(expectedBytes), string(actualBytes), "%s does not match expected output", name) -} - +// TestInvalidSDLsRejected verifies that all invalid fixtures are rejected by the Go parser. func TestInvalidSDLsRejected(t *testing.T) { - invalidDir := filepath.Join(fixturesInputRoot, "invalid") + s := newParityTestSuite(t) + invalidDir := filepath.Join(s.fixturesInputRoot, "invalid") entries, err := os.ReadDir(invalidDir) if os.IsNotExist(err) { @@ -131,12 +184,48 @@ func TestInvalidSDLsRejected(t *testing.T) { } } +// TestSemanticOnlyInvalid tests SDL files that pass schema validation but are +// rejected by both Go and TS parsers due to semantic constraints not expressible +// in JSON Schema (e.g., unused endpoints, cross-reference errors, duplicate mounts). +func TestSemanticOnlyInvalid(t *testing.T) { + s := newParityTestSuite(t) + semanticDir := filepath.Join(s.fixturesInputRoot, "semantic-only-invalid") + + entries, err := os.ReadDir(semanticDir) + if os.IsNotExist(err) { + t.Skip("Semantic-only invalid fixtures directory does not exist yet") + return + } + require.NoError(t, err) + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + fixturePath := filepath.Join(semanticDir, entry.Name()) + t.Run(entry.Name(), func(t *testing.T) { + inputBytes, err := os.ReadFile(fixturePath) + require.NoError(t, err) + + // Schema should accept (structurally valid) + schemaErr := validateInputAgainstSchema(inputBytes) + require.NoError(t, schemaErr, "Schema should accept this input (semantic-only invalid)") + + // Go parser should reject (semantically invalid) + _, goErr := ReadFile(fixturePath) + require.Error(t, goErr, "Go parser should reject this input (semantic validation)") + }) + } +} + // TestSchemaOnlyValidations tests SDL files that are rejected by the JSON schema // but accepted by the Go parser. These represent validations that exist only in // the schema layer (e.g., string length limits, enum value constraints) but are // not enforced in the Go validation logic. func TestSchemaOnlyValidations(t *testing.T) { - schemaOnlyDir := filepath.Join(fixturesInputRoot, "schema-only-invalid") + s := newParityTestSuite(t) + schemaOnlyDir := filepath.Join(s.fixturesInputRoot, "schema-only-invalid") entries, err := os.ReadDir(schemaOnlyDir) if os.IsNotExist(err) { @@ -163,3 +252,47 @@ func TestSchemaOnlyValidations(t *testing.T) { }) } } + +// compileSchemaFromPath loads a YAML JSON Schema file and compiles it for validation. +func compileSchemaFromPath(schemaPath string) (*gojsonschema.Schema, error) { + schemaBytes, err := os.ReadFile(schemaPath) + if err != nil { + return nil, fmt.Errorf("failed to read schema file: %w", err) + } + + var schemaData any + if err := yaml.Unmarshal(schemaBytes, &schemaData); err != nil { + return nil, fmt.Errorf("failed to parse YAML schema: %w", err) + } + + jsonBytes, err := json.Marshal(schemaData) + if err != nil { + return nil, fmt.Errorf("failed to convert schema to JSON: %w", err) + } + + schemaLoader := gojsonschema.NewSchemaLoader() + schema, err := schemaLoader.Compile(gojsonschema.NewBytesLoader(jsonBytes)) + if err != nil { + return nil, fmt.Errorf("failed to compile schema: %w", err) + } + + return schema, nil +} + +// validateDataAgainstCompiledSchema validates JSON bytes against a pre-compiled schema. +func validateDataAgainstCompiledSchema(data []byte, schema *gojsonschema.Schema) error { + result, err := schema.Validate(gojsonschema.NewBytesLoader(data)) + if err != nil { + return fmt.Errorf("failed to validate against schema: %w", err) + } + + if !result.Valid() { + var errors []string + for _, desc := range result.Errors() { + errors = append(errors, desc.String()) + } + return fmt.Errorf("schema validation failed: %v", errors) + } + + return nil +} diff --git a/go/sdl/schema_validator.go b/go/sdl/schema_validator.go index 5a33ea3b..069d0a08 100644 --- a/go/sdl/schema_validator.go +++ b/go/sdl/schema_validator.go @@ -138,14 +138,25 @@ func checkSchemaValidationResult(schemaErr error, goValidationErr error) { } func (sv *SchemaValidator) checkSchemaValidationResult(schemaErr error, goValidationErr error) { + logger := sv.logger.Load().(loggerHolder).Logger + if schemaErr == nil && goValidationErr != nil { - logger := sv.logger.Load().(loggerHolder).Logger + // Schema passed but Go rejected — semantic validation not covered by schema logger.Warn( - "SDL schema validation mismatch", + "SDL schema validation mismatch: schema passed, Go rejected", "schema_validation", "passed", "go_error", goValidationErr.Error(), ) } + + if schemaErr != nil && goValidationErr == nil { + // Schema rejected but Go accepted — schema is stricter than Go parser + logger.Warn( + "SDL schema validation mismatch: schema rejected, Go accepted", + "schema_error", schemaErr.Error(), + "go_validation", "passed", + ) + } } func sanitizeSchemaRefs(node any) error { diff --git a/go/sdl/sdl.go b/go/sdl/sdl.go index 2317eb1d..c7669adb 100644 --- a/go/sdl/sdl.go +++ b/go/sdl/sdl.go @@ -102,13 +102,23 @@ func Read(buf []byte) (sdlObj SDL, err error) { return nil, err } - if err = obj.validate(); err != nil { + if err = validateSDL(obj); err != nil { return nil, err } + return obj, nil +} + +// validateSDL runs semantic validation, deployment group validation, +// and manifest validation on an SDL instance. +func validateSDL(obj *sdl) error { + if err := obj.validate(); err != nil { + return fmt.Errorf("sdl validation: %w", err) + } + dgroups, err := obj.DeploymentGroups() if err != nil { - return nil, err + return fmt.Errorf("deployment groups: %w", err) } vgroups := make([]dtypes.GroupSpec, 0, len(dgroups)) @@ -117,19 +127,19 @@ func Read(buf []byte) (sdlObj SDL, err error) { } if err = dtypes.ValidateDeploymentGroups(vgroups); err != nil { - return nil, err + return fmt.Errorf("validate deployment groups: %w", err) } m, err := obj.Manifest() if err != nil { - return nil, err + return fmt.Errorf("manifest: %w", err) } if err = m.Validate(); err != nil { - return nil, err + return fmt.Errorf("validate manifest: %w", err) } - return obj, nil + return nil } // Version creates the deterministic Deployment Version hash from the SDL. @@ -164,3 +174,35 @@ func (s *sdl) validate() error { return s.data.validate() } + +// ReadFileStrict reads from given path and returns SDL instance with strict +// schema enforcement. Unlike ReadFile, it returns an error immediately if the +// input fails schema validation against sdl-input.schema.yaml. +func ReadFileStrict(path string) (SDL, error) { + buf, err := os.ReadFile(path) //nolint: gosec + if err != nil { + return nil, err + } + return ReadStrict(buf) +} + +// ReadStrict reads buffer data and returns SDL instance with strict schema +// enforcement. It validates against sdl-input.schema.yaml first and returns +// an error immediately on schema failure, then continues with existing Go +// semantic validation. +func ReadStrict(buf []byte) (SDL, error) { + if err := validateInputAgainstSchema(buf); err != nil { + return nil, fmt.Errorf("strict schema validation failed: %w", err) + } + + obj := &sdl{} + if err := yaml.Unmarshal(buf, obj); err != nil { + return nil, err + } + + if err := validateSDL(obj); err != nil { + return nil, err + } + + return obj, nil +} diff --git a/go/sdl/strict_test.go b/go/sdl/strict_test.go new file mode 100644 index 00000000..1b7c072a --- /dev/null +++ b/go/sdl/strict_test.go @@ -0,0 +1,94 @@ +package sdl + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestReadStrictValidInputs verifies that ReadFileStrict accepts all valid fixtures. +func TestReadStrictValidInputs(t *testing.T) { + s := newParityTestSuite(t) + + for _, version := range []string{"v2.0", "v2.1"} { + inputDir := filepath.Join(s.fixturesInputRoot, version) + + entries, err := os.ReadDir(inputDir) + require.NoError(t, err) + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + fixtureName := entry.Name() + inputPath := filepath.Join(inputDir, fixtureName, "input.yaml") + + t.Run(version+"/"+fixtureName, func(t *testing.T) { + sdl, err := ReadFileStrict(inputPath) + require.NoError(t, err, "ReadFileStrict should accept valid fixture %s", fixtureName) + require.NotNil(t, sdl) + }) + } + } +} + +// TestReadStrictRejectsInvalid verifies that ReadStrict rejects all invalid fixtures. +func TestReadStrictRejectsInvalid(t *testing.T) { + s := newParityTestSuite(t) + invalidDir := filepath.Join(s.fixturesInputRoot, "invalid") + + entries, err := os.ReadDir(invalidDir) + if os.IsNotExist(err) { + t.Skip("Invalid fixtures directory does not exist yet") + return + } + require.NoError(t, err) + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + fixturePath := filepath.Join(invalidDir, entry.Name()) + t.Run(entry.Name(), func(t *testing.T) { + _, err := ReadFileStrict(fixturePath) + require.Error(t, err, "ReadFileStrict should reject invalid fixture") + }) + } +} + +// TestReadStrictRejectsSchemaOnlyInvalid verifies that ReadStrict rejects +// inputs that the schema rejects but the lenient Go parser accepts. +// This is the key difference from ReadFile. +func TestReadStrictRejectsSchemaOnlyInvalid(t *testing.T) { + s := newParityTestSuite(t) + schemaOnlyDir := filepath.Join(s.fixturesInputRoot, "schema-only-invalid") + + entries, err := os.ReadDir(schemaOnlyDir) + if os.IsNotExist(err) { + t.Skip("Schema-only invalid fixtures directory does not exist yet") + return + } + require.NoError(t, err) + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + fixturePath := filepath.Join(schemaOnlyDir, entry.Name()) + t.Run(entry.Name(), func(t *testing.T) { + // Lenient ReadFile should accept these + _, lenientErr := ReadFile(fixturePath) + require.NoError(t, lenientErr, "ReadFile (lenient) should accept this input") + + // Strict ReadFileStrict should reject these + _, strictErr := ReadFileStrict(fixturePath) + require.Error(t, strictErr, "ReadFileStrict should reject schema-only-invalid input") + require.Contains(t, strictErr.Error(), "strict schema validation failed") + }) + } +} diff --git a/make/test.mk b/make/test.mk index 7f31a81f..d9c1db7b 100644 --- a/make/test.mk +++ b/make/test.mk @@ -55,9 +55,14 @@ generate-sdl-fixtures: ## Regenerate manifest.json and group-specs.json. Run onl .PHONY: test-sdl-parity test-sdl-parity: $(AKASH_TS_NODE_MODULES) ## Run SDL parity tests for Go and TypeScript (uses committed fixtures) + @echo "Running TS schema drift check..." + @cd ts && npm run compile:validators + @git diff --exit-code ts/src/sdl/validateSDL/validateSDLInput.ts || \ + (echo "ERROR: TS generated validator is out of sync with Go schema. Run 'cd ts && npm run compile:validators' and commit." && exit 1) + @echo "" @echo "Running Go SDL parity and validation tests..." - @cd go/sdl && go test -v -run "TestParity|TestInvalidSDLsRejected|TestSchemaValidation" + @cd go/sdl && go test -v -run "TestParity|TestInvalidSDLsRejected|TestSchemaValidation|TestReadStrict|TestSemanticOnlyInvalid" @echo "" @echo "Running TypeScript SDL parity tests..." - @echo " (includes: v2.0 fixtures, v2.1 fixtures, and invalid SDL rejection)" + @echo " (includes: v2.0 fixtures, v2.1 fixtures, invalid SDL rejection, schema-only-invalid)" @cd ts && npm run test:sdk-compatibility diff --git a/specs/sdl/README.md b/specs/sdl/README.md new file mode 100644 index 00000000..bf88f379 --- /dev/null +++ b/specs/sdl/README.md @@ -0,0 +1,35 @@ +# SDL Schema Documentation + +## Schema Files + +**`sdl-input.schema.yaml`** (`go/sdl/`) +- Validates user YAML input +- Embedded in Go binary for runtime validation (logs warnings, doesn't block) +- Enforces stricter rules than Go parser (email length, denom pattern, GPU vendor, version enum) + +**`manifest-output.schema.yaml`** (`specs/sdl/`) +- Validates generated manifest JSON + +**`groups-output.schema.yaml`** (`specs/sdl/`) +- Validates generated deployment groups JSON + +## Validation Capabilities + +`sdl-input.schema.yaml` validates: +- **Types & Constraints**: Required fields, enums, string patterns, min/max, minLength +- **Patterns**: Endpoint names (`^[a-z]+[-_\da-z]+$`), denom (`^(uakt|ibc/.*)$`) +- **Conditionals**: RAM storage → persistent=false, IP endpoint → global=true +- **Strict Rules**: Email ≥5 chars, password ≥6 chars, version ∈ {2.0, 2.1}, GPU vendor (nvidia only) + +## Validation Limitations + +Schema validates structure only. **Go/TS parsers handle:** +- Cross-references (deployment → profiles, params.storage → compute.storage) +- Semantic constraints (unused endpoints, port collisions, mount uniqueness) +- Parser-level checks (count ≥ 1, unknown fields — TS raises validation errors, Go rejects during unmarshal) + +## Test Fixtures + +`testdata/sdl/input/invalid/` — Both schema and Go parser reject +`testdata/sdl/input/schema-only-invalid/` — Schema rejects, Go parser accepts (stricter rules) +`testdata/sdl/input/v2.0/`, `v2.1/` — Valid fixtures for parity tests (Go ↔ TS output comparison) diff --git a/specs/sdl/groups-output.schema.yaml b/specs/sdl/groups-output.schema.yaml new file mode 100644 index 00000000..0b6895c7 --- /dev/null +++ b/specs/sdl/groups-output.schema.yaml @@ -0,0 +1,257 @@ +items: + additionalProperties: false + properties: + name: + type: string + requirements: + additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + signed_by: + additionalProperties: false + properties: + all_of: + anyOf: + - items: + type: string + type: array + - type: 'null' + any_of: + anyOf: + - items: + type: string + type: array + - type: 'null' + type: object + required: + - signed_by + type: object + resources: + anyOf: + - items: + additionalProperties: false + properties: + count: + type: number + price: + additionalProperties: false + properties: + amount: + anyOf: + - type: string + - type: 'null' + denom: + anyOf: + - type: string + - type: 'null' + required: + - amount + type: object + prices: + anyOf: + - items: + additionalProperties: false + properties: + amount: + anyOf: + - type: string + - type: 'null' + denom: + anyOf: + - type: string + - type: 'null' + required: + - amount + - denom + type: object + type: array + - type: 'null' + resource: + additionalProperties: false + properties: + cpu: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + units: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - units + type: object + - type: 'null' + endpoints: + anyOf: + - items: + additionalProperties: false + properties: + kind: + anyOf: + - type: number + - type: 'null' + sequence_number: + type: number + required: + - sequence_number + type: object + type: array + - type: 'null' + gpu: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + units: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - units + type: object + - type: 'null' + id: + type: number + memory: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + size: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - size + type: object + - type: 'null' + storage: + anyOf: + - items: + additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + name: + type: string + size: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - name + - size + type: object + type: array + - type: 'null' + required: + - id + type: object + anyOf: + - required: + - resource + - count + - price + - required: + - resource + - count + - prices + type: object + type: array + - type: 'null' + required: + - name + - requirements + type: object +type: array diff --git a/specs/sdl/manifest-output.schema.yaml b/specs/sdl/manifest-output.schema.yaml new file mode 100644 index 00000000..797ec291 --- /dev/null +++ b/specs/sdl/manifest-output.schema.yaml @@ -0,0 +1,314 @@ +items: + additionalProperties: false + properties: + name: + type: string + services: + anyOf: + - items: + additionalProperties: false + properties: + args: + anyOf: + - items: + type: string + type: array + - type: 'null' + command: + anyOf: + - items: + type: string + type: array + - type: 'null' + count: + type: number + credentials: + $ref: '#/$defs/credentials' + env: + anyOf: + - items: + type: string + type: array + - type: 'null' + expose: + anyOf: + - items: + additionalProperties: false + properties: + endpointSequenceNumber: + type: number + externalPort: + type: number + global: + type: boolean + hosts: + anyOf: + - items: + type: string + type: array + - type: 'null' + httpOptions: + additionalProperties: false + properties: + maxBodySize: + type: number + nextCases: + anyOf: + - items: + type: string + type: array + - type: 'null' + nextTimeout: + type: number + nextTries: + type: number + readTimeout: + type: number + sendTimeout: + type: number + required: + - maxBodySize + - readTimeout + - sendTimeout + - nextTries + - nextTimeout + type: object + ip: + type: string + port: + type: number + proto: + type: string + service: + type: string + required: + - port + - externalPort + - proto + - service + - global + - httpOptions + - ip + - endpointSequenceNumber + type: object + type: array + - type: 'null' + image: + type: string + name: + type: string + params: + anyOf: + - additionalProperties: false + properties: + credentials: + $ref: '#/$defs/credentials' + storage: + anyOf: + - items: + additionalProperties: false + properties: + mount: + type: string + name: + type: string + readOnly: + type: boolean + required: + - name + - mount + - readOnly + type: object + type: array + - type: 'null' + type: object + - type: 'null' + resources: + additionalProperties: false + properties: + cpu: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + units: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - units + type: object + - type: 'null' + endpoints: + anyOf: + - items: + additionalProperties: false + properties: + kind: + anyOf: + - type: number + - type: 'null' + sequence_number: + type: number + required: + - sequence_number + type: object + type: array + - type: 'null' + gpu: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + units: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - units + type: object + - type: 'null' + id: + type: number + memory: + anyOf: + - additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + size: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - size + type: object + - type: 'null' + storage: + anyOf: + - items: + additionalProperties: false + properties: + attributes: + anyOf: + - items: + additionalProperties: false + properties: + key: + anyOf: + - type: string + - type: 'null' + value: + anyOf: + - type: string + - type: 'null' + type: object + type: array + - type: 'null' + name: + type: string + size: + additionalProperties: false + properties: + val: + type: string + required: + - val + type: object + required: + - name + - size + type: object + type: array + - type: 'null' + required: + - id + type: object + required: + - name + - image + - resources + - count + type: object + type: array + - type: 'null' + required: + - name + type: object +type: array +$defs: + credentials: + anyOf: + - additionalProperties: false + properties: + email: + type: string + host: + type: string + password: + type: string + username: + type: string + required: + - host + - email + - username + - password + type: object + - type: 'null' diff --git a/testdata/sdl/input/semantic-only-invalid/duplicate-mount-path.yaml b/testdata/sdl/input/semantic-only-invalid/duplicate-mount-path.yaml new file mode 100644 index 00000000..748d561b --- /dev/null +++ b/testdata/sdl/input/semantic-only-invalid/duplicate-mount-path.yaml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=../../../../go/sdl/sdl-input.schema.yaml +# Structurally valid per schema, but semantically invalid: +# Two persistent volumes mounted at the same path +--- +version: "2.0" +services: + web: + image: nginx + expose: + - port: 80 + to: + - global: true + params: + storage: + data1: + mount: /data + data2: + mount: /data +profiles: + compute: + web: + resources: + cpu: + units: 100m + memory: + size: 512Mi + storage: + - size: 1Gi + - name: data1 + size: 5Gi + attributes: + persistent: true + class: beta2 + - name: data2 + size: 5Gi + attributes: + persistent: true + class: beta2 + placement: + akash: + pricing: + web: + denom: uakt + amount: 100 +deployment: + web: + akash: + profile: web + count: 1 diff --git a/testdata/sdl/input/semantic-only-invalid/endpoint-not-used.yaml b/testdata/sdl/input/semantic-only-invalid/endpoint-not-used.yaml new file mode 100644 index 00000000..7b88351f --- /dev/null +++ b/testdata/sdl/input/semantic-only-invalid/endpoint-not-used.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=../../../../go/sdl/sdl-input.schema.yaml +# Structurally valid per schema, but semantically invalid: +# endpoint declared but never referenced in any service expose +--- +version: "2.0" +services: + web: + image: nginx + expose: + - port: 80 + to: + - global: true +profiles: + compute: + web: + resources: + cpu: + units: 100m + memory: + size: 512Mi + storage: + size: 1Gi + placement: + akash: + pricing: + web: + denom: uakt + amount: 100 +deployment: + web: + akash: + profile: web + count: 1 +endpoints: + myendpoint: + kind: ip diff --git a/testdata/sdl/input/semantic-only-invalid/missing-compute-profile.yaml b/testdata/sdl/input/semantic-only-invalid/missing-compute-profile.yaml new file mode 100644 index 00000000..2ba52718 --- /dev/null +++ b/testdata/sdl/input/semantic-only-invalid/missing-compute-profile.yaml @@ -0,0 +1,33 @@ +# yaml-language-server: $schema=../../../../go/sdl/sdl-input.schema.yaml +# Structurally valid per schema, but semantically invalid: +# Deployment references a compute profile that doesn't exist +--- +version: "2.0" +services: + web: + image: nginx + expose: + - port: 80 + to: + - global: true +profiles: + compute: + web: + resources: + cpu: + units: 100m + memory: + size: 512Mi + storage: + size: 1Gi + placement: + akash: + pricing: + web: + denom: uakt + amount: 100 +deployment: + web: + akash: + profile: nonexistent + count: 1 diff --git a/testdata/sdl/output-fixtures/v2.0/gpu-basic/groups.json b/testdata/sdl/output-fixtures/v2.0/gpu-basic/groups.json new file mode 100644 index 00000000..af9be965 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/gpu-basic/groups.json @@ -0,0 +1,65 @@ +[ + { + "name": "westcoast", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "host", + "value": "akash" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "2147483648" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "10737418240" + } + } + ], + "gpu": { + "units": { + "val": "1" + }, + "attributes": [ + { + "key": "vendor/nvidia/model/rtx3080", + "value": "true" + } + ] + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "1000.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/http-options/groups.json b/testdata/sdl/output-fixtures/v2.0/http-options/groups.json new file mode 100644 index 00000000..ed12b2dd --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/http-options/groups.json @@ -0,0 +1,59 @@ +[ + { + "name": "akash", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "host", + "value": "akash" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "500" + } + }, + "memory": { + "size": { + "val": "536870912" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "150.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/ip-endpoint/groups.json b/testdata/sdl/output-fixtures/v2.0/ip-endpoint/groups.json new file mode 100644 index 00000000..7ab0c45b --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/ip-endpoint/groups.json @@ -0,0 +1,71 @@ +[ + { + "name": "akash", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "host", + "value": "akash" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "250" + } + }, + "memory": { + "size": { + "val": "536870912" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + }, + { + "kind": 2, + "sequence_number": 1 + }, + { + "kind": 1, + "sequence_number": 0 + }, + { + "kind": 2, + "sequence_number": 2 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "200.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/multiple-services/groups.json b/testdata/sdl/output-fixtures/v2.0/multiple-services/groups.json new file mode 100644 index 00000000..51095c83 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/multiple-services/groups.json @@ -0,0 +1,130 @@ +[ + { + "name": "datacenter", + "requirements": { + "signed_by": { + "all_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ], + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63", + "akash18qa2a2ltfyvkyj0ggj3hkvuj6twzyumuaru9s4" + ] + }, + "attributes": [ + { + "key": "region", + "value": "us-east" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "500" + } + }, + "memory": { + "size": { + "val": "1073741824" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "2147483648" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [] + }, + "count": 3, + "price": { + "denom": "uakt", + "amount": "300.000000000000000000" + } + }, + { + "resource": { + "id": 2, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "2147483648" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "10737418240" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "500.000000000000000000" + } + }, + { + "resource": { + "id": 3, + "cpu": { + "units": { + "val": "100" + } + }, + "memory": { + "size": { + "val": "268435456" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "536870912" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 2, + "price": { + "denom": "uakt", + "amount": "100.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/persistent-storage/groups.json b/testdata/sdl/output-fixtures/v2.0/persistent-storage/groups.json new file mode 100644 index 00000000..eed7fffb --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/persistent-storage/groups.json @@ -0,0 +1,70 @@ +[ + { + "name": "datacenter", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "region", + "value": "us-west" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "500" + } + }, + "memory": { + "size": { + "val": "1073741824" + } + }, + "storage": [ + { + "name": "data", + "size": { + "val": "53687091200" + }, + "attributes": [ + { + "key": "class", + "value": "beta2" + }, + { + "key": "persistent", + "value": "true" + } + ] + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "kind": 1, + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "500.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/placement/groups.json b/testdata/sdl/output-fixtures/v2.0/placement/groups.json new file mode 100644 index 00000000..f524c6d2 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/placement/groups.json @@ -0,0 +1,59 @@ +[ + { + "name": "us-west", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1vz375dkt0c60annyp6mkzeejfq0qpyevhseu05" + ] + }, + "attributes": [ + { + "key": "region", + "value": "us-west" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "500" + } + }, + "memory": { + "size": { + "val": "536870912" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "536870912" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "1000.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/port-ranges/groups.json b/testdata/sdl/output-fixtures/v2.0/port-ranges/groups.json new file mode 100644 index 00000000..eb773e60 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/port-ranges/groups.json @@ -0,0 +1,60 @@ +[ + { + "name": "akash", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": null + }, + "attributes": null + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "2147483648" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + }, + { + "kind": 1, + "sequence_number": 0 + }, + { + "kind": 1, + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "2000.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/pricing/groups.json b/testdata/sdl/output-fixtures/v2.0/pricing/groups.json new file mode 100644 index 00000000..b077af83 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/pricing/groups.json @@ -0,0 +1,102 @@ +[ + { + "name": "provider-a", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": null + }, + "attributes": null + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "1073741824" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "5000.000000000000000000" + } + } + ] + }, + { + "name": "provider-b", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": null + }, + "attributes": null + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "1073741824" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "3000.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/simple/groups.json b/testdata/sdl/output-fixtures/v2.0/simple/groups.json new file mode 100644 index 00000000..833dd05d --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/simple/groups.json @@ -0,0 +1,67 @@ +[ + { + "name": "westcoast", + "requirements": { + "signed_by": { + "all_of": [ + "3", + "4" + ], + "any_of": [ + "1", + "2" + ] + }, + "attributes": [ + { + "key": "region", + "value": "us-west" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "100" + } + }, + "memory": { + "size": { + "val": "134217728" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + }, + { + "kind": 1, + "sequence_number": 0 + } + ] + }, + "count": 2, + "price": { + "denom": "uakt", + "amount": "50.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.0/storage-classes/groups.json b/testdata/sdl/output-fixtures/v2.0/storage-classes/groups.json new file mode 100644 index 00000000..6281c282 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.0/storage-classes/groups.json @@ -0,0 +1,87 @@ +[ + { + "name": "datacenter", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "region", + "value": "us-central" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "1000" + } + }, + "memory": { + "size": { + "val": "2147483648" + } + }, + "storage": [ + { + "name": "cache", + "size": { + "val": "1073741824" + }, + "attributes": [ + { + "key": "persistent", + "value": "false" + } + ] + }, + { + "name": "logs", + "size": { + "val": "5368709120" + } + }, + { + "name": "data", + "size": { + "val": "21474836480" + }, + "attributes": [ + { + "key": "class", + "value": "beta3" + }, + { + "key": "persistent", + "value": "true" + } + ] + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "400.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.1/credentials/groups.json b/testdata/sdl/output-fixtures/v2.1/credentials/groups.json new file mode 100644 index 00000000..40bfa950 --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.1/credentials/groups.json @@ -0,0 +1,59 @@ +[ + { + "name": "akash", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "host", + "value": "akash" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "500" + } + }, + "memory": { + "size": { + "val": "1073741824" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "5368709120" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "250.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/testdata/sdl/output-fixtures/v2.1/ip-endpoint/groups.json b/testdata/sdl/output-fixtures/v2.1/ip-endpoint/groups.json new file mode 100644 index 00000000..f7a3f9bc --- /dev/null +++ b/testdata/sdl/output-fixtures/v2.1/ip-endpoint/groups.json @@ -0,0 +1,114 @@ +[ + { + "name": "akash", + "requirements": { + "signed_by": { + "all_of": null, + "any_of": [ + "akash1365yvmc4s7awdyj3n2sav7xfx76adc6dnmlx63" + ] + }, + "attributes": [ + { + "key": "host", + "value": "akash" + } + ] + }, + "resources": [ + { + "resource": { + "id": 1, + "cpu": { + "units": { + "val": "250" + } + }, + "memory": { + "size": { + "val": "536870912" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "1073741824" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "kind": 1, + "sequence_number": 0 + }, + { + "kind": 2, + "sequence_number": 3 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "200.000000000000000000" + } + }, + { + "resource": { + "id": 2, + "cpu": { + "units": { + "val": "100" + } + }, + "memory": { + "size": { + "val": "268435456" + } + }, + "storage": [ + { + "name": "default", + "size": { + "val": "536870912" + } + } + ], + "gpu": { + "units": { + "val": "0" + } + }, + "endpoints": [ + { + "sequence_number": 0 + }, + { + "kind": 2, + "sequence_number": 1 + }, + { + "kind": 1, + "sequence_number": 0 + }, + { + "kind": 2, + "sequence_number": 2 + } + ] + }, + "count": 1, + "price": { + "denom": "uakt", + "amount": "150.000000000000000000" + } + } + ] + } +] \ No newline at end of file diff --git a/ts/src/sdl/manifest/generateManifest.ts b/ts/src/sdl/manifest/generateManifest.ts index a57e163b..95e14125 100644 --- a/ts/src/sdl/manifest/generateManifest.ts +++ b/ts/src/sdl/manifest/generateManifest.ts @@ -130,18 +130,13 @@ export function generateManifest(sdl: SDLInput, networkId: NetworkId = MAINNET_I group.dgroup.resources[location].resource!.endpoints.push( ...buildServiceEndpoints(service, endpointSequenceNumbers), ); + group.dgroup.resources[location].resource!.endpoints.sort( + (a, b) => a.sequenceNumber - b.sequenceNumber, + ); } } } - for (const group of groupsMap.values()) { - for (const resourceUnit of group.dgroup.resources) { - resourceUnit.resource!.endpoints.sort( - (a, b) => a.kind - b.kind || a.sequenceNumber - b.sequenceNumber, - ); - } - } - const sortedGroupNames = [...groupsMap.keys()].sort(); let groups: Group[] | undefined; let groupSpecs: GroupSpec[] | undefined; diff --git a/ts/src/sdl/manifest/generateManifestVersion.spec.ts b/ts/src/sdl/manifest/generateManifestVersion.spec.ts index b1bb888f..72c1c638 100644 --- a/ts/src/sdl/manifest/generateManifestVersion.spec.ts +++ b/ts/src/sdl/manifest/generateManifestVersion.spec.ts @@ -621,6 +621,57 @@ describe(generateManifestVersion.name, () => { }); }); + describe("LegacyDec amount formatting", () => { + it("formats integer amount with 18 decimal places", () => { + const obj = [{ price: { denom: "uakt", amount: "1000" } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"1000.000000000000000000"'); + }); + + it("formats decimal string amount to 18 places", () => { + const obj = [{ price: { denom: "uakt", amount: "1.5" } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"1.500000000000000000"'); + }); + + it("formats zero amount", () => { + const obj = [{ price: { denom: "uakt", amount: "0" } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"0.000000000000000000"'); + }); + + it("formats small decimal amount", () => { + const obj = [{ price: { denom: "uakt", amount: "0.001" } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"0.001000000000000000"'); + }); + + it("formats numeric amount passed directly to replacer", () => { + const obj = [{ price: { denom: "uakt", amount: 100 } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"100.000000000000000000"'); + }); + + it("formats numeric decimal amount passed directly to replacer", () => { + const obj = [{ price: { denom: "uakt", amount: 2.5 } }]; + const json = manifestToSortedJSON(obj as never); + + expect(json).toContain('"2.500000000000000000"'); + }); + + it("formats amount in groupSpecs through full pipeline", () => { + const { groupSpecs } = setupWithGroupSpecs({ amount: 5000 }); + const json = manifestToSortedJSON(groupSpecs); + + expect(json).toContain('"5000.000000000000000000"'); + }); + }); + function setup(options: { sdl?: SDLInput; image?: string; @@ -658,6 +709,13 @@ describe(generateManifestVersion.name, () => { } } + function setupWithGroupSpecs(options: { amount?: number | string } = {}) { + const sdl = createBasicSdl({ amount: options.amount }); + const result = generateManifest(sdl); + assertBuildResult(result); + return { groupSpecs: result.value.groupSpecs }; + } + function createBasicSdl(options: { image?: string; command?: string[]; @@ -670,8 +728,9 @@ describe(generateManifestVersion.name, () => { to?: Array<{ global?: boolean }>; accept?: string[]; }>; + amount?: number | string; } = {}): SDLInput { - const { image = "nginx", command, args, env, credentials, expose } = options; + const { image = "nginx", command, args, env, credentials, expose, amount = 1000 } = options; return { version: "2.0", @@ -704,7 +763,7 @@ describe(generateManifestVersion.name, () => { placement: { dcloud: { pricing: { - web: { denom: "uakt", amount: 1000 }, + web: { denom: "uakt", amount }, }, }, }, diff --git a/ts/src/sdl/manifest/generateManifestVersion.ts b/ts/src/sdl/manifest/generateManifestVersion.ts index a63fb6ab..028e219f 100644 --- a/ts/src/sdl/manifest/generateManifestVersion.ts +++ b/ts/src/sdl/manifest/generateManifestVersion.ts @@ -1,10 +1,11 @@ import { default as stableStringify } from "json-stable-stringify"; +import { LegacyDec } from "../../encoding/customTypes/LegacyDec.ts"; import type { GenerateManifestOkResult, Manifest } from "./generateManifest.ts"; const decoder = new TextDecoder(); const encoder = new TextEncoder(); -const NULLABLE_MANIFEST_KEYS = new Set(["command", "args", "env", "hosts"]); +const NULLABLE_MANIFEST_KEYS = new Set(["command", "args", "env", "hosts", "allOf", "anyOf"]); const OMITTED_MANIFEST_KEYS = new Set(["kind", "attributes"]); export async function generateManifestVersion(manifest: Manifest): Promise { @@ -37,14 +38,42 @@ function manifestReplacer(this: unknown, key: string | number, value: unknown): return null; } + // Format price amount as LegacyDec (18 decimal places) to match Go output + if (key === "amount" && typeof this === "object" && this && Object.hasOwn(this, "denom") && (typeof value === "string" || typeof value === "number")) { + return formatLegacyDec(String(value)); + } + if (OMITTED_MANIFEST_KEYS.has(key) && ((Array.isArray(value) && value.length === 0) || value === 0)) { + // In requirements context (group-specs), empty attributes should be null, not omitted + if (key === "attributes" && typeof this === "object" && this && Object.hasOwn(this, "signedBy")) { + return null; + } return undefined; } return value; } -const MANIFEST_VERSION_FIELD_MAPPING: Record = { quantity: "size", sequenceNumber: "sequence_number" }; +const LEGACY_DEC_PRECISION = 18; + +function formatLegacyDec(s: string): string { + if (!s) return "0.000000000000000000"; + const atomics = LegacyDec.encode(s); + const sign = atomics.startsWith("-") ? "-" : ""; + const abs = sign ? atomics.slice(1) : atomics; + const padded = abs.padStart(LEGACY_DEC_PRECISION + 1, "0"); + const intPart = padded.slice(0, -LEGACY_DEC_PRECISION); + const fracPart = padded.slice(-LEGACY_DEC_PRECISION); + return `${sign}${intPart}.${fracPart}`; +} + +const MANIFEST_VERSION_FIELD_MAPPING: Record = { + quantity: "size", + sequenceNumber: "sequence_number", + signedBy: "signed_by", + allOf: "all_of", + anyOf: "any_of", +}; const MANIFEST_VERSION_FIELD_REGEX = new RegExp(`"(${Object.keys(MANIFEST_VERSION_FIELD_MAPPING).join("|")})":`, "g"); function renameFields(jsonStr: string): string { MANIFEST_VERSION_FIELD_REGEX.lastIndex = 0; // reset regex state diff --git a/ts/test/functional/sdl-parity.spec.ts b/ts/test/functional/sdl-parity.spec.ts index f1e8263a..2c735578 100644 --- a/ts/test/functional/sdl-parity.spec.ts +++ b/ts/test/functional/sdl-parity.spec.ts @@ -1,8 +1,9 @@ +import fs from "node:fs"; +import path from "node:path"; + import { describe, expect, it } from "@jest/globals"; import type { ErrorObject, ValidateFunction } from "ajv"; import { Ajv } from "ajv"; -import fs from "node:fs"; -import path from "node:path"; import { LegacyDec } from "../../src/encoding/customTypes/LegacyDec.ts"; import { generateManifest } from "../../src/sdl/manifest/generateManifest.ts"; @@ -14,13 +15,16 @@ const PROJECT_ROOT = path.join(__dirname, "../../.."); const FIXTURES_INPUT_ROOT = path.join(PROJECT_ROOT, "testdata/sdl/input"); const FIXTURES_OUTPUT_ROOT = path.join(PROJECT_ROOT, "testdata/sdl/output-fixtures"); const INPUT_SCHEMA_PATH = path.join(PROJECT_ROOT, "go/sdl/sdl-input.schema.yaml"); +const MANIFEST_OUTPUT_SCHEMA_PATH = path.join(PROJECT_ROOT, "specs/sdl/manifest-output.schema.yaml"); +const GROUPS_OUTPUT_SCHEMA_PATH = path.join(PROJECT_ROOT, "specs/sdl/groups-output.schema.yaml"); describe("SDL Parity Tests", () => { describe("v2.0", () => { loadFixtures("v2.0").forEach((fixture) => { it(fixture.name, () => { - const { manifest, expectedManifest } = setup(fixture); + const { manifest, expectedManifest, groupSpecs, expectedGroupSpecs } = setup(fixture); expect(manifest).toEqual(expectedManifest); + expect(groupSpecs).toEqual(expectedGroupSpecs); }); }); }); @@ -28,8 +32,9 @@ describe("SDL Parity Tests", () => { describe("v2.1", () => { loadFixtures("v2.1").forEach((fixture) => { it(fixture.name, () => { - const { manifest, expectedManifest } = setup(fixture); + const { manifest, expectedManifest, groupSpecs, expectedGroupSpecs } = setup(fixture); expect(manifest).toEqual(expectedManifest); + expect(groupSpecs).toEqual(expectedGroupSpecs); }); }); }); @@ -45,7 +50,7 @@ describe("SDL Parity Tests", () => { } fs.globSync("*.yaml", { cwd: invalidDir }).forEach((filename) => { - it(filename, () => { + it(filename, () => { const fixturePath = path.join(invalidDir, filename); const input = fs.readFileSync(fixturePath, "utf8"); const sdl: SDLInput = yaml.raw(input); @@ -55,6 +60,96 @@ describe("SDL Parity Tests", () => { }); }); + describe("semantic-only-invalid SDLs", () => { + const semanticInvalidDir = path.join(FIXTURES_INPUT_ROOT, "semantic-only-invalid"); + + if (!fs.existsSync(semanticInvalidDir)) { + it("semantic-only-invalid fixtures directory must exist", () => { + throw new Error(`Semantic-only-invalid fixtures directory not found: ${semanticInvalidDir}`); + }); + return; + } + + fs.globSync("*.yaml", { cwd: semanticInvalidDir }).forEach((filename) => { + it(`schema accepts: ${filename}`, () => { + const fixturePath = path.join(semanticInvalidDir, filename); + const input = fs.readFileSync(fixturePath, "utf8"); + const inputYAML: SDLInput = yaml.raw(input); + + const schemaValidator = compileSchema(INPUT_SCHEMA_PATH); + const schemaValid = schemaValidator(inputYAML); + expect(schemaValid).toBe(true); + }); + + it(`parser rejects: ${filename}`, () => { + const fixturePath = path.join(semanticInvalidDir, filename); + const input = fs.readFileSync(fixturePath, "utf8"); + const inputYAML: SDLInput = yaml.raw(input); + + const result = generateManifest(inputYAML); + expect(result.ok).toBe(false); + }); + }); + }); + + describe("schema-only-invalid SDLs", () => { + const schemaOnlyInvalidDir = path.join(FIXTURES_INPUT_ROOT, "schema-only-invalid"); + + if (!fs.existsSync(schemaOnlyInvalidDir)) { + it("schema-only-invalid fixtures directory must exist", () => { + throw new Error(`Schema-only-invalid fixtures directory not found: ${schemaOnlyInvalidDir}`); + }); + return; + } + + fs.globSync("*.yaml", { cwd: schemaOnlyInvalidDir }).forEach((filename) => { + it(`schema rejects: ${filename}`, () => { + const fixturePath = path.join(schemaOnlyInvalidDir, filename); + const input = fs.readFileSync(fixturePath, "utf8"); + const inputYAML: SDLInput = yaml.raw(input); + + const schemaValidator = compileSchema(INPUT_SCHEMA_PATH); + const schemaValid = schemaValidator(inputYAML); + expect(schemaValid).toBe(false); + }); + + it(`generateManifest also rejects: ${filename}`, () => { + const fixturePath = path.join(schemaOnlyInvalidDir, filename); + const input = fs.readFileSync(fixturePath, "utf8"); + const inputYAML: SDLInput = yaml.raw(input); + + const result = generateManifest(inputYAML); + expect(result.ok).toBe(false); + }); + }); + }); + + describe("canonical byte-level equality", () => { + const canonicalFixtures = [ + { version: "v2.0", name: "simple" }, + { version: "v2.0", name: "ip-endpoint" }, + { version: "v2.1", name: "credentials" }, + ]; + + canonicalFixtures.forEach(({ version, name }) => { + it(`${version}/${name} manifest canonical JSON matches`, () => { + const inputPath = path.join(FIXTURES_INPUT_ROOT, version, name, "input.yaml"); + const manifestPath = path.join(FIXTURES_OUTPUT_ROOT, version, name, "manifest.json"); + + const rawSDL = fs.readFileSync(inputPath, "utf8"); + const untrustedSDL: SDLInput = yaml.raw(rawSDL); + const result = generateManifest(untrustedSDL); + if (!result.ok) throw new Error(`generateManifest failed`); + + const canonicalTS = manifestToSortedJSON(result.value.groups); + const goFixture = JSON.parse(fs.readFileSync(manifestPath, "utf8")); + const canonicalGo = manifestToSortedJSON(goFixture); + + expect(canonicalTS).toBe(canonicalGo); + }); + }); + }); + function setup(fixture: Fixture) { const rawSDL = fs.readFileSync(fixture.inputPath, "utf8"); const untrustedSDL: SDLInput = yaml.raw(rawSDL); @@ -67,10 +162,18 @@ describe("SDL Parity Tests", () => { const manifest = JSON.parse(manifestToSortedJSON(result.value.groups), normalizeManifestJSON); const expectedManifest = JSON.parse(fs.readFileSync(fixture.manifestPath, "utf8")); + const groupSpecs = JSON.parse(manifestToSortedJSON(result.value.groupSpecs)); + const expectedGroupSpecs = JSON.parse(fs.readFileSync(fixture.groupSpecsPath, "utf8")); + + validateAgainstSchema("manifest output", manifest, MANIFEST_OUTPUT_SCHEMA_PATH); + validateAgainstSchema("groups output", groupSpecs, GROUPS_OUTPUT_SCHEMA_PATH); + return { manifest, - expectedManifest - } + expectedManifest, + groupSpecs, + expectedGroupSpecs, + }; } }); @@ -89,15 +192,21 @@ function loadFixtures(version: string): Fixture[] { const fixtureName = entry.name; const inputPath = path.join(inputVersionDir, fixtureName, "input.yaml"); const manifestPath = path.join(FIXTURES_OUTPUT_ROOT, version, fixtureName, "manifest.json"); + const groupSpecsPath = path.join(FIXTURES_OUTPUT_ROOT, version, fixtureName, "group-specs.json"); if (!fs.existsSync(manifestPath)) { throw new Error(`manifest.json not generated for ${fixtureName} (run: make generate-sdl-fixtures)`); } + if (!fs.existsSync(groupSpecsPath)) { + throw new Error(`group-specs.json not generated for ${fixtureName} (run: make generate-sdl-fixtures)`); + } + return { name: fixtureName, inputPath, manifestPath, + groupSpecsPath, }; }); } @@ -132,9 +241,9 @@ function compileSchema(schemaPath: string): ValidateFunction { } function normalizeManifestJSON(this: unknown, key: string, value: unknown): unknown { - if (typeof this !== 'object' || this === null) return value; + if (typeof this !== "object" || this === null) return value; - if (key === "amount" && 'denom' in this && this.denom !== undefined) { + if (key === "amount" && "denom" in this && this.denom !== undefined) { return LegacyDec.encode(value as string); } @@ -149,4 +258,5 @@ interface Fixture { name: string; inputPath: string; manifestPath: string; + groupSpecsPath: string; }