diff --git a/docs/arbitrary-versions-feature.md b/docs/arbitrary-versions-feature.md new file mode 100644 index 00000000000..6b0d83d91c6 --- /dev/null +++ b/docs/arbitrary-versions-feature.md @@ -0,0 +1,229 @@ +# Arbitrary OpenShift Versions Feature + +## Overview + +This feature allows ARO clusters to be created with arbitrary OpenShift version strings instead of being limited to pre-defined versions stored in CosmosDB. This capability can be enabled through either: + +1. **AFEC Feature Flag**: Protected by subscription-level feature registration for production/testing environments +2. **Development Environment**: Automatically enabled when running in local development mode + +This dual-enablement approach supports both secure production testing and convenient development workflows. + +## Implementation Details + +### 1. AFEC Feature Flag + +A new feature flag has been added to control access to this functionality: + +```go +// pkg/api/featureflags.go +FeatureFlagArbitraryVersions = "Microsoft.RedHatOpenShift/ArbitraryVersions" +``` + +### 2. Enhanced Version Validation Logic + +The `validateInstallVersion()` function in `pkg/frontend/validate.go` has been enhanced to: + +- Accept a subscription document parameter to check for feature flags +- Check for both the `FeatureFlagArbitraryVersions` AFEC flag and development environment +- Apply different validation logic based on enablement conditions: + +```go +allowArbitraryVersions := f.env.IsLocalDevelopmentMode() || + (subscription != nil && feature.IsRegisteredForFeature(subscription.Subscription.Properties, api.FeatureFlagArbitraryVersions)) +``` + +**When arbitrary versions are enabled (either via AFEC flag OR development mode):** +- Only validates semantic version format using `semver.NewVersion` +- Bypasses the requirement for versions to be in the CosmosDB enabled list +- Allows custom version strings like `4.15.0-custom.build.123` + +**When arbitrary versions are disabled:** +- Maintains existing behavior (version must be in enabled list AND valid semver) +- Preserves backward compatibility + +### 3. Enhanced Image Resolution with ACR Fallback + +The version resolution system in `pkg/cluster/version.go` has been enhanced to support arbitrary versions: + +**Resolution Order:** +1. **CosmosDB First**: Check if the version exists in the OpenShiftVersions collection +2. **ACR Fallback**: If not found and arbitrary versions are enabled (via AFEC flag OR development mode), generate image specs using ACR patterns +3. **Error**: If neither found and arbitrary versions disabled, return error + +**ACR Image Generation:** +- **Installer Image**: `{ACRDomain}/aro-installer:{major.minor}` (e.g., `arosvc.azurecr.io/aro-installer:4.15`) +- **OpenShift Image (Hive)**: `{ACRDomain}/ocp-release:{full-version}` (e.g., `arosvc.azurecr.io/ocp-release:4.15.0-custom.build.123`) +- **OpenShift Image (Traditional)**: `quay.io/openshift-release-dev/ocp-release:{full-version}` + +**Installer Pull Spec Override**: Still honors `env.LiveConfig().DefaultInstallerPullSpecOverride()` when set + +### 4. Updated Function Signatures + +The following functions were updated to support subscription-aware validation: + +- `validateInstallVersion(ctx, oc, subscription)` - Now accepts subscription document +- `openShiftClusterDocumentVersioner.GetWithSubscription()` - New method with subscription support +- PUT/PATCH handler passes subscription to validation +- Preflight validation retrieves and passes subscription document + +### 5. Test Coverage + +Comprehensive test cases have been added covering: + +**Frontend Validation Tests:** +- ✅ Arbitrary valid semver versions with feature flag enabled +- ✅ Arbitrary valid semver versions in development mode +- ✅ Invalid versions with feature flag enabled (proper error handling) +- ✅ Invalid versions in development mode (proper error handling) +- ✅ Arbitrary versions without feature flag (blocked as expected) +- ✅ Both AFEC flag and development mode enabled (should work) +- ✅ Development mode overrides normal validation for arbitrary versions +- ✅ Existing functionality preserved for standard versions + +**Image Resolution Tests:** +- ✅ ACR fallback for traditional installer (quay.io OpenShift images) +- ✅ ACR fallback for Hive installer (ACR OpenShift images) +- ✅ ACR fallback in development mode (both traditional and hive) +- ✅ CosmosDB versions take precedence over ACR fallback +- ✅ Invalid semantic versions with proper error messages +- ✅ Invalid semantic versions in development mode +- ✅ Prerelease and development version handling +- ✅ Major.minor version extraction for installer images +- ✅ Both AFEC flag and development mode scenarios +- ✅ Development mode override behavior for version resolution + +## Usage + +### Enabling the Feature + +There are two ways to enable arbitrary version support: + +#### Option 1: AFEC Feature Flag (Production/Testing) + +To enable arbitrary versions for a subscription: + +```bash +# Register the feature flag for a subscription +az feature register --namespace Microsoft.RedHatOpenShift --name ArbitraryVersions + +# Verify registration (may take a few minutes) +az feature show --namespace Microsoft.RedHatOpenShift --name ArbitraryVersions +``` + +#### Option 2: Development Environment (Local Development) + +For local development, the feature is automatically enabled when `RP_MODE=development` is set: + +```bash +# Set development mode environment variable +export RP_MODE=development + +# Now arbitrary versions are automatically enabled without AFEC registration +``` + +**Note**: Development mode bypasses AFEC flag requirements and is intended for local development only. + +### Example Version Strings + +Once enabled, cluster creation requests can specify custom versions such as: + +- `4.15.0-custom.build.123` - Custom builds +- `4.14.0-0.nightly-2024-01-01-000000` - Nightly builds +- `4.13.25+dev.branch.feature` - Development branches +- `4.16.0-rc.1` - Release candidates + +### API Usage + +No changes to the API structure are required. Simply specify the desired version in the cluster creation request: + +```json +{ + "properties": { + "clusterProfile": { + "version": "4.15.0-custom.build.123" + } + } +} +``` + +## Security Considerations + +- **Dual-Layer Protection**: Feature is gated behind either: + - **AFEC Protection**: Subscription-level feature registration for production environments + - **Development Mode**: Local development environment detection (`RP_MODE=development`) +- **Validation**: Still enforces semantic versioning format to prevent invalid strings +- **Environment Isolation**: Development mode only works in local development environments +- **Audit Trail**: Feature flag registration is tracked in Azure subscription logs + +## Image Resolution Behavior + +### ACR Fallback Logic + +When using arbitrary versions, the system follows this resolution order: + +1. **Check CosmosDB**: First attempts to find the exact version in the OpenShiftVersions collection +2. **Generate ACR Specs**: If not found and feature flag enabled, generates image specifications: + - Installer image uses `{major.minor}` tagging (e.g., `4.15` for version `4.15.0-custom.build.123`) + - OpenShift image includes full version string +3. **Installation Attempt**: The installation will proceed with generated image specifications +4. **Runtime Validation**: If the images don't exist in ACR, the installation will fail during image pull + +### Image Availability + +The ACR fallback assumes images follow these naming conventions: +- **Production ACR**: `arosvc.azurecr.io/aro-installer:4.15` +- **Integration ACR**: `arointsvc.azurecr.io/aro-installer:4.15` + +**Important**: The feature enables specifying arbitrary versions, but actual installation success depends on the availability of corresponding images in the configured ACR registry. + +## Error Handling + +### Invalid Semantic Version +``` +400: InvalidParameter: properties.clusterProfile.version: +The requested OpenShift version 'not-a-valid-version' is not a valid semantic version. +``` + +### Feature Not Enabled +``` +400: InvalidParameter: properties.clusterProfile.version: +The requested OpenShift version '4.15.0-custom.build.123' is invalid. +``` + +### Installation-Time Errors +If ACR images don't exist, installation will fail with image pull errors during the cluster creation process. + +## Files Modified + +- `pkg/api/featureflags.go` - Added new feature flag constant +- `pkg/frontend/validate.go` - Enhanced validation logic with AFEC flag support +- `pkg/frontend/openshiftcluster_putorpatch.go` - Updated function calls +- `pkg/frontend/openshiftcluster_preflightvalidation.go` - Added subscription retrieval +- `pkg/frontend/validate_test.go` - Added comprehensive test coverage +- `pkg/cluster/version.go` - Enhanced image resolution with ACR fallback logic +- `pkg/cluster/install_version.go` - Updated to use subscription-aware version resolver +- `pkg/cluster/install_version_test.go` - Added ACR fallback test coverage + +## Compatibility + +This feature maintains full backward compatibility. Existing clusters and installations are unaffected when the feature flag is not enabled. The implementation follows the same pattern as the existing MTU3900 feature flag. + +## Testing + +The feature includes comprehensive unit tests that validate: + +1. **AFEC Flag Scenarios**: Feature flag enabled/disabled with valid and invalid versions +2. **Development Mode Scenarios**: Local development environment detection and behavior +3. **Combined Scenarios**: Both AFEC flag and development mode enabled +4. **Backward Compatibility**: Existing behavior preserved when feature is disabled +5. **Default Behavior**: Default version assignment when no version specified +6. **Error Handling**: Accurate error messages for all failure scenarios +7. **ACR Fallback**: Image resolution patterns for arbitrary versions +8. **Precedence Rules**: CosmosDB versions take precedence over ACR fallback + +## Future Considerations + +- Consider adding validation for minimum supported version patterns +- Potential integration with custom installer image specifications +- Monitoring and alerting for non-standard version usage \ No newline at end of file diff --git a/pkg/api/featureflags.go b/pkg/api/featureflags.go index 078f9ec7a34..7cb8b7c8e40 100644 --- a/pkg/api/featureflags.go +++ b/pkg/api/featureflags.go @@ -9,4 +9,10 @@ const ( // Unit (MTU) on Azure virtual networks, which as of late 2021 is 3900 bytes. // Otherwise cluster nodes will use the DHCP-provided MTU of 1500 bytes. FeatureFlagMTU3900 = "Microsoft.RedHatOpenShift/MTU3900" + + // FeatureFlagArbitraryVersions allows specifying arbitrary OpenShift version + // strings during cluster creation instead of being limited to pre-defined + // versions stored in CosmosDB. This is intended for testing and development + // scenarios where custom builds or pre-release versions need to be installed. + FeatureFlagArbitraryVersions = "Microsoft.RedHatOpenShift/ArbitraryVersions" ) diff --git a/pkg/cluster/install_version.go b/pkg/cluster/install_version.go index 8e7c5d51824..a207e713905 100644 --- a/pkg/cluster/install_version.go +++ b/pkg/cluster/install_version.go @@ -10,5 +10,6 @@ import ( ) func (m *manager) openShiftVersionFromVersion(ctx context.Context) (*api.OpenShiftVersion, error) { - return m.openShiftClusterDocumentVersioner.Get(ctx, m.doc, m.dbOpenShiftVersions, m.env, m.installViaHive) + // Use the enhanced version resolver with subscription data for AFEC flag support + return m.openShiftClusterDocumentVersioner.GetWithSubscription(ctx, m.doc, m.dbOpenShiftVersions, m.env, m.installViaHive, m.subscriptionDoc) } diff --git a/pkg/cluster/install_version_test.go b/pkg/cluster/install_version_test.go index b3307e175ed..9a250d074a0 100644 --- a/pkg/cluster/install_version_test.go +++ b/pkg/cluster/install_version_test.go @@ -110,3 +110,194 @@ func TestGetOpenShiftVersionFromVersion(t *testing.T) { }) } } + +func TestGetOpenShiftVersionFromVersionWithArbitraryVersions(t *testing.T) { + const testACRDomain = "acrdomain.io" + + key := "/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/resourceGroup/providers/Microsoft.RedHatOpenShift/openShiftClusters/resourceName" + + for _, tt := range []struct { + name string + version string + installViaHive bool + arbitraryVersionsEnabled bool + isLocalDevelopmentMode bool + wantErrString string + wantInstallerPullspec string + wantOpenShiftPullspec string + wantVersion string + }{ + { + name: "arbitrary version with feature flag enabled - traditional install", + version: "4.15.0-custom.build.123", + installViaHive: false, + arbitraryVersionsEnabled: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.15", + wantOpenShiftPullspec: "quay.io/openshift-release-dev/ocp-release:4.15.0-custom.build.123", + wantVersion: "4.15.0-custom.build.123", + }, + { + name: "arbitrary version with feature flag enabled - hive install", + version: "4.16.5-dev.branch.456", + installViaHive: true, + arbitraryVersionsEnabled: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.16", + wantOpenShiftPullspec: "acrdomain.io/ocp-release:4.16.5-dev.branch.456", + wantVersion: "4.16.5-dev.branch.456", + }, + { + name: "arbitrary version with feature flag disabled", + version: "4.15.0-custom.build.123", + installViaHive: false, + arbitraryVersionsEnabled: false, + wantErrString: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version '4.15.0-custom.build.123' is not supported.", + }, + { + name: "invalid semantic version with feature flag enabled", + version: "not-a-valid-version", + installViaHive: false, + arbitraryVersionsEnabled: true, + wantErrString: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version 'not-a-valid-version' is not a valid semantic version.", + }, + { + name: "version in CosmosDB takes precedence over arbitrary versions", + version: "4.14.38", // This version exists in our test fixture below + installViaHive: false, + arbitraryVersionsEnabled: true, + wantInstallerPullspec: "installerimage:1.0.0", // From fixture + wantOpenShiftPullspec: "openshiftimage:1.0.0", // From fixture + wantVersion: "4.14.38", + }, + { + name: "prerelease arbitrary version", + version: "4.17.0-0.nightly-2024-12-01-123456", + installViaHive: true, + arbitraryVersionsEnabled: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.17", + wantOpenShiftPullspec: "acrdomain.io/ocp-release:4.17.0-0.nightly-2024-12-01-123456", + wantVersion: "4.17.0-0.nightly-2024-12-01-123456", + }, + { + name: "arbitrary version in development mode - traditional install", + version: "4.18.0-dev.custom.build", + installViaHive: false, + isLocalDevelopmentMode: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.18", + wantOpenShiftPullspec: "quay.io/openshift-release-dev/ocp-release:4.18.0-dev.custom.build", + wantVersion: "4.18.0-dev.custom.build", + }, + { + name: "arbitrary version in development mode - hive install", + version: "4.19.0-0.nightly-dev-123", + installViaHive: true, + isLocalDevelopmentMode: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.19", + wantOpenShiftPullspec: "acrdomain.io/ocp-release:4.19.0-0.nightly-dev-123", + wantVersion: "4.19.0-0.nightly-dev-123", + }, + { + name: "invalid semantic version in development mode", + version: "not-a-valid-version", + isLocalDevelopmentMode: true, + wantErrString: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version 'not-a-valid-version' is not a valid semantic version.", + }, + { + name: "both AFEC flag and dev mode enabled - should work", + version: "4.20.0-combined.test", + installViaHive: false, + arbitraryVersionsEnabled: true, + isLocalDevelopmentMode: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.20", + wantOpenShiftPullspec: "quay.io/openshift-release-dev/ocp-release:4.20.0-combined.test", + wantVersion: "4.20.0-combined.test", + }, + { + name: "dev mode overrides normal validation for arbitrary versions", + version: "4.21.0-0.dev-override-123", + isLocalDevelopmentMode: true, + wantInstallerPullspec: "acrdomain.io/aro-installer:4.21", + wantOpenShiftPullspec: "quay.io/openshift-release-dev/ocp-release:4.21.0-0.dev-override-123", + wantVersion: "4.21.0-0.dev-override-123", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + controller := gomock.NewController(t) + + tlc := testliveconfig.NewTestLiveConfig(false, false) + _env := mock_env.NewMockInterface(controller) + _env.EXPECT().ACRDomain().AnyTimes().Return(testACRDomain) + _env.EXPECT().LiveConfig().AnyTimes().Return(tlc) + _env.EXPECT().IsLocalDevelopmentMode().AnyTimes().Return(tt.isLocalDevelopmentMode) + + uuidGen := deterministicuuid.NewTestUUIDGenerator(deterministicuuid.OPENSHIFT_VERSIONS) + dbOpenShiftVersions, _ := testdatabase.NewFakeOpenShiftVersions(uuidGen) + + // Add a known version to CosmosDB for precedence testing + dbOpenShiftVersions.Create(ctx, &api.OpenShiftVersionDocument{ + ID: "1", + OpenShiftVersion: &api.OpenShiftVersion{ + Properties: api.OpenShiftVersionProperties{ + Version: "4.14.38", + OpenShiftPullspec: "openshiftimage:1.0.0", + InstallerPullspec: "installerimage:1.0.0", + Enabled: true, + }, + }, + }) + + // Create subscription document with optional feature flag + subscription := &api.SubscriptionDocument{ + Subscription: &api.Subscription{ + Properties: &api.SubscriptionProperties{ + RegisteredFeatures: []api.RegisteredFeatureProfile{}, + }, + }, + } + + // Add arbitrary versions feature flag if enabled for this test + if tt.arbitraryVersionsEnabled { + subscription.Subscription.Properties.RegisteredFeatures = append( + subscription.Subscription.Properties.RegisteredFeatures, + api.RegisteredFeatureProfile{ + Name: api.FeatureFlagArbitraryVersions, + State: "Registered", + }, + ) + } + + m := &manager{ + env: _env, + dbOpenShiftVersions: dbOpenShiftVersions, + doc: &api.OpenShiftClusterDocument{ + Key: strings.ToLower(key), + OpenShiftCluster: &api.OpenShiftCluster{ + ID: key, + Properties: api.OpenShiftClusterProperties{ + ClusterProfile: api.ClusterProfile{ + Version: tt.version, + }, + }, + }, + }, + subscriptionDoc: subscription, + installViaHive: tt.installViaHive, + openShiftClusterDocumentVersioner: &openShiftClusterDocumentVersionerService{}, + } + + version, err := m.openShiftVersionFromVersion(ctx) + + if len(tt.wantErrString) > 0 { + assert.Equal(t, tt.wantErrString, err.Error(), "Unexpected error message") + assert.Nil(t, version, "Expected nil version on error") + } else { + assert.NoError(t, err, "Unexpected error") + assert.NotNil(t, version, "Expected non-nil version") + assert.Equal(t, tt.wantVersion, version.Properties.Version, "Unexpected version") + assert.Equal(t, tt.wantInstallerPullspec, version.Properties.InstallerPullspec, "Unexpected installer pullspec") + assert.Equal(t, tt.wantOpenShiftPullspec, version.Properties.OpenShiftPullspec, "Unexpected OpenShift pullspec") + assert.True(t, version.Properties.Enabled, "Expected version to be enabled") + } + }) + } +} diff --git a/pkg/cluster/version.go b/pkg/cluster/version.go index 6e2aea36880..7978f769a63 100644 --- a/pkg/cluster/version.go +++ b/pkg/cluster/version.go @@ -9,9 +9,12 @@ import ( "net/http" "strings" + "github.com/coreos/go-semver/semver" + "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/database" "github.com/Azure/ARO-RP/pkg/env" + "github.com/Azure/ARO-RP/pkg/util/feature" ) // openShiftClusterDocumentVersioner is the interface that validates and obtains the version from an OpenShiftClusterDocument. @@ -19,12 +22,20 @@ type openShiftClusterDocumentVersioner interface { // Get validates and obtains the OpenShift version of the OpenShiftClusterDocument doc using dbOpenShiftVersions, env and installViaHive parameters. Get(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool) (*api.OpenShiftVersion, error) + + // GetWithSubscription validates and obtains the OpenShift version with subscription support for AFEC flags. + GetWithSubscription(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool, subscription *api.SubscriptionDocument) (*api.OpenShiftVersion, error) } // openShiftClusterDocumentVersionerService is the default implementation of the openShiftClusterDocumentVersioner interface. type openShiftClusterDocumentVersionerService struct{} func (service *openShiftClusterDocumentVersionerService) Get(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool) (*api.OpenShiftVersion, error) { + // For backward compatibility, call the enhanced version without subscription data + return service.GetWithSubscription(ctx, doc, dbOpenShiftVersions, env, installViaHive, nil) +} + +func (service *openShiftClusterDocumentVersionerService) GetWithSubscription(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool, subscription *api.SubscriptionDocument) (*api.OpenShiftVersion, error) { requestedInstallVersion := doc.OpenShiftCluster.Properties.ClusterProfile.Version // TODO: Refactor to use changefeeds rather than querying the database every time @@ -35,14 +46,13 @@ func (service *openShiftClusterDocumentVersionerService) Get(ctx context.Context } activeOpenShiftVersions := make([]*api.OpenShiftVersion, 0) - for _, doc := range docs.OpenShiftVersionDocuments { - if doc.OpenShiftVersion.Properties.Enabled { - activeOpenShiftVersions = append(activeOpenShiftVersions, doc.OpenShiftVersion) + for _, versionDoc := range docs.OpenShiftVersionDocuments { + if versionDoc.OpenShiftVersion.Properties.Enabled { + activeOpenShiftVersions = append(activeOpenShiftVersions, versionDoc.OpenShiftVersion) } } - errUnsupportedVersion := api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", fmt.Sprintf("The requested OpenShift version '%s' is not supported.", requestedInstallVersion)) - + // First, try to find the version in CosmosDB (existing behavior) for _, active := range activeOpenShiftVersions { if requestedInstallVersion == active.Properties.Version { if installViaHive { @@ -58,9 +68,58 @@ func (service *openShiftClusterDocumentVersionerService) Get(ctx context.Context } } + // If not found in CosmosDB, check if arbitrary versions are enabled via AFEC flag or development environment + allowArbitraryVersions := env.IsLocalDevelopmentMode() || + (subscription != nil && feature.IsRegisteredForFeature(subscription.Subscription.Properties, api.FeatureFlagArbitraryVersions)) + + if allowArbitraryVersions { + return service.generateACRVersionSpec(ctx, requestedInstallVersion, env, installViaHive) + } + + errUnsupportedVersion := api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", fmt.Sprintf("The requested OpenShift version '%s' is not supported.", requestedInstallVersion)) return nil, errUnsupportedVersion } +// generateACRVersionSpec creates an OpenShiftVersion spec for arbitrary versions using ACR naming patterns +func (service *openShiftClusterDocumentVersionerService) generateACRVersionSpec(ctx context.Context, requestedVersion string, env env.Interface, installViaHive bool) (*api.OpenShiftVersion, error) { + // Parse the version to extract major.minor for ACR image tagging + parsedVersion, err := semver.NewVersion(requestedVersion) + if err != nil { + return nil, api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", fmt.Sprintf("The requested OpenShift version '%s' is not a valid semantic version.", requestedVersion)) + } + + // Generate ACR-based image specifications + majorMinor := fmt.Sprintf("%d.%d", parsedVersion.Major, parsedVersion.Minor) + acrDomain := env.ACRDomain() + + // Honor any pull spec override set + installerPullspec := fmt.Sprintf("%s/aro-installer:%s", acrDomain, majorMinor) + installerPullSpecOverride := env.LiveConfig().DefaultInstallerPullSpecOverride(ctx) + if installerPullSpecOverride != "" { + installerPullspec = installerPullSpecOverride + } + + // For OpenShift pullspec, use either ACR (for Hive) or quay.io (for traditional installer) + var openShiftPullspec string + if installViaHive { + // For Hive installations, use ACR domain + // This is a best-effort approach - the exact image may not exist + openShiftPullspec = fmt.Sprintf("%s/ocp-release:%s", acrDomain, requestedVersion) + } else { + // For traditional installations, use quay.io pattern + openShiftPullspec = fmt.Sprintf("quay.io/openshift-release-dev/ocp-release:%s", requestedVersion) + } + + return &api.OpenShiftVersion{ + Properties: api.OpenShiftVersionProperties{ + Version: requestedVersion, + OpenShiftPullspec: openShiftPullspec, + InstallerPullspec: installerPullspec, + Enabled: true, // Enabled by virtue of being generated for arbitrary versions + }, + }, nil +} + type FakeOpenShiftClusterDocumentVersionerService struct { expectedOpenShiftVersion *api.OpenShiftVersion expectedError error @@ -69,3 +128,7 @@ type FakeOpenShiftClusterDocumentVersionerService struct { func (fake *FakeOpenShiftClusterDocumentVersionerService) Get(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool) (*api.OpenShiftVersion, error) { return fake.expectedOpenShiftVersion, fake.expectedError } + +func (fake *FakeOpenShiftClusterDocumentVersionerService) GetWithSubscription(ctx context.Context, doc *api.OpenShiftClusterDocument, dbOpenShiftVersions database.OpenShiftVersions, env env.Interface, installViaHive bool, subscription *api.SubscriptionDocument) (*api.OpenShiftVersion, error) { + return fake.expectedOpenShiftVersion, fake.expectedError +} diff --git a/pkg/frontend/openshiftcluster_preflightvalidation.go b/pkg/frontend/openshiftcluster_preflightvalidation.go index 76de8529673..2062de92c9b 100644 --- a/pkg/frontend/openshiftcluster_preflightvalidation.go +++ b/pkg/frontend/openshiftcluster_preflightvalidation.go @@ -78,6 +78,19 @@ func (f *frontend) preflightValidation(w http.ResponseWriter, r *http.Request) { func (f *frontend) _preflightValidation(ctx context.Context, log *logrus.Entry, raw json.RawMessage, apiVersion string, resourceID string) api.ValidationResult { log.Infof("running preflight validation on resource: %s", resourceID) + + // Get subscription document for feature flag validation + subscription, err := f.getSubscriptionDocument(ctx, resourceID) + if err != nil { + log.Error(err) + return api.ValidationResult{ + Status: api.ValidationStatusFailed, + Error: &api.CloudErrorBody{ + Message: fmt.Sprintf("Failed to get subscription: %s", err.Error()), + }, + } + } + dbOpenShiftClusters, err := f.dbGroup.OpenShiftClusters() if err != nil { log.Error(err) @@ -131,7 +144,7 @@ func (f *frontend) _preflightValidation(ctx context.Context, log *logrus.Entry, }, } } - if err := f.validateInstallVersion(ctx, oc); err != nil { + if err := f.validateInstallVersion(ctx, oc, subscription); err != nil { return api.ValidationResult{ Status: api.ValidationStatusFailed, Error: &api.CloudErrorBody{ diff --git a/pkg/frontend/openshiftcluster_putorpatch.go b/pkg/frontend/openshiftcluster_putorpatch.go index ad1d9843fcf..83d88b44a26 100644 --- a/pkg/frontend/openshiftcluster_putorpatch.go +++ b/pkg/frontend/openshiftcluster_putorpatch.go @@ -277,7 +277,7 @@ func (f *frontend) _putOrPatchOpenShiftCluster(ctx context.Context, log *logrus. } if isCreate { - err = f.validateInstallVersion(ctx, doc.OpenShiftCluster) + err = f.validateInstallVersion(ctx, doc.OpenShiftCluster, subscription) if err != nil { return nil, err } diff --git a/pkg/frontend/validate.go b/pkg/frontend/validate.go index 403cd3e5c82..16fa74fbafd 100644 --- a/pkg/frontend/validate.go +++ b/pkg/frontend/validate.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/ARO-RP/pkg/api/validate" "github.com/Azure/ARO-RP/pkg/database/cosmosdb" utilnamespace "github.com/Azure/ARO-RP/pkg/util/namespace" + "github.com/Azure/ARO-RP/pkg/util/feature" ) func validateTerminalProvisioningState(state api.ProvisioningState) error { @@ -230,7 +231,7 @@ func validateAdminMasterVMSize(vmSize string) error { // validateInstallVersion validates the install version set in the clusterprofile.version // TODO convert this into static validation instead of this receiver function in the validation for frontend. -func (f *frontend) validateInstallVersion(ctx context.Context, oc *api.OpenShiftCluster) error { +func (f *frontend) validateInstallVersion(ctx context.Context, oc *api.OpenShiftCluster, subscription *api.SubscriptionDocument) error { f.ocpVersionsMu.RLock() // If this request is from an older API or the user did not specify // the version to install, use the default version. @@ -242,6 +243,19 @@ func (f *frontend) validateInstallVersion(ctx context.Context, oc *api.OpenShift _, err := semver.NewVersion(oc.Properties.ClusterProfile.Version) + // Check if arbitrary versions are enabled via AFEC flag or development environment + allowArbitraryVersions := f.env.IsLocalDevelopmentMode() || + (subscription != nil && feature.IsRegisteredForFeature(subscription.Subscription.Properties, api.FeatureFlagArbitraryVersions)) + + // If arbitrary versions are enabled, only validate that it's a valid semver format + if allowArbitraryVersions { + if err != nil { + return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", fmt.Sprintf("The requested OpenShift version '%s' is not a valid semantic version.", oc.Properties.ClusterProfile.Version)) + } + return nil + } + + // Default behavior: version must be in the enabled list AND be valid semver if !ok || err != nil { return api.NewCloudError(http.StatusBadRequest, api.CloudErrorCodeInvalidParameter, "properties.clusterProfile.version", fmt.Sprintf("The requested OpenShift version '%s' is invalid.", oc.Properties.ClusterProfile.Version)) } diff --git a/pkg/frontend/validate_test.go b/pkg/frontend/validate_test.go index 073f1156fba..dc44a63696a 100644 --- a/pkg/frontend/validate_test.go +++ b/pkg/frontend/validate_test.go @@ -9,9 +9,11 @@ import ( "strings" "testing" + "github.com/golang/mock/gomock" "k8s.io/apimachinery/pkg/runtime/schema" "github.com/Azure/ARO-RP/pkg/api" + mock_env "github.com/Azure/ARO-RP/pkg/util/mocks/env" utilerror "github.com/Azure/ARO-RP/test/util/error" ) @@ -272,11 +274,13 @@ func TestValidateInstallVersion(t *testing.T) { defaultOcpVersion := "4.12.25" for _, tt := range []struct { - test string - version string - availableVersions []string - wantVersion string - wantErr string + test string + version string + availableVersions []string + arbitraryVersionsEnabled bool + isLocalDevelopmentMode bool + wantVersion string + wantErr string }{ { test: "Valid and available OCP version specified returns no error", @@ -304,18 +308,90 @@ func TestValidateInstallVersion(t *testing.T) { version: "4.14.16+installerref-abcdef", availableVersions: []string{"4.12.25", "4.13.40", "4.14.16", "4.14.16+installerref-abcdef"}, }, + { + test: "Arbitrary valid semver version with feature flag enabled returns no error", + version: "4.15.0-custom.build.123", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + arbitraryVersionsEnabled: true, + }, + { + test: "Arbitrary invalid version with feature flag enabled returns error", + version: "not-a-valid-version", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + arbitraryVersionsEnabled: true, + wantErr: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version 'not-a-valid-version' is not a valid semantic version.", + }, + { + test: "Arbitrary version without feature flag enabled returns error", + version: "4.15.0-custom.build.123", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + arbitraryVersionsEnabled: false, + wantErr: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version '4.15.0-custom.build.123' is invalid.", + }, + { + test: "Arbitrary valid semver version with dev mode enabled returns no error", + version: "4.15.0-dev.branch.789", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + isLocalDevelopmentMode: true, + }, + { + test: "Arbitrary invalid version with dev mode enabled returns error", + version: "invalid-version-string", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + isLocalDevelopmentMode: true, + wantErr: "400: InvalidParameter: properties.clusterProfile.version: The requested OpenShift version 'invalid-version-string' is not a valid semantic version.", + }, + { + test: "Both AFEC flag and dev mode enabled - should work", + version: "4.16.0-rc.1", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + arbitraryVersionsEnabled: true, + isLocalDevelopmentMode: true, + }, + { + test: "Dev mode overrides normal validation for arbitrary versions", + version: "4.17.0-0.nightly-2024-12-01-000000", + availableVersions: []string{"4.12.25", "4.13.40", "4.14.16"}, + isLocalDevelopmentMode: true, + }, } { t.Run(tt.test, func(t *testing.T) { ctx := context.Background() + controller := gomock.NewController(t) + defer controller.Finish() enabledOcpVersions := map[string]*api.OpenShiftVersion{} for _, av := range tt.availableVersions { enabledOcpVersions[av] = &api.OpenShiftVersion{} } + mockEnv := mock_env.NewMockInterface(controller) + mockEnv.EXPECT().IsLocalDevelopmentMode().AnyTimes().Return(tt.isLocalDevelopmentMode) + f := frontend{ enabledOcpVersions: enabledOcpVersions, defaultOcpVersion: defaultOcpVersion, + env: mockEnv, + } + + // Create subscription document with optional feature flag + subscription := &api.SubscriptionDocument{ + Subscription: &api.Subscription{ + Properties: &api.SubscriptionProperties{ + RegisteredFeatures: []api.RegisteredFeatureProfile{}, + }, + }, + } + + // Add arbitrary versions feature flag if enabled for this test + if tt.arbitraryVersionsEnabled { + subscription.Subscription.Properties.RegisteredFeatures = append( + subscription.Subscription.Properties.RegisteredFeatures, + api.RegisteredFeatureProfile{ + Name: api.FeatureFlagArbitraryVersions, + State: "Registered", + }, + ) } oc := &api.OpenShiftCluster{ @@ -326,7 +402,7 @@ func TestValidateInstallVersion(t *testing.T) { }, } - err := f.validateInstallVersion(ctx, oc) + err := f.validateInstallVersion(ctx, oc, subscription) if tt.wantVersion != "" && oc.Properties.ClusterProfile.Version != tt.wantVersion { t.Errorf("wanted clusterdoc updated with version %s but got %s", tt.wantVersion, oc.Properties.ClusterProfile.Version) } diff --git a/pkg/util/cluster/cluster.go b/pkg/util/cluster/cluster.go index f4b65c586da..1e5d4e68c90 100644 --- a/pkg/util/cluster/cluster.go +++ b/pkg/util/cluster/cluster.go @@ -83,6 +83,7 @@ func (cc *ClusterConfig) IsLocalDevelopmentMode() bool { return strings.EqualFold(cc.RpMode, "development") } + type Cluster struct { log *logrus.Entry Config *ClusterConfig diff --git a/python/az/aro/azext_aro/_client_factory.py b/python/az/aro/azext_aro/_client_factory.py index a3ce4b98aa7..9fe11e01002 100644 --- a/python/az/aro/azext_aro/_client_factory.py +++ b/python/az/aro/azext_aro/_client_factory.py @@ -14,7 +14,7 @@ def cf_aro(cli_ctx, *_): if rp_mode_development(): opt_args = { - "endpoint": "https://localhost:8443/", + "base_url": "https://localhost:8443/", "connection_verify": False } urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) diff --git a/python/az/aro/azext_aro/custom.py b/python/az/aro/azext_aro/custom.py index 50aa15cdc9a..27cf5ffe337 100644 --- a/python/az/aro/azext_aro/custom.py +++ b/python/az/aro/azext_aro/custom.py @@ -506,7 +506,7 @@ def aro_get_versions(client, location): items = client.open_shift_versions.list(location) versions = [] for item in items: - versions.append(item.properties.version) + versions.append(item.version) return sorted(versions)