diff --git a/.gitignore b/.gitignore index 4031de17..1e6e8147 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ scripts/dashboard/ray-upstream scripts/dashboard/vllm-upstream scripts/dashboard/output -cluster-image-builder/downloader \ No newline at end of file +cluster-image-builder/downloader +scripts/builder/dist diff --git a/Makefile b/Makefile index a5800dd0..975dcfbf 100644 --- a/Makefile +++ b/Makefile @@ -308,3 +308,11 @@ sync-deploy-manifests: vendir ## Sync third-party dependencies using vendir sync-grafana-dashboards: vendir ## Sync grafana dashboards using vendir cd scripts/dashboard && $(VENDIR) sync && bash sync-grafana-dashboards.sh +.PHONY: sync-images-list +sync-images-list: ## Sync images list for building package + helm template neutree ./deploy/chart/neutree \ + --set api.image.tag=latest \ + --set core.image.tag=latest \ + --set dbScripts.image.tag=latest | \ + grep -Eoh 'image:\s*["]?[a-zA-Z0-9./_-]+:[a-zA-Z0-9._-]+["]?' | \ + awk '{print $$2}' | tr -d '"' | sort -u > scripts/builder/image-lists/controlplane/images.txt diff --git a/cmd/neutree-api/app/builder.go b/cmd/neutree-api/app/builder.go index 6b084308..7032e1da 100644 --- a/cmd/neutree-api/app/builder.go +++ b/cmd/neutree-api/app/builder.go @@ -10,6 +10,7 @@ import ( "github.com/neutree-ai/neutree/cmd/neutree-api/app/config" "github.com/neutree-ai/neutree/internal/middleware" "github.com/neutree-ai/neutree/internal/routes/auth" + "github.com/neutree-ai/neutree/internal/routes/credentials" "github.com/neutree-ai/neutree/internal/routes/models" "github.com/neutree-ai/neutree/internal/routes/proxies" "github.com/neutree-ai/neutree/internal/routes/system" @@ -41,6 +42,8 @@ func NewBuilder() *Builder { "system": SystemRouteFactory(system.RegisterSystemRoutes), // Auth route (no auth required for authentication itself) "auth": AuthRouteFactory(auth.RegisterAuthRoutes), + // Credentials route is used for get resource with sensitive data + "credentials": CredentialsRouteFactory(credentials.RegisterCredentialsRoutes), // PostgREST proxy routes // Auth middleware is applied to: // - Validate and pass-through JWT tokens to PostgREST @@ -72,7 +75,6 @@ func NewBuilder() *Builder { for name, middlewareInit := range defaultMiddlewareInits { b.middlewareInits[name] = middlewareInit } - // Register default middlewares to routes defaultRoutesToMiddlewares := map[string][]string{ "models": {"auth"}, @@ -94,6 +96,7 @@ func NewBuilder() *Builder { "rest/model-catalogs": {"auth"}, "rest/oem-configs": {"auth"}, "rest/rpc": {"auth"}, + "credentials": {"auth"}, } for route, middlewares := range defaultRoutesToMiddlewares { diff --git a/cmd/neutree-api/app/factory.go b/cmd/neutree-api/app/factory.go index 5890592b..99a21d8f 100644 --- a/cmd/neutree-api/app/factory.go +++ b/cmd/neutree-api/app/factory.go @@ -7,6 +7,7 @@ import ( "github.com/neutree-ai/neutree/cmd/neutree-api/app/config" "github.com/neutree-ai/neutree/internal/middleware" "github.com/neutree-ai/neutree/internal/routes/auth" + "github.com/neutree-ai/neutree/internal/routes/credentials" "github.com/neutree-ai/neutree/internal/routes/models" "github.com/neutree-ai/neutree/internal/routes/proxies" "github.com/neutree-ai/neutree/internal/routes/system" @@ -87,6 +88,19 @@ func AuthRouteFactory(register AuthRegisterFunc) RouteFactory { } } +type CredentialsRegisterFunc func(group *gin.RouterGroup, middlewares []gin.HandlerFunc, deps *credentials.Dependencies) + +func CredentialsRouteFactory(register CredentialsRegisterFunc) RouteFactory { + return func(deps *RouteOptions) error { + register(deps.Group, deps.Middlewares, &credentials.Dependencies{ + Storage: deps.Config.Storage, + StorageAccessURL: deps.Config.StorageAccessURL, + }) + + return nil + } +} + type MiddlewareOptions struct { Config *config.APIConfig } diff --git a/cmd/neutree-cli/app/cmd/cmd.go b/cmd/neutree-cli/app/cmd/cmd.go index b63ce1a9..4b3a0504 100644 --- a/cmd/neutree-cli/app/cmd/cmd.go +++ b/cmd/neutree-cli/app/cmd/cmd.go @@ -6,9 +6,9 @@ import ( "github.com/spf13/cobra" - "github.com/neutree-ai/neutree/cmd/neutree-cli/app/cmd/engine" "github.com/neutree-ai/neutree/cmd/neutree-cli/app/cmd/launch" "github.com/neutree-ai/neutree/cmd/neutree-cli/app/cmd/model" + "github.com/neutree-ai/neutree/cmd/neutree-cli/app/cmd/packageimport" ) func NewNeutreeCliCommand() *cobra.Command { @@ -39,7 +39,7 @@ Examples: neutreeCliCmd.AddCommand(launch.NewLaunchCmd()) neutreeCliCmd.AddCommand(model.NewModelCmd()) - neutreeCliCmd.AddCommand(engine.NewEngineCmd()) + neutreeCliCmd.AddCommand(packageimport.NewImportCmd()) return neutreeCliCmd } diff --git a/cmd/neutree-cli/app/cmd/engine/engine.go b/cmd/neutree-cli/app/cmd/engine/engine.go deleted file mode 100644 index c3540c06..00000000 --- a/cmd/neutree-cli/app/cmd/engine/engine.go +++ /dev/null @@ -1,47 +0,0 @@ -package engine - -import ( - "github.com/spf13/cobra" -) - -var ( - serverURL string - apiKey string -) - -func NewEngineCmd() *cobra.Command { - engineCmd := &cobra.Command{ - Use: "engine", - Short: "Manage Neutree engines", - Long: `Manage Neutree engine versions, including importing engine version packages. - -An engine version package contains: - • Engine version metadata - • Container images for different accelerators - • Values schema for configuration - • Deploy templates for different cluster types and modes - -Examples: - # Import an engine version package - neutree-cli engine import --package vllm-v0.5.0.tar.gz --registry registry.example.com --workspace default - - # Import without pushing images (for testing) - neutree-cli engine import --package vllm-v0.5.0.tar.gz --skip-image-push - - # Validate an engine version package - neutree-cli engine validate --package vllm-v0.5.0.tar.gz - - # Force overwrite existing version - neutree-cli engine import --package vllm-v0.5.0.tar.gz --registry registry.example.com --force -`, - } - - // Add global flags - engineCmd.PersistentFlags().StringVar(&serverURL, "server-url", "", "Server URL") - engineCmd.PersistentFlags().StringVar(&apiKey, "api-key", "", "API key") - - engineCmd.AddCommand(NewImportCmd()) - engineCmd.AddCommand(NewValidateCmd()) - - return engineCmd -} diff --git a/cmd/neutree-cli/app/cmd/engine/import.go b/cmd/neutree-cli/app/cmd/engine/import.go deleted file mode 100644 index 7d9deee3..00000000 --- a/cmd/neutree-cli/app/cmd/engine/import.go +++ /dev/null @@ -1,171 +0,0 @@ -package engine - -import ( - "context" - "fmt" - - "github.com/spf13/cobra" - "k8s.io/klog/v2" - - "github.com/neutree-ai/neutree/pkg/client" - "github.com/neutree-ai/neutree/pkg/engine" -) - -type ImportOptions struct { - packagePath string - registry string - workspace string - skipImagePush bool - force bool - extractPath string -} - -func NewImportCmd() *cobra.Command { - opts := &ImportOptions{} - - cmd := &cobra.Command{ - Use: "import", - Short: "Import an engine version package", - Long: `Import an engine version package into Neutree. - -This command will: - 1. Extract the engine version package - 2. Parse the manifest and validate the package structure - 3. Load container images from the package - 4. Push images to the specified registry (unless --skip-image-push is set) - 5. Update or create the engine definition with the new version - -The package must be a tar.gz, zip, or tar file containing: - • manifest.yaml - Package metadata and engine version definition - • images/*.tar - Container images for different accelerators - -Example manifest.yaml structure: ---- -manifest_version: "1.0" -package: - metadata: - engine_name: "vllm" - version: "v0.5.0" - description: "vLLM engine with CUDA support" - package_version: "1.0" - images: - - accelerator: "nvidia-gpu" - image_name: "neutree/vllm-cuda" - tag: "v0.5.0" - image_file: "images/vllm-cuda-v0.5.0.tar" - platform: "linux/amd64" - engine_version: - version: "v0.5.0" - values_schema: - type: "object" - properties: - gpu_memory_utilization: - type: "number" - default: 0.9 - deploy_template: - kubernetes: - default: base64-encoded-yaml-string - -Examples: - # Import with image push - neutree-cli engine import --package vllm-v0.5.0.tar.gz \ - --registry test \ - --workspace default \ - --server-url http://localhost:8080 \ - --api-key your-api-key - - # Import without pushing images (for testing/development) - neutree-cli engine import --package vllm-v0.5.0.tar.gz --skip-image-push - - # Force overwrite existing version - neutree-cli engine import --package vllm-v0.5.0.tar.gz \ - neutree-cli engine import --package vllm-v0.5.0.tar.gz \ - --registry test \ - --workspace default \ - --server-url http://localhost:8080 \ - --api-key your-api-key \ - --force -`, - RunE: func(cmd *cobra.Command, args []string) error { - return runImport(opts) - }, - } - - cmd.Flags().StringVarP(&opts.packagePath, "package", "p", "", "Path to the engine version package file (required)") - cmd.Flags().StringVarP(&opts.registry, "registry", "r", "", "Container registry to push images to (e.g., registry.example.com)") - cmd.Flags().StringVarP(&opts.workspace, "workspace", "w", "default", "Workspace to import the engine to") - cmd.Flags().BoolVar(&opts.skipImagePush, "skip-image-push", false, "Skip pushing images to registry") - cmd.Flags().BoolVarP(&opts.force, "force", "f", false, "Force overwrite if engine version already exists") - cmd.Flags().StringVar(&opts.extractPath, "extract-path", "", "Path to extract package to (default: temporary directory)") - - _ = cmd.MarkFlagRequired("package") - - return cmd -} - -func runImport(opts *ImportOptions) error { - ctx := context.Background() - - // Validate API connection - if serverURL == "" { - return fmt.Errorf("API URL is required (use --api-url or set NEUTREE_API_URL env var)") - } - - // Initialize API client - klog.Info("Initializing API client...") - - clientOpts := []client.ClientOption{} - if apiKey != "" { - clientOpts = append(clientOpts, client.WithAPIKey(apiKey)) - } - - apiClient := client.NewClient(serverURL, clientOpts...) - - // Create importer with engines service - importer, err := engine.NewImporter(apiClient) - if err != nil { - return err - } - - // Prepare import options - importOpts := &engine.ImportOptions{ - PackagePath: opts.packagePath, - ImageRegistry: opts.registry, - Workspace: opts.workspace, - SkipImagePush: opts.skipImagePush, - Force: opts.force, - ExtractPath: opts.extractPath, - } - - // Import the package - klog.Infof("Importing engine version package: %s", opts.packagePath) - - result, err := importer.Import(ctx, importOpts) - if err != nil { - return fmt.Errorf("failed to import engine version package: %w", err) - } - - // Print results - fmt.Printf("\n✓ Successfully imported engine version package\n\n") - fmt.Printf("Engine Name: %s\n", result.EngineName) - fmt.Printf("Version: %s\n", result.Version) - fmt.Printf("Engine Updated: %v\n", result.EngineUpdated) - - if len(result.ImagesImported) > 0 { - fmt.Printf("\nImages Imported:\n") - - for _, img := range result.ImagesImported { - fmt.Printf(" • %s\n", img) - } - } - - if len(result.Errors) > 0 { - fmt.Printf("\nWarnings/Errors:\n") - - for _, e := range result.Errors { - fmt.Printf(" ⚠ %s\n", e.Error()) - } - } - - return nil -} diff --git a/cmd/neutree-cli/app/cmd/engine/validate.go b/cmd/neutree-cli/app/cmd/engine/validate.go deleted file mode 100644 index b71a7705..00000000 --- a/cmd/neutree-cli/app/cmd/engine/validate.go +++ /dev/null @@ -1,62 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/spf13/cobra" - "k8s.io/klog/v2" - - "github.com/neutree-ai/neutree/pkg/engine" -) - -type ValidateOptions struct { - PackagePath string -} - -func NewValidateCmd() *cobra.Command { - opts := &ValidateOptions{} - - cmd := &cobra.Command{ - Use: "validate", - Short: "Validate an engine version package", - Long: `Validate an engine version package without importing it. - -This command will: - 1. Extract the package to a temporary directory - 2. Parse and validate the manifest file - 3. Check that all referenced image files exist - 4. Verify the package structure is correct - -This is useful for testing packages before importing them. - -Examples: - # Validate a package - neutree-cli engine validate --package vllm-v0.5.0.tar.gz - - # Check package structure - neutree-cli engine validate -p /path/to/engine-package.tar.gz -`, - RunE: func(cmd *cobra.Command, args []string) error { - return runValidate(opts) - }, - } - - cmd.Flags().StringVarP(&opts.PackagePath, "package", "p", "", "Path to the engine version package file (required)") - _ = cmd.MarkFlagRequired("package") - - return cmd -} - -func runValidate(opts *ValidateOptions) error { - klog.Infof("Validating engine version package: %s", opts.PackagePath) - - // Validate the package - if err := engine.ValidatePackage(opts.PackagePath); err != nil { - return fmt.Errorf("package validation failed: %w", err) - } - - fmt.Printf("\n✓ Package validation successful\n\n") - fmt.Printf("The package at '%s' is valid and ready to be imported.\n", opts.PackagePath) - - return nil -} diff --git a/cmd/neutree-cli/app/cmd/packageimport/cluster.go b/cmd/neutree-cli/app/cmd/packageimport/cluster.go new file mode 100644 index 00000000..fa445b16 --- /dev/null +++ b/cmd/neutree-cli/app/cmd/packageimport/cluster.go @@ -0,0 +1,120 @@ +package packageimport + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + "github.com/neutree-ai/neutree/internal/cli/packageimport" + "github.com/neutree-ai/neutree/pkg/client" +) + +type ClusterImportOptions struct { + packagePath string + extractPath string +} + +func NewClusterImportCmd() *cobra.Command { + opts := &ClusterImportOptions{} + + cmd := &cobra.Command{ + Use: "cluster", + Short: "Import a cluster image package with cluster container images", + Long: `Import a cluster image package into Neutree. + +This command imports container images required for clusters. It performs the following steps: + 1. Extracts the cluster image package archive + 2. Parses and validates the manifest.yaml structure + 3. Loads container images from the package + 4. Pushes images to the configured image registry in the workspace + +Package Requirements: +The package must be a tar.gz archive containing: + • manifest.yaml - Package metadata and image definitions + • images/*.tar - Container image tar files + +Example manifest.yaml: +--- +manifest_version: "1.0" + +metadata: + description: "Cluster image package for Neutree" + version: "v1.0.0" + +images: + - image_name: "neutree/neutree-serve" + tag: "v1.0.0" + image_file: "images/all-images.tar" +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runClusterImport(opts) + }, + } + + cmd.Flags().StringVarP(&opts.packagePath, "package", "p", "", "Path to the cluster image package file (required)") + cmd.Flags().StringVar(&opts.extractPath, "extract-path", "/tmp", "Path to extract package to (default: temporary directory)") + + _ = cmd.MarkFlagRequired("package") + + return cmd +} + +func runClusterImport(opts *ClusterImportOptions) error { + ctx := context.Background() + + // Validate API connection + if serverURL == "" { + return fmt.Errorf("API URL is required (use --api-url or set NEUTREE_API_URL env var)") + } + + // Initialize API client + klog.Info("Initializing API client...") + + clientOpts := []client.ClientOption{} + if apiKey != "" { + clientOpts = append(clientOpts, client.WithAPIKey(apiKey)) + } + + apiClient := client.NewClient(serverURL, clientOpts...) + + importer := packageimport.NewImporter(apiClient) + + // Prepare import options + importOpts := &packageimport.ImportOptions{ + PackagePath: opts.packagePath, + ImageRegistry: registry, + Workspace: workspace, + ExtractPath: opts.extractPath, + } + + // Import the package + klog.Infof("Importing cluster package: %s", opts.packagePath) + + result, err := importer.Import(ctx, importOpts) + if err != nil { + return fmt.Errorf("failed to import cluster package: %w", err) + } + + // Print results + fmt.Printf("\n✓ Successfully imported cluster package\n\n") + + if len(result.ImagesImported) > 0 { + fmt.Printf("\nImages Imported:\n") + + for _, img := range result.ImagesImported { + fmt.Printf(" • %s\n", img) + } + } + + if len(result.Errors) > 0 { + fmt.Printf("\nWarnings/Errors:\n") + + for _, e := range result.Errors { + fmt.Printf(" ⚠ %s\n", e.Error()) + } + } + + return nil +} diff --git a/cmd/neutree-cli/app/cmd/packageimport/controlplane.go b/cmd/neutree-cli/app/cmd/packageimport/controlplane.go new file mode 100644 index 00000000..85126d14 --- /dev/null +++ b/cmd/neutree-cli/app/cmd/packageimport/controlplane.go @@ -0,0 +1,125 @@ +package packageimport + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + "github.com/neutree-ai/neutree/internal/cli/packageimport" +) + +type ControlPlaneImportOptions struct { + packagePath string + extractPath string + + mirrorRegistry string + registryUsername string + registryPassword string + importLocal bool +} + +func NewControlPlaneImportCmd() *cobra.Command { + opts := &ControlPlaneImportOptions{} + + cmd := &cobra.Command{ + Use: "controlplane", + Short: "Import a control plane image package with Neutree system components", + Long: `Import a control plane image package into Neutree. + +This command imports container images for Neutree control plane components. It performs: + 1. Extracts the control plane image package archive + 2. Parses and validates the manifest.yaml structure + 3. Loads container images from the package + 4. Optionally pushes images to a mirror registry or loads them locally + +Package Requirements: +The package must be a tar.gz archive containing: + • manifest.yaml - Package metadata and image definitions + • images/*.tar - Container image tar files for control plane components + +Example manifest.yaml: +--- +manifest_version: "1.0" + +metadata: + description: "Control plane image package for Neutree" + version: "v1.0.0" + +images: + - image_name: "neutree/neutree-api" + tag: "v1.0.0" + image_file: "images/all-images.tar" + +Note: This command does not require API connection and can run standalone. +Use --local flag to skip registry push and only load images to local Docker. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runControlPlaneImport(opts) + }, + } + + cmd.Flags().StringVarP(&opts.packagePath, "package", "p", "", "Path to the control plane version package file (required)") + cmd.Flags().StringVar(&opts.extractPath, "extract-path", "/tmp", "Path to extract package to (default: temporary directory)") + cmd.Flags().StringVar(&opts.mirrorRegistry, "mirror-registry", "", "Container image registry to push images to (required)") + cmd.Flags().StringVar(&opts.registryUsername, "registry-username", "", "Username for the container image registry (if required)") + cmd.Flags().StringVar(&opts.registryPassword, "registry-password", "", "Password for the container image registry (if required)") + cmd.Flags().BoolVar(&opts.importLocal, "local", false, "Skip pushing images to the registry, only load images locally") + + _ = cmd.MarkFlagRequired("package") + + return cmd +} + +func runControlPlaneImport(opts *ControlPlaneImportOptions) error { + ctx := context.Background() + + // ControlPlane no need to create apiclient + importer := packageimport.NewImporter(nil) + + // Prepare import options + importOpts := &packageimport.ImportOptions{ + PackagePath: opts.packagePath, + Workspace: workspace, + ExtractPath: opts.extractPath, + } + + // if skipImagePush is not set, configure registry info + if !opts.importLocal { + importOpts.MirrorRegistry = opts.mirrorRegistry + importOpts.RegistryUser = opts.registryUsername + importOpts.RegistryPassword = opts.registryPassword + } else { + importOpts.SkipImagePush = true + } + + // Import the package + klog.Infof("Importing cluster package: %s", opts.packagePath) + + result, err := importer.Import(ctx, importOpts) + if err != nil { + return fmt.Errorf("failed to import control plane package: %w", err) + } + + // Print results + fmt.Printf("\n✓ Successfully imported control plane package\n\n") + + if len(result.ImagesImported) > 0 { + fmt.Printf("\nImages Imported:\n") + + for _, img := range result.ImagesImported { + fmt.Printf(" • %s\n", img) + } + } + + if len(result.Errors) > 0 { + fmt.Printf("\nWarnings/Errors:\n") + + for _, e := range result.Errors { + fmt.Printf(" ⚠ %s\n", e.Error()) + } + } + + return nil +} diff --git a/cmd/neutree-cli/app/cmd/packageimport/engine.go b/cmd/neutree-cli/app/cmd/packageimport/engine.go new file mode 100644 index 00000000..4cfc9ffc --- /dev/null +++ b/cmd/neutree-cli/app/cmd/packageimport/engine.go @@ -0,0 +1,158 @@ +package packageimport + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + engine "github.com/neutree-ai/neutree/internal/cli/packageimport" + "github.com/neutree-ai/neutree/pkg/client" +) + +type EngineImportOptions struct { + packagePath string + skipImagePush bool + force bool + extractPath string +} + +func NewEngineImportCmd() *cobra.Command { + opts := &EngineImportOptions{} + + cmd := &cobra.Command{ + Use: "engine", + Short: "Import an engine version package with model serving images and engine definitions", + Long: `Import an engine version package into Neutree. + +This command imports engine versions that define model serving capabilities. It performs: + 1. Extracts the engine version package archive + 2. Parses and validates the manifest.yaml structure + 3. Loads container images for different accelerators (CUDA, ROCm, CPU) + 4. Pushes images to the configured image registry in the workspace + 5. Creates or updates the engine definition with the new version + +Package Requirements: +The package must be a tar.gz archive containing: + • manifest.yaml - Engine metadata, version definitions, and container images + • images/*.tar - Container image tar files for different accelerators + +Example manifest.yaml: +--- +manifest_version: "1.0" + +metadata: + description: "vLLM engine package for LLM inference" + version: "v0.6.0" + +engines: + - name: "vllm" + supported_tasks: ["text-generation"] + engine_versions: + - version: "v0.6.0" + images: + - image_name: "neutree/vllm-cuda" + tag: "v0.6.0" + accelerator: "nvidia.com/gpu" + - image_name: "neutree/vllm-rocm" + tag: "v0.6.0" + accelerator: "amd.com/gpu" + +images: + - image_name: "neutree/vllm-cuda" + tag: "v0.6.0" + image_file: "images/vllm-cuda.tar" + - image_name: "neutree/vllm-rocm" + tag: "v0.6.0" + image_file: "images/vllm-rocm.tar" + +Use --force to overwrite existing engine versions. +Use --skip-image-push to only update engine definitions without pushing images. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runEngineImport(opts) + }, + } + + cmd.Flags().StringVarP(&opts.packagePath, "package", "p", "", "Path to the engine version package file (required)") + cmd.Flags().BoolVar(&opts.skipImagePush, "skip-image-push", false, "Skip pushing images to registry") + cmd.Flags().BoolVarP(&opts.force, "force", "f", false, "Force overwrite if engine version already exists") + cmd.Flags().StringVar(&opts.extractPath, "extract-path", "", "Path to extract package to (default: temporary directory)") + + _ = cmd.MarkFlagRequired("package") + + return cmd +} + +func runEngineImport(opts *EngineImportOptions) error { + ctx := context.Background() + + // Validate API connection + if serverURL == "" { + return fmt.Errorf("API URL is required (use --api-url or set NEUTREE_API_URL env var)") + } + + // Initialize API client + klog.Info("Initializing API client...") + + clientOpts := []client.ClientOption{} + if apiKey != "" { + clientOpts = append(clientOpts, client.WithAPIKey(apiKey)) + } + + apiClient := client.NewClient(serverURL, clientOpts...) + + // Create importer with engines service + importer := engine.NewImporter(apiClient) + + // Prepare import options + importOpts := &engine.ImportOptions{ + PackagePath: opts.packagePath, + ImageRegistry: registry, + Workspace: workspace, + SkipImagePush: opts.skipImagePush, + SkipImageLoad: opts.skipImagePush, + Force: opts.force, + ExtractPath: opts.extractPath, + } + + // Import the package + klog.Infof("Importing engine version package: %s", opts.packagePath) + + result, err := importer.Import(ctx, importOpts) + if err != nil { + return fmt.Errorf("failed to import engine version package: %w", err) + } + + // Print results + fmt.Printf("\n✓ Successfully imported engine version package\n\n") + + if len(result.ImagesImported) > 0 { + fmt.Printf("\nImages Imported:\n") + + for _, img := range result.ImagesImported { + fmt.Printf(" • %s\n", img) + } + } + + if len(result.EnginesImported) > 0 { + fmt.Printf("\nEngines Imported:\n") + + for _, eng := range result.EnginesImported { + for _, ver := range eng.EngineVersions { + fmt.Printf(" • %s:%s\n", eng.Name, ver.Version) + } + } + } + + if len(result.Errors) > 0 { + fmt.Printf("\nWarnings/Errors:\n") + + for _, e := range result.Errors { + fmt.Printf(" ⚠ %s\n", e.Error()) + } + } + + return nil +} diff --git a/cmd/neutree-cli/app/cmd/packageimport/import.go b/cmd/neutree-cli/app/cmd/packageimport/import.go new file mode 100644 index 00000000..9eb36eaa --- /dev/null +++ b/cmd/neutree-cli/app/cmd/packageimport/import.go @@ -0,0 +1,41 @@ +package packageimport + +import "github.com/spf13/cobra" + +var ( + serverURL string + apiKey string + registry string + workspace string +) + +func NewImportCmd() *cobra.Command { + importCmd := &cobra.Command{ + Use: "import", + Short: "Import Neutree packages including engines, clusters, and control plane components", + Long: `Import Neutree packages into the system. + +This command provides subcommands to import different types of Neutree packages: + • engine - Import engine version packages with model serving images + • cluster - Import cluster image packages for compute clusters + • controlplane - Import control plane component images + • validate - Validate package structure without importing + +All packages follow a standard format containing a manifest.yaml and container images. +Use the appropriate subcommand based on the package type you want to import. +`, + } + + // Add global flags + importCmd.PersistentFlags().StringVar(&serverURL, "server-url", "", "Server URL") + importCmd.PersistentFlags().StringVar(&apiKey, "api-key", "", "API key") + importCmd.PersistentFlags().StringVar(®istry, "registry", "", "Image registry") + importCmd.PersistentFlags().StringVar(&workspace, "workspace", "default", "Workspace") + + importCmd.AddCommand(NewClusterImportCmd()) + importCmd.AddCommand(NewEngineImportCmd()) + importCmd.AddCommand(NewValidateCmd()) + importCmd.AddCommand(NewControlPlaneImportCmd()) + + return importCmd +} diff --git a/cmd/neutree-cli/app/cmd/packageimport/validate.go b/cmd/neutree-cli/app/cmd/packageimport/validate.go new file mode 100644 index 00000000..a399fcd9 --- /dev/null +++ b/cmd/neutree-cli/app/cmd/packageimport/validate.go @@ -0,0 +1,66 @@ +package packageimport + +import ( + "fmt" + + "github.com/spf13/cobra" + "k8s.io/klog/v2" + + "github.com/neutree-ai/neutree/internal/cli/packageimport" +) + +type ValidateOptions struct { + PackagePath string +} + +func NewValidateCmd() *cobra.Command { + opts := &ValidateOptions{} + + cmd := &cobra.Command{ + Use: "validate", + Short: "Validate a Neutree package structure without importing it", + Long: `Validate a Neutree package without performing the actual import. + +This command performs validation checks on a Neutree package to ensure it can be imported successfully: + 1. Extracts the package to a temporary directory + 2. Parses and validates the manifest.yaml structure + 3. Verifies that all referenced image files exist in the package + 4. Checks manifest schema and required fields + +This is useful for: + • Testing package integrity before importing + • Debugging package creation issues + • Verifying package format compliance + +The command does not: + • Load Docker images + • Push images to registries + • Create or modify engine definitions + • Require API connectivity + +This command can be used with engine, cluster, and control plane packages, but performs only generic manifest validation (not type-specific checks). +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runValidate(opts) + }, + } + + cmd.Flags().StringVarP(&opts.PackagePath, "package", "p", "", "Path to the neutree package file (required)") + _ = cmd.MarkFlagRequired("package") + + return cmd +} + +func runValidate(opts *ValidateOptions) error { + klog.Infof("Validating neutree package: %s", opts.PackagePath) + + // Validate the package + if err := packageimport.ValidatePackage(opts.PackagePath); err != nil { + return fmt.Errorf("package validation failed: %w", err) + } + + fmt.Printf("\n✓ Package validation successful\n\n") + fmt.Printf("The package at '%s' is valid and ready to be imported.\n", opts.PackagePath) + + return nil +} diff --git a/db/migrations/037_sensitive_field_permission.down.sql b/db/migrations/037_sensitive_field_permission.down.sql new file mode 100644 index 00000000..ecd01858 --- /dev/null +++ b/db/migrations/037_sensitive_field_permission.down.sql @@ -0,0 +1,13 @@ +UPDATE api.roles +SET spec = ROW( + (spec).preset_key, + array_remove( + array_remove( + (spec).permissions, + 'image_registry:read-credentials'::api.permission_action + ), + 'model_registry:read-credentials'::api.permission_action + ), + 'cluster:read-credentials'::api.permission_action +)::api.role_spec +WHERE (metadata).name = 'admin'; \ No newline at end of file diff --git a/db/migrations/037_sensitive_field_permission.up.sql b/db/migrations/037_sensitive_field_permission.up.sql new file mode 100644 index 00000000..10a78e49 --- /dev/null +++ b/db/migrations/037_sensitive_field_permission.up.sql @@ -0,0 +1,7 @@ +-- Add read-credentials permission for all resources that contain sensitive credentials +-- These permissions allow reading sensitive fields like passwords, tokens, kubeconfig, etc. + +-- Core resources with credentials +ALTER TYPE api.permission_action ADD VALUE 'image_registry:read-credentials'; +ALTER TYPE api.permission_action ADD VALUE 'model_registry:read-credentials'; +ALTER TYPE api.permission_action ADD VALUE 'cluster:read-credentials'; \ No newline at end of file diff --git a/deploy/chart/neutree/values.yaml b/deploy/chart/neutree/values.yaml index bc21bad1..2cd9a58a 100644 --- a/deploy/chart/neutree/values.yaml +++ b/deploy/chart/neutree/values.yaml @@ -245,7 +245,7 @@ grafana: type: Recreate image: # -- The Docker registry - registry: "" + registry: "docker.io" # -- Docker image repository repository: grafana/grafana # Overrides the Grafana image tag whose default is the chart appVersion diff --git a/pkg/engine/extractor.go b/internal/cli/packageimport/extractor.go similarity index 99% rename from pkg/engine/extractor.go rename to internal/cli/packageimport/extractor.go index 2b726b9c..d2b9ca20 100644 --- a/pkg/engine/extractor.go +++ b/internal/cli/packageimport/extractor.go @@ -1,4 +1,4 @@ -package engine +package packageimport import ( "archive/tar" diff --git a/pkg/engine/image_pusher.go b/internal/cli/packageimport/image_pusher.go similarity index 60% rename from pkg/engine/image_pusher.go rename to internal/cli/packageimport/image_pusher.go index c29a3e75..72b46cdd 100644 --- a/pkg/engine/image_pusher.go +++ b/internal/cli/packageimport/image_pusher.go @@ -1,33 +1,25 @@ -package engine +package packageimport import ( "context" - "encoding/base64" - "encoding/json" "fmt" "os" "strings" "github.com/docker/docker/api/types/image" - "github.com/docker/docker/api/types/registry" "github.com/docker/docker/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/pkg/errors" "k8s.io/klog/v2" - - v1 "github.com/neutree-ai/neutree/api/v1" - "github.com/neutree-ai/neutree/internal/util" - apiclient "github.com/neutree-ai/neutree/pkg/client" ) // ImagePusher handles pushing container images to registries type ImagePusher struct { - apiClient *apiclient.Client dockerClient *client.Client } // NewImagePusher creates a new ImagePusher -func NewImagePusher(apiClient *apiclient.Client) (*ImagePusher, error) { +func NewImagePusher() (*ImagePusher, error) { // Create Docker client dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { @@ -35,55 +27,61 @@ func NewImagePusher(apiClient *apiclient.Client) (*ImagePusher, error) { } return &ImagePusher{ - apiClient: apiClient, dockerClient: dockerClient, }, nil } -// LoadAndPushImages loads images from tar files and pushes them to the registry -// It handles getting the image registry and pushing images -func (p *ImagePusher) LoadAndPushImages(ctx context.Context, workspace, registryName string, manifest *PackageManifest, extractedPath string) ([]string, error) { - // Get image registry - imageRegistry, err := p.apiClient.ImageRegistries.Get(workspace, registryName) - if err != nil { - return nil, errors.Wrap(err, "failed to get image registry") - } +func (p *ImagePusher) LoadImages(ctx context.Context, manifest *PackageManifest, extractedPath string) error { + klog.Infof("Loading images from extracted path: %s", extractedPath) - klog.Infof("Using image registry: %s", imageRegistry.Metadata.Name) - - // Load and push images - return p.loadAndPushImages(ctx, imageRegistry, manifest, extractedPath) + // Load images + return p.loadImages(ctx, manifest, extractedPath) } -// loadAndPushImages is the internal implementation that loads and pushes images -func (p *ImagePusher) loadAndPushImages(ctx context.Context, imageRegistry *v1.ImageRegistry, manifest *PackageManifest, extractedPath string) ([]string, error) { - var pushedImages []string - var errs []error +// loadImages is the internal implementation that loads images +func (p *ImagePusher) loadImages(ctx context.Context, manifest *PackageManifest, extractedPath string) error { + alreadyLoadedImageFile := make(map[string]bool) - targetImagePrefix, err := util.GetImagePrefix(imageRegistry) - if err != nil { - return nil, errors.Wrap(err, "failed to get image prefix") - } - - // Initialize the Images map if it doesn't exist - if manifest.Package.EngineVersion.Images == nil { - manifest.Package.EngineVersion.Images = make(map[string]*v1.EngineImage) - } - - for _, imgSpec := range manifest.Package.Images { + for _, imgSpec := range manifest.Images { imagePath := fmt.Sprintf("%s/%s", extractedPath, imgSpec.ImageFile) + // Skip loading if already loaded + if alreadyLoadedImageFile[imgSpec.ImageFile] { + klog.Infof("Image file %s already loaded, skipping load", imgSpec.ImageFile) + continue + } + // Load the image klog.Infof("Loading image from %s", imagePath) if err := p.loadImage(ctx, imagePath); err != nil { - errs = append(errs, errors.Wrapf(err, "failed to load image: %s", imagePath)) - continue + return errors.Wrapf(err, "failed to load image: %s", imagePath) } + alreadyLoadedImageFile[imgSpec.ImageFile] = true + } + + return nil +} + +func (p *ImagePusher) PushImagesToMirrorRegistry(ctx context.Context, + mirrorRegistry string, registryAuth string, manifest *PackageManifest) ([]string, error) { + klog.Infof("Pushing images to mirror registry: %s", mirrorRegistry) + + // Load and push images + return p.pushImages(ctx, mirrorRegistry, registryAuth, manifest) +} + +// loadAndPushImages is the internal implementation that loads and pushes images +func (p *ImagePusher) pushImages(ctx context.Context, mirrorRegistry string, registryAuth string, + manifest *PackageManifest) ([]string, error) { + var pushedImages []string + var errs []error + + for _, imgSpec := range manifest.Images { // Build the original and target image references originalImage := fmt.Sprintf("%s:%s", imgSpec.ImageName, imgSpec.Tag) - targetImage := p.buildTargetImage(targetImagePrefix, imgSpec) + targetImage := p.buildTargetImage(mirrorRegistry, imgSpec) // Tag the image with the target registry klog.Infof("Tagging image %s as %s", originalImage, targetImage) @@ -96,22 +94,13 @@ func (p *ImagePusher) loadAndPushImages(ctx context.Context, imageRegistry *v1.I // Push the image to the registry klog.Infof("Pushing image %s to registry", targetImage) - if err := p.pushImage(ctx, imageRegistry, targetImage); err != nil { + if err := p.pushImage(ctx, targetImage, registryAuth); err != nil { errs = append(errs, errors.Wrapf(err, "failed to push image")) continue } pushedImages = append(pushedImages, targetImage) klog.Infof("Successfully pushed image: %s", targetImage) - - // Update the manifest's EngineVersion.Images with the processed image name (without old registry) - processedImageName := p.extractImageNameWithoutRegistry(imgSpec.ImageName) - manifest.Package.EngineVersion.Images[imgSpec.Accelerator] = &v1.EngineImage{ - ImageName: processedImageName, - Tag: imgSpec.Tag, - } - - klog.V(2).Infof("Updated manifest image for accelerator %s: %s:%s", imgSpec.Accelerator, processedImageName, imgSpec.Tag) } if len(errs) > 0 { @@ -124,25 +113,10 @@ func (p *ImagePusher) loadAndPushImages(ctx context.Context, imageRegistry *v1.I // buildTargetImage builds the target image reference with registry and repo func (p *ImagePusher) buildTargetImage(imagePrefix string, imgSpec *ImageSpec) string { // Remove any existing registry from the image name - imageName := p.extractImageNameWithoutRegistry(imgSpec.ImageName) + imageName := extractImageNameWithoutRegistry(imgSpec.ImageName) return fmt.Sprintf("%s/%s:%s", imagePrefix, imageName, imgSpec.Tag) } -// extractImageNameWithoutRegistry removes any existing registry prefix from the image name -func (p *ImagePusher) extractImageNameWithoutRegistry(imageName string) string { - // Check if there's a registry prefix (contains / before any other structure) - if idx := strings.Index(imageName, "/"); idx != -1 { - // Check if this is a registry prefix (contains . or :) - firstPart := imageName[:idx] - if strings.Contains(firstPart, ".") || strings.Contains(firstPart, ":") { - // This is a registry, remove it - return imageName[idx+1:] - } - } - // No registry prefix found, return as-is - return imageName -} - // loadImage loads a Docker image from a tar file func (p *ImagePusher) loadImage(ctx context.Context, imagePath string) error { // Open the tar file @@ -180,30 +154,10 @@ func (p *ImagePusher) tagImage(ctx context.Context, sourceImage, targetImage str } // pushImage pushes a Docker image to a registry -func (p *ImagePusher) pushImage(ctx context.Context, imageRegistry *v1.ImageRegistry, imageName string) error { +func (p *ImagePusher) pushImage(ctx context.Context, imageName string, registryAuth string) error { // Create push options with auth - pushOptions := image.PushOptions{} - - // Add auth if credentials are available - userName, password := util.GetImageRegistryAuthInfo(imageRegistry) - if userName != "" && password != "" { - registryHost, err := util.GetImageRegistryHost(imageRegistry) - if err != nil { - return errors.Wrap(err, "failed to get registry host") - } - - authConfig := registry.AuthConfig{ - Username: userName, - Password: password, - ServerAddress: registryHost, - } - - authConfigBytes, err := json.Marshal(authConfig) - if err != nil { - return errors.Wrap(err, "failed to marshal auth config") - } - - pushOptions.RegistryAuth = base64.URLEncoding.EncodeToString(authConfigBytes) + pushOptions := image.PushOptions{ + RegistryAuth: registryAuth, } // Push the image @@ -237,3 +191,18 @@ func (w *klogWriter) Write(p []byte) (int, error) { return len(p), nil } + +// extractImageNameWithoutRegistry removes any existing registry prefix from the image name +func extractImageNameWithoutRegistry(imageName string) string { + // Check if there's a registry prefix (contains / before any other structure) + if idx := strings.Index(imageName, "/"); idx != -1 { + // Check if this is a registry prefix (contains . or :) + firstPart := imageName[:idx] + if strings.Contains(firstPart, ".") || strings.Contains(firstPart, ":") { + // This is a registry, remove it + return imageName[idx+1:] + } + } + // No registry prefix found, return as-is + return imageName +} diff --git a/internal/cli/packageimport/image_pusher_test.go b/internal/cli/packageimport/image_pusher_test.go new file mode 100644 index 00000000..5c3cdbf3 --- /dev/null +++ b/internal/cli/packageimport/image_pusher_test.go @@ -0,0 +1,129 @@ +package packageimport + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImagePusherBuildTargetImage(t *testing.T) { + pusher, err := NewImagePusher() // No API client needed for testing buildTargetImage + require.NoError(t, err, "Failed to create ImagePusher") + + tests := []struct { + name string + imagePrefix string + imgSpec *ImageSpec + expected string + }{ + { + name: "with prefix", + imagePrefix: "registry.example.com/neutree", + imgSpec: &ImageSpec{ + ImageName: "vllm-cuda", + Tag: "v0.5.0", + }, + expected: "registry.example.com/neutree/vllm-cuda:v0.5.0", + }, + { + name: "without prefix", + imagePrefix: "registry.example.com", + imgSpec: &ImageSpec{ + ImageName: "vllm-cuda", + Tag: "v0.5.0", + }, + expected: "registry.example.com/vllm-cuda:v0.5.0", + }, + { + name: "remove existing registry", + imagePrefix: "new-registry.com/neutree", + imgSpec: &ImageSpec{ + ImageName: "old-registry.com/vllm-cuda", + Tag: "v0.5.0", + }, + expected: "new-registry.com/neutree/vllm-cuda:v0.5.0", + }, + { + name: "remove existing registry with port", + imagePrefix: "new-registry.com/neutree", + imgSpec: &ImageSpec{ + ImageName: "old-registry.com:5000/vllm-cuda", + Tag: "v0.5.0", + }, + expected: "new-registry.com/neutree/vllm-cuda:v0.5.0", + }, + { + name: "keep organization name without dots", + imagePrefix: "registry.example.com/neutree", + imgSpec: &ImageSpec{ + ImageName: "myorg/vllm-cuda", + Tag: "v0.5.0", + }, + expected: "registry.example.com/neutree/myorg/vllm-cuda:v0.5.0", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := pusher.buildTargetImage(tt.imagePrefix, tt.imgSpec) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestImagePusherExtractImageNameWithoutRegistry(t *testing.T) { + tests := []struct { + name string + imageName string + expected string + }{ + { + name: "simple image name", + imageName: "vllm-cuda", + expected: "vllm-cuda", + }, + { + name: "image with organization", + imageName: "myorg/vllm-cuda", + expected: "myorg/vllm-cuda", + }, + { + name: "image with registry domain", + imageName: "registry.example.com/vllm-cuda", + expected: "vllm-cuda", + }, + { + name: "image with registry and org", + imageName: "registry.example.com/myorg/vllm-cuda", + expected: "myorg/vllm-cuda", + }, + { + name: "image with registry port", + imageName: "registry.example.com:5000/vllm-cuda", + expected: "vllm-cuda", + }, + { + name: "image with registry port and org", + imageName: "registry.example.com:5000/myorg/vllm-cuda", + expected: "myorg/vllm-cuda", + }, + { + name: "dockerhub official image", + imageName: "nginx", + expected: "nginx", + }, + { + name: "dockerhub user image", + imageName: "username/image", + expected: "username/image", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := extractImageNameWithoutRegistry(tt.imageName) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/internal/cli/packageimport/importer.go b/internal/cli/packageimport/importer.go new file mode 100644 index 00000000..d7a78e35 --- /dev/null +++ b/internal/cli/packageimport/importer.go @@ -0,0 +1,296 @@ +package packageimport + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + + "github.com/docker/docker/api/types/registry" + "github.com/pkg/errors" + "k8s.io/klog/v2" + + v1 "github.com/neutree-ai/neutree/api/v1" + "github.com/neutree-ai/neutree/internal/util" + "github.com/neutree-ai/neutree/pkg/client" +) + +// Importer handles importing packages +type Importer struct { + apiClient *client.Client + extractor *Extractor + parser *Parser + validator *Validator +} + +// NewImporter creates a new Importer +func NewImporter(apiClient *client.Client) *Importer { + return &Importer{ + apiClient: apiClient, + extractor: NewExtractor(), + parser: NewParser(), + validator: NewValidator(), + } +} + +// Import imports a package +func (i *Importer) Import(ctx context.Context, opts *ImportOptions) (*ImportResult, error) { + result := &ImportResult{ + ImagesImported: []string{}, + Errors: []error{}, + } + + // Validate options + if err := i.validateOptions(opts); err != nil { + return nil, errors.Wrap(err, "invalid import options") + } + + // Create temporary directory if not specified + if opts.ExtractPath == "" { + tempDir, err := os.MkdirTemp("", "neutree-*") + if err != nil { + return nil, errors.Wrap(err, "failed to create temporary directory") + } + + opts.ExtractPath = tempDir + + defer os.RemoveAll(tempDir) + } + + klog.Infof("Extracting package to %s", opts.ExtractPath) + + // Extract the package + if err := i.extractor.Extract(opts.PackagePath, opts.ExtractPath); err != nil { + return nil, errors.Wrap(err, "failed to extract package") + } + + // Parse the manifest + klog.Info("Parsing manifest") + + manifest, err := i.parser.ParseManifest(opts.ExtractPath) + if err != nil { + return nil, errors.Wrap(err, "failed to parse manifest") + } + + // Push images + klog.Info("Pushing images to registry") + + pushedImages, err := i.pushImages(ctx, opts, manifest) + if err != nil { + return nil, errors.Wrap(err, "failed to push images to registry") + } + + result.ImagesImported = pushedImages + + if len(manifest.Engines) > 0 { + klog.Infof("Engines to import: %d", len(manifest.Engines)) + + for _, engine := range manifest.Engines { + if err := i.updateEngine(ctx, engine, opts); err != nil { + result.Errors = append(result.Errors, err) + return result, errors.Wrap(err, "failed to process engine upload") + } + + klog.Infof("Successfully imported engine %s", engine.Name) + } + + result.EnginesImported = manifest.Engines + } + + return result, nil +} + +func (i *Importer) pushImages(ctx context.Context, opts *ImportOptions, manifest *PackageManifest) ([]string, error) { + klog.Info("Loading and pushing images to registry") + + if opts.SkipImageLoad { + klog.Info("Skipping image load as per configuration") + return []string{}, nil + } + + imagePusher, err := NewImagePusher() + if err != nil { + return nil, errors.Wrap(err, "failed to create image pusher") + } + + err = imagePusher.LoadImages(ctx, manifest, opts.ExtractPath) + if err != nil { + return nil, errors.Wrap(err, "failed to load images") + } + + if opts.SkipImagePush { + klog.Info("Skipping image push as per configuration") + return []string{}, nil + } + + mirrorRegistry := opts.MirrorRegistry + user, token := opts.RegistryUser, opts.RegistryPassword + + if opts.ImageRegistry != "" { + imgRegistrys, err := i.apiClient.ImageRegistries.List(client.ImageRegistryListOptions{ + Workspace: opts.Workspace, + Name: opts.ImageRegistry, + WithCreds: true, + }) + if err != nil { + return nil, errors.Wrap(err, "failed to get image registry") + } + + if len(imgRegistrys) == 0 { + return nil, errors.Errorf("image registry %s not found", opts.ImageRegistry) + } + + targetRegistry := &imgRegistrys[0] + + klog.Info("Pushing image to registry") + + mirrorRegistry, err = util.GetImagePrefix(targetRegistry) + if err != nil { + return nil, errors.Wrap(err, "failed to get image prefix") + } + + user, token = util.GetImageRegistryAuthInfo(targetRegistry) + } + + authConfig := registry.AuthConfig{ + Username: user, + Password: token, + ServerAddress: mirrorRegistry, + } + + authConfigBytes, err := json.Marshal(authConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal auth config") + } + + registryAuth := base64.URLEncoding.EncodeToString(authConfigBytes) + + pushedImages, err := imagePusher.PushImagesToMirrorRegistry(ctx, mirrorRegistry, registryAuth, manifest) + if err != nil { + return nil, errors.Wrap(err, "failed to push images to mirror registry") + } + + return pushedImages, nil +} + +// validateOptions validates the import options +func (i *Importer) validateOptions(opts *ImportOptions) error { + if opts.PackagePath == "" { + return errors.New("package path is required") + } + + if _, err := os.Stat(opts.PackagePath); os.IsNotExist(err) { + return errors.Errorf("package file not found: %s", opts.PackagePath) + } + + if opts.SkipImageLoad && !opts.SkipImagePush { + return errors.New("cannot skip image load when image push is enabled") + } + + if !opts.SkipImagePush { + if (opts.Workspace == "" || opts.ImageRegistry == "") && (opts.MirrorRegistry == "" || opts.RegistryUser == "" || opts.RegistryPassword == "") { + return errors.New("image registry config is required when not skipping image push") + } + } + + return nil +} + +// updateEngine updates the engine with the new version +func (i *Importer) updateEngine(_ context.Context, engineMetadata *EngineMetadata, opts *ImportOptions) error { + newEngine := &v1.Engine{ + APIVersion: "v1", + Kind: "Engine", + Metadata: &v1.Metadata{ + Name: engineMetadata.Name, + Workspace: opts.Workspace, + }, + Spec: &v1.EngineSpec{ + Versions: engineMetadata.EngineVersions, + SupportedTasks: engineMetadata.SupportedTasks, + }, + } + + engineList, err := i.apiClient.Engines.List(client.ListOptions{ + Workspace: opts.Workspace, + Name: engineMetadata.Name, + }) + + if err != nil { + return errors.Wrap(err, "failed to check if engine exists") + } + + if len(engineList) == 0 { + // Create new engine + return i.apiClient.Engines.Create(opts.Workspace, newEngine) + } + + existedEngine := &engineList[0] + + // Update existing engine + // Check if version already exists and remove it if force is enabled + for _, newVersion := range newEngine.Spec.Versions { + found := false + + for idx, oldVersion := range existedEngine.Spec.Versions { + if oldVersion.Version == newVersion.Version && opts.Force { + // merge + existedEngine.Spec.Versions[idx] = util.MergeEngineVersion(oldVersion, newVersion) + found = true + + break + } + } + + if !found { + existedEngine.Spec.Versions = append(existedEngine.Spec.Versions, newVersion) + } + } + + return i.apiClient.Engines.Update(opts.Workspace, existedEngine.GetID(), existedEngine) +} + +// Validator handles validation of packages +type Validator struct { + extractor *Extractor + parser *Parser +} + +// NewValidator creates a new Validator +func NewValidator() *Validator { + return &Validator{ + extractor: NewExtractor(), + parser: NewParser(), + } +} + +// ValidatePackage validates a package without importing it +func (v *Validator) ValidatePackage(packagePath string) error { + // Create temporary directory + tempDir, err := os.MkdirTemp("", "neutree-validate-*") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + defer os.RemoveAll(tempDir) + + // Extract the package + if err := v.extractor.Extract(packagePath, tempDir); err != nil { + return errors.Wrap(err, "failed to extract package") + } + + // Parse the manifest + _, err = v.parser.ParseManifest(tempDir) + if err != nil { + return errors.Wrap(err, "failed to parse manifest") + } + + klog.Info("Package validation successful") + + return nil +} + +// ValidatePackage is a convenience function that creates a validator and validates a package +func ValidatePackage(packagePath string) error { + v := NewValidator() + return v.ValidatePackage(packagePath) +} diff --git a/internal/cli/packageimport/importer_test.go b/internal/cli/packageimport/importer_test.go new file mode 100644 index 00000000..eebb4959 --- /dev/null +++ b/internal/cli/packageimport/importer_test.go @@ -0,0 +1,123 @@ +package packageimport + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExtractorValidation(t *testing.T) { + extractor := NewExtractor() + + // Test invalid package format + err := extractor.Extract("invalid.xyz", "/tmp/test") + assert.Error(t, err) + assert.Contains(t, err.Error(), "unsupported package format") +} + +func TestImportOptionsValidation(t *testing.T) { + importer := NewImporter(nil) + + tests := []struct { + name string + opts *ImportOptions + setupFunc func() string // Returns temp file path + cleanupFunc func(string) + expectError bool + errorMsg string + }{ + { + name: "valid options with skip image push", + setupFunc: func() string { + tmpFile, _ := os.CreateTemp("", "test-*.tar.gz") + tmpFile.Close() + return tmpFile.Name() + }, + cleanupFunc: func(path string) { + os.Remove(path) + }, + opts: &ImportOptions{ + PackagePath: "", // Will be set by setupFunc + SkipImagePush: true, + }, + expectError: false, + }, + { + name: "missing package path", + opts: &ImportOptions{ + PackagePath: "", + }, + expectError: true, + errorMsg: "package path is required", + }, + { + name: "package file not found", + opts: &ImportOptions{ + PackagePath: "/nonexistent/package.tar.gz", + }, + expectError: true, + errorMsg: "package file not found", + }, + { + name: "with registry when not skipping push", + setupFunc: func() string { + tmpFile, _ := os.CreateTemp("", "test-*.tar.gz") + tmpFile.Close() + return tmpFile.Name() + }, + cleanupFunc: func(path string) { + os.Remove(path) + }, + opts: &ImportOptions{ + PackagePath: "", // Will be set by setupFunc + SkipImagePush: false, + ImageRegistry: "registry.example.com", + Workspace: "default", + }, + expectError: false, + }, + { + name: "with mirror registry when not skipping push", + setupFunc: func() string { + tmpFile, _ := os.CreateTemp("", "test-*.tar.gz") + tmpFile.Close() + return tmpFile.Name() + }, + cleanupFunc: func(path string) { + os.Remove(path) + }, + opts: &ImportOptions{ + PackagePath: "", // Will be set by setupFunc + SkipImagePush: false, + MirrorRegistry: "registry.mirror.com", + RegistryUser: "user", + RegistryPassword: "pass", + }, + expectError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setupFunc != nil { + tmpPath := tt.setupFunc() + tt.opts.PackagePath = tmpPath + if tt.cleanupFunc != nil { + defer tt.cleanupFunc(tmpPath) + } + } + + err := importer.validateOptions(tt.opts) + if tt.expectError { + require.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/internal/cli/packageimport/parser.go b/internal/cli/packageimport/parser.go new file mode 100644 index 00000000..59f11a14 --- /dev/null +++ b/internal/cli/packageimport/parser.go @@ -0,0 +1,157 @@ +package packageimport + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" +) + +const ( + // ManifestFileName is the name of the manifest file in the package + ManifestFileName = "manifest.yaml" +) + +// Parser handles parsing of engine version package manifests +type Parser struct{} + +// NewParser creates a new Parser +func NewParser() *Parser { + return &Parser{} +} + +// ParseManifest parses the manifest file from the extracted package directory +func (p *Parser) ParseManifest(extractedPath string) (*PackageManifest, error) { + manifestPath := filepath.Join(extractedPath, ManifestFileName) + if _, err := os.Stat(manifestPath); err == nil { + manifests, err := p.parseYAMLManifest(manifestPath) + if err != nil { + return nil, errors.Wrap(err, "failed to parse YAML manifest") + } + + err = p.validateManifest(manifests, extractedPath) + if err != nil { + return nil, errors.Wrap(err, "manifest validation failed") + } + + return manifests, nil + } + + return nil, errors.New("manifest file not found") +} + +// parseYAMLManifest parses a YAML manifest file +func (p *Parser) parseYAMLManifest(path string) (*PackageManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, errors.Wrap(err, "failed to read manifest file") + } + + var manifest PackageManifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal YAML manifest") + } + + // Process each engine version + for idx := range manifest.Engines { + for vidx := range manifest.Engines[idx].EngineVersions { + ev := manifest.Engines[idx].EngineVersions[vidx] + // Decode base64-encoded values schema if present + valueSchemaStr, ok := ev.ValuesSchema["values_schema_base64"] + if ok { + valueSchemaBase64Str, ok := valueSchemaStr.(string) + if !ok { + return nil, errors.New("invalid values_schema_base64 format in manifest") + } + + valueSchemaJson, err := base64.StdEncoding.DecodeString(valueSchemaBase64Str) + if err != nil { + return nil, errors.Wrap(err, "failed to decode values schema from base64") + } + + var decodedSchema map[string]interface{} + if err := json.Unmarshal(valueSchemaJson, &decodedSchema); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal values schema JSON") + } + + ev.ValuesSchema = decodedSchema + } + + // set default tag and extract image name without registry + for i := range ev.Images { + if ev.Images[i].Tag == "" { + ev.Images[i].Tag = ev.Version + } + + ev.Images[i].ImageName = extractImageNameWithoutRegistry(ev.Images[i].ImageName) + } + + manifest.Engines[idx].EngineVersions[vidx] = ev + } + } + + return &manifest, nil +} + +func (p *Parser) validateManifest(manifest *PackageManifest, extractedPath string) error { + if manifest == nil { + return errors.New("manifest is nil") + } + + if len(manifest.Images) == 0 { + return errors.New("no images specified") + } + + // Validate each image spec + for i, img := range manifest.Images { + if img.ImageName == "" { + return errors.Errorf("image %d: image name is empty", i) + } + + if img.Tag == "" { + return errors.Errorf("image %d: tag is empty", i) + } + + imagePath := filepath.Join(extractedPath, img.ImageFile) + if _, err := os.Stat(imagePath); os.IsNotExist(err) { + return errors.Errorf("image file not found: %s", img.ImageFile) + } + } + + for idx := range manifest.Engines { + if err := p.validateEngineConfig(manifest.Engines[idx]); err != nil { + return errors.Wrap(err, "invalid engine configuration in manifest") + } + } + + return nil +} + +func (p *Parser) validateEngineConfig(engine *EngineMetadata) error { + if engine == nil { + return errors.New("engine is nil") + } + + if engine.Name == "" { + return errors.New("engine name is empty") + } + + if len(engine.EngineVersions) == 0 { + return errors.New("no engine versions defined") + } + + for _, ev := range engine.EngineVersions { + if ev == nil { + return errors.New("engine version definition is nil") + } + + if ev.Version == "" { + return errors.New("engine version field is empty") + } + } + + return nil +} diff --git a/internal/cli/packageimport/parser_test.go b/internal/cli/packageimport/parser_test.go new file mode 100644 index 00000000..5111d1b3 --- /dev/null +++ b/internal/cli/packageimport/parser_test.go @@ -0,0 +1,361 @@ +package packageimport + +import ( + "os" + "testing" + + v1 "github.com/neutree-ai/neutree/api/v1" + "github.com/stretchr/testify/assert" +) + +func TestParserParseManifest(t *testing.T) { + parser := NewParser() + + manifestContent := ` +manifest_version: "1.0" + +images: + - image_name: "vllm" + tag: "v0.10.2" + image_file: "images/vllm.tar" + +engines: +- name: vllm + engine_versions: + - version: "v0.10.2" + + values_schema: + values_schema_base64: "eyJ0ZXN0IjoidmFsdWVzIn0K" + supported_tasks: + - "text-generation" + + images: + nvidia_gpu: + image_name: "vllm" + tag: "v0.10.2" +` + + manifestContentWithoutImageTag := ` +manifest_version: "1.0" + +images: + - image_name: "vllm" + tag: "v0.10.2" + image_file: "images/vllm.tar" + +engines: +- name: vllm + engine_versions: + - version: "v0.10.2" + + values_schema: + values_schema_base64: "eyJ0ZXN0IjoidmFsdWVzIn0K" + supported_tasks: + - "text-generation" + + images: + nvidia_gpu: + image_name: "vllm" + tag: "" +` + + manifestContentWithInvalidValueScheme := ` +manifest_version: "1.0" + +images: + - image_name: "vllm" + tag: "v0.10.2" + image_file: "images/vllm.tar" + +engines: +- name: vllm + engine_versions: + - version: "v0.10.2" + + values_schema: + values_schema_base64: "invalid-base64" + supported_tasks: + - "text-generation" + + images: + nvidia_gpu: + image_name: "vllm" + tag: "" +` + tests := []struct { + name string + content string + expectManifest *PackageManifest + expectError bool + }{ + { + name: "valid manifest", + content: manifestContent, + expectManifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "vllm", + Tag: "v0.10.2", + ImageFile: "images/vllm.tar", + }, + }, + Engines: []*EngineMetadata{ + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v0.10.2", + Images: map[string]*v1.EngineImage{ + "nvidia_gpu": { + ImageName: "vllm", + Tag: "v0.10.2", + }, + }, + ValuesSchema: map[string]interface{}{ + "test": "value", + }, + }, + }, + SupportedTasks: []string{"text_generation"}, + }, + }, + }, + expectError: false, + }, + { + name: "manifest with missing image tag", + content: manifestContentWithoutImageTag, + expectManifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "vllm", + Tag: "v0.10.2", + ImageFile: "images/vllm.tar", + }, + }, + Engines: []*EngineMetadata{ + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v0.10.2", + Images: map[string]*v1.EngineImage{ + "nvidia_gpu": { + ImageName: "vllm", + Tag: "v0.10.2", + }, + }, + ValuesSchema: map[string]interface{}{ + "test": "value", + }, + }, + }, + SupportedTasks: []string{"text_generation"}, + }, + }, + }, + expectError: false, + }, + + { + name: "without manifest file", + content: "", + expectManifest: nil, + expectError: true, + }, + { + name: "manifest with invalid yaml", + content: "invalid_yaml: [unclosed_list", + expectManifest: nil, + expectError: true, + }, + { + name: "manifest with invalid values schema", + content: manifestContentWithInvalidValueScheme, + expectManifest: nil, + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + manifestPath := dir + "/manifest.yaml" + if tt.content != "" { + err := os.WriteFile(manifestPath, []byte(tt.content), 0644) + assert.NoError(t, err, "Failed to write manifest file") + } + + manifest, err := parser.parseYAMLManifest(manifestPath) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ObjectsAreEqual(tt.expectManifest, manifest) + } + }) + } +} + +func TestParserValidateManifest(t *testing.T) { + dir := t.TempDir() + err := os.MkdirAll(dir+"/images", os.ModePerm) + assert.NoError(t, err, "Failed to create test images directory") + + // Create a dummy image file for testing + dummyImagePath := dir + "/images/test.tar" + _, err = os.Create(dummyImagePath) + assert.NoError(t, err, "Failed to create dummy image file") + parser := NewParser() + + tests := []struct { + name string + manifest *PackageManifest + expectError bool + errorMsg string + }{ + { + name: "valid manifest", + manifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "test/image", + Tag: "v1.0.0", + ImageFile: "images/test.tar", + }, + }, + Engines: []*EngineMetadata{ + + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v1.0.0", + Images: map[string]*v1.EngineImage{ + "nvidia_gpu": { + ImageName: "vllm", + Tag: "v0.8.5", + }, + }, + }, + }, + SupportedTasks: []string{v1.TextGenerationModelTask}, + }, + }, + }, + expectError: false, + }, + { + name: "missing image name", + manifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "", + Tag: "v1.0.0", + ImageFile: "images/test.tar", + }, + }, + Engines: []*EngineMetadata{ + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v1.0.0", + }, + }, + }, + }, + }, + expectError: true, + errorMsg: "image 0: image name is empty", + }, + { + name: "missing image file", + manifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "test/image", + Tag: "v1.0.0", + ImageFile: "images/test-no-image-file.tar", + }, + }, + Engines: []*EngineMetadata{ + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v1.0.0", + }, + }, + }, + }, + }, + expectError: true, + errorMsg: "image file not found", + }, + { + name: "missing engine name", + manifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{ + { + ImageName: "test/image", + Tag: "v1.0.0", + ImageFile: "images/test.tar", + }, + }, + Engines: []*EngineMetadata{ + { + Name: "", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v1.0.0", + }, + }, + }, + }, + }, + expectError: true, + errorMsg: "engine name is empty", + }, + { + name: "no images", + manifest: &PackageManifest{ + ManifestVersion: "1.0", + Images: []*ImageSpec{}, + Engines: []*EngineMetadata{ + { + Name: "test-engine", + EngineVersions: []*v1.EngineVersion{ + { + Version: "v1.0.0", + }, + }, + }, + }, + }, + expectError: true, + errorMsg: "no images specified", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + err := parser.validateManifest(tt.manifest, dir) + if tt.expectError { + assert.Error(t, err) + if tt.errorMsg != "" { + assert.Contains(t, err.Error(), tt.errorMsg) + } + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/pkg/engine/types.go b/internal/cli/packageimport/types.go similarity index 60% rename from pkg/engine/types.go rename to internal/cli/packageimport/types.go index 0751af57..97a23fd7 100644 --- a/pkg/engine/types.go +++ b/internal/cli/packageimport/types.go @@ -1,53 +1,49 @@ -package engine +package packageimport import ( v1 "github.com/neutree-ai/neutree/api/v1" ) -// EngineVersionPackage represents the complete structure of an engine version package -type EngineVersionPackage struct { - // Metadata contains information about the engine version package +type PackageManifest struct { + // ManifestVersion is the version of the manifest format + ManifestVersion string `json:"manifest_version" yaml:"manifest_version"` + + // Metadata contains information about the package Metadata *PackageMetadata `json:"metadata" yaml:"metadata"` - // Images contains the list of container images for different accelerators + // Images contains the list of container images Images []*ImageSpec `json:"images" yaml:"images"` - // EngineVersion contains the engine version definition - EngineVersion *v1.EngineVersion `json:"engine_version" yaml:"engine_version"` + // Engines contains the list of engines need to be imported + Engines []*EngineMetadata `json:"engines" yaml:"engines"` } -// PackageMetadata contains metadata about the engine version package -type PackageMetadata struct { - // Name of the engine (e.g., "vllm", "llama-cpp") - EngineName string `json:"engine_name" yaml:"engine_name"` +type EngineMetadata struct { + // Name of the engine + Name string `json:"name" yaml:"name"` - // Version of the engine (e.g., "v0.5.0", "v1.0.0") - Version string `json:"version" yaml:"version"` + EngineVersions []*v1.EngineVersion `json:"engine_versions" yaml:"engine_versions"` - // Description provides details about this engine version - Description string `json:"description,omitempty" yaml:"description,omitempty"` + SupportedTasks []string `json:"supported_tasks,omitempty" yaml:"supported_tasks,omitempty"` +} +// PackageMetadata contains metadata about the engine version package +type PackageMetadata struct { // Author of the package Author string `json:"author,omitempty" yaml:"author,omitempty"` // CreatedAt timestamp CreatedAt string `json:"created_at,omitempty" yaml:"created_at,omitempty"` - // PackageVersion is the version of the package format itself - PackageVersion string `json:"package_version" yaml:"package_version"` + // Version is the version of the neutree format itself + Version string `json:"version" yaml:"version"` // Tags for categorizing the package Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` - - // SupportTasks lists the tasks supported by this engine - SupportTasks []string `json:"support_tasks,omitempty" yaml:"support_tasks,omitempty"` } // ImageSpec describes a container image for a specific accelerator type ImageSpec struct { - // Accelerator type (e.g., "nvidia-gpu", "amd-gpu", "cpu") - Accelerator string `json:"accelerator" yaml:"accelerator"` - // ImageName is the full image reference without tag // Example: "neutree/vllm-cuda" ImageName string `json:"image_name" yaml:"image_name"` @@ -55,10 +51,6 @@ type ImageSpec struct { // Tag is the image tag Tag string `json:"tag" yaml:"tag"` - // ImageFile is the path to the image tarball in the package - // Example: "images/vllm-cuda-v0.5.0.tar" - ImageFile string `json:"image_file" yaml:"image_file"` - // Platform specifies the platform (e.g., "linux/amd64", "linux/arm64") Platform string `json:"platform,omitempty" yaml:"platform,omitempty"` @@ -67,15 +59,9 @@ type ImageSpec struct { // Digest is the image digest Digest string `json:"digest,omitempty" yaml:"digest,omitempty"` -} - -// PackageManifest is the root manifest file in the engine version package -type PackageManifest struct { - // ManifestVersion is the version of the manifest format - ManifestVersion string `json:"manifest_version" yaml:"manifest_version"` - // Package contains the engine version package details - Package *EngineVersionPackage `json:"package" yaml:"package"` + // ImageFile is the path to the image file within the package + ImageFile string `json:"image_file" yaml:"image_file"` } // ImportOptions contains options for importing an engine version package @@ -86,12 +72,24 @@ type ImportOptions struct { // ImageRegistry is the target image registry to push images to ImageRegistry string + // MirrorRegistry is an optional mirror registry to push images to + MirrorRegistry string + + // RegistryUser is the username for the mirror image registry + RegistryUser string + + // RegistryPassword is the password for the mirror image registry + RegistryPassword string + // Workspace is the workspace to import the engine to Workspace string // SkipImagePush skips pushing images to the registry SkipImagePush bool + // SkipImageLoad skips loading images from files + SkipImageLoad bool + // Force forces the import even if the engine version already exists Force bool @@ -101,17 +99,14 @@ type ImportOptions struct { // ImportResult contains the result of importing an engine version package type ImportResult struct { - // EngineName is the name of the engine - EngineName string - - // Version is the version of the engine - Version string - // ImagesImported is the list of images that were imported ImagesImported []string - // EngineUpdated indicates whether the engine was updated - EngineUpdated bool + // EnginesImported is the list of engines that were imported + EnginesImported []*EngineMetadata + + // Version is the imported package version + Version string // Errors contains any errors that occurred during import Errors []error diff --git a/internal/routes/credentials/credentials.go b/internal/routes/credentials/credentials.go new file mode 100644 index 00000000..cd17f431 --- /dev/null +++ b/internal/routes/credentials/credentials.go @@ -0,0 +1,57 @@ +package credentials + +import ( + "github.com/gin-gonic/gin" + + "github.com/neutree-ai/neutree/internal/middleware" + "github.com/neutree-ai/neutree/internal/routes/proxies" + "github.com/neutree-ai/neutree/pkg/storage" +) + +// Dependencies defines the dependencies for credentials handlers +type Dependencies struct { + Storage storage.Storage + StorageAccessURL string +} + +// RegisterCredentialsRoutes registers credentials retrieval routes +// This is a separate API group to ensure explicit intent when accessing sensitive data +// For every resource, we will check the permission before return the sensitive data +// Now only support clusters, image registries, and model registries +func RegisterCredentialsRoutes(group *gin.RouterGroup, middlewares []gin.HandlerFunc, deps *Dependencies) { + credGroup := group.Group("/credentials") + credGroup.Use(middlewares...) + + proxyDeps := &proxies.Dependencies{ + Storage: deps.Storage, + StorageAccessURL: deps.StorageAccessURL, + } + + // Cluster credentials (kubeconfig, SSH keys, etc.) + credGroup.GET("/clusters", + middleware.RequirePermission("cluster:read-credentials", middleware.PermissionDependencies{ + Storage: deps.Storage, + }), + handleResourceCredentials(proxyDeps, "clusters")) + + // Image registry credentials (username, password, token) + credGroup.GET("/image_registries", + middleware.RequirePermission("image_registry:read-credentials", middleware.PermissionDependencies{ + Storage: deps.Storage, + }), + handleResourceCredentials(proxyDeps, "image_registries")) + + // Model registry credentials + credGroup.GET("/model_registries", + middleware.RequirePermission("model_registry:read-credentials", middleware.PermissionDependencies{ + Storage: deps.Storage, + }), + handleResourceCredentials(proxyDeps, "model_registries")) +} + +func handleResourceCredentials(deps *proxies.Dependencies, tabelName string) gin.HandlerFunc { + return func(c *gin.Context) { + proxyHandler := proxies.CreateProxyHandler(deps.StorageAccessURL, tabelName, proxies.CreatePostgrestAuthModifier(c)) + proxyHandler(c) + } +} diff --git a/internal/routes/proxies/proxies.go b/internal/routes/proxies/proxies.go index 91db7468..75e4f442 100644 --- a/internal/routes/proxies/proxies.go +++ b/internal/routes/proxies/proxies.go @@ -293,7 +293,7 @@ func handleRayDashboardProxy(deps *Dependencies) gin.HandlerFunc { } } -func createPostgrestAuthModifier(c *gin.Context) func(*http.Request) { +func CreatePostgrestAuthModifier(c *gin.Context) func(*http.Request) { return func(req *http.Request) { if postgrestToken, exists := middleware.GetPostgrestToken(c); exists && postgrestToken != "" { req.Header.Set("Authorization", "Bearer "+postgrestToken) @@ -310,7 +310,7 @@ func handlePostgrestRPCProxy(deps *Dependencies) gin.HandlerFunc { path = "rpc/" + path - proxyHandler := CreateProxyHandler(deps.StorageAccessURL, path, createPostgrestAuthModifier(c)) + proxyHandler := CreateProxyHandler(deps.StorageAccessURL, path, CreatePostgrestAuthModifier(c)) proxyHandler(c) } } diff --git a/internal/routes/proxies/proxies_test.go b/internal/routes/proxies/proxies_test.go index ffc9bbfe..080e100d 100644 --- a/internal/routes/proxies/proxies_test.go +++ b/internal/routes/proxies/proxies_test.go @@ -357,7 +357,7 @@ func TestHandleRayDashboardProxy_MissingDashboardURL(t *testing.T) { mockStorage.AssertExpectations(t) } -// TestCreatePostgrestAuthModifier tests the createPostgrestAuthModifier function +// TestCreatePostgrestAuthModifier tests the CreatePostgrestAuthModifier function func TestCreatePostgrestAuthModifier(t *testing.T) { gin.SetMode(gin.TestMode) @@ -371,7 +371,7 @@ func TestCreatePostgrestAuthModifier(t *testing.T) { req.Header.Set("Authorization", "sk_original_api_key") // Apply the modifier - modifier := createPostgrestAuthModifier(c) + modifier := CreatePostgrestAuthModifier(c) modifier(req) // Verify the Authorization header was replaced @@ -388,7 +388,7 @@ func TestCreatePostgrestAuthModifier(t *testing.T) { req.Header.Set("Authorization", originalAuth) // Apply the modifier - modifier := createPostgrestAuthModifier(c) + modifier := CreatePostgrestAuthModifier(c) modifier(req) // Verify the Authorization header was not modified @@ -406,7 +406,7 @@ func TestCreatePostgrestAuthModifier(t *testing.T) { req.Header.Set("Authorization", originalAuth) // Apply the modifier - modifier := createPostgrestAuthModifier(c) + modifier := CreatePostgrestAuthModifier(c) modifier(req) // Empty postgrest_token means GetPostgrestToken returns false, diff --git a/internal/routes/proxies/resource_proxy.go b/internal/routes/proxies/resource_proxy.go index 49f35690..18edff37 100644 --- a/internal/routes/proxies/resource_proxy.go +++ b/internal/routes/proxies/resource_proxy.go @@ -223,7 +223,7 @@ func CreateStructProxyHandler[T any](deps *Dependencies, tableName string) gin.H return func(c *gin.Context) { // Create proxy handler - proxyHandler := CreateProxyHandler(deps.StorageAccessURL, tableName, createPostgrestAuthModifier(c)) + proxyHandler := CreateProxyHandler(deps.StorageAccessURL, tableName, CreatePostgrestAuthModifier(c)) // If no field filtering is needed, use proxy directly if len(excludeFields) == 0 { diff --git a/pkg/client/image_registry.go b/pkg/client/image_registry.go index d0b55d75..ba67d72e 100644 --- a/pkg/client/image_registry.go +++ b/pkg/client/image_registry.go @@ -27,12 +27,16 @@ func NewImageRegistriesService(client *Client) *ImageRegistriesService { type ImageRegistryListOptions struct { Workspace string Name string + WithCreds bool } // List lists all image registries in the specified workspace func (s *ImageRegistriesService) List(opts ImageRegistryListOptions) ([]v1.ImageRegistry, error) { // Build URL with query parameters baseURL := fmt.Sprintf("%s/api/v1/image_registries", s.client.baseURL) + if opts.WithCreds { + baseURL = fmt.Sprintf("%s/api/v1/credentials/image_registries", s.client.baseURL) + } params := url.Values{} if opts.Name != "" { diff --git a/pkg/engine/engine_version_test.go b/pkg/engine/engine_version_test.go deleted file mode 100644 index 84f3f8ff..00000000 --- a/pkg/engine/engine_version_test.go +++ /dev/null @@ -1,355 +0,0 @@ -package engine - -import ( - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - v1 "github.com/neutree-ai/neutree/api/v1" -) - -func TestExtractorTarGz(t *testing.T) { - // This is a unit test structure - actual implementation would need test fixtures - extractor := NewExtractor() - assert.NotNil(t, extractor) - - // Test would require creating a test tar.gz file - // For now, just verify the extractor can be instantiated -} - -func TestExtractorValidation(t *testing.T) { - extractor := NewExtractor() - - // Test invalid package format - err := extractor.Extract("invalid.xyz", "/tmp/test") - assert.Error(t, err) - assert.Contains(t, err.Error(), "unsupported package format") -} - -func TestExtractorSecurityCheck(t *testing.T) { - // Test that extractor prevents path traversal attacks - extractor := NewExtractor() - assert.NotNil(t, extractor) - - // Would need to create a malicious tar file for full test - // This validates the security check logic exists -} - -func TestParserValidateManifest(t *testing.T) { - parser := NewParser() - - tests := []struct { - name string - manifest *PackageManifest - expectError bool - errorMsg string - }{ - { - name: "valid manifest", - manifest: &PackageManifest{ - ManifestVersion: "1.0", - Package: &EngineVersionPackage{ - Metadata: &PackageMetadata{ - EngineName: "test-engine", - Version: "v1.0.0", - PackageVersion: "1.0", - }, - Images: []*ImageSpec{ - { - Accelerator: "nvidia-gpu", - ImageName: "test/image", - Tag: "v1.0.0", - ImageFile: "images/test.tar", - }, - }, - EngineVersion: &v1.EngineVersion{ - Version: "v1.0.0", - }, - }, - }, - expectError: false, - }, - { - name: "missing metadata", - manifest: &PackageManifest{ - ManifestVersion: "1.0", - Package: &EngineVersionPackage{ - Metadata: nil, - }, - }, - expectError: true, - errorMsg: "metadata is nil", - }, - { - name: "missing engine name", - manifest: &PackageManifest{ - ManifestVersion: "1.0", - Package: &EngineVersionPackage{ - Metadata: &PackageMetadata{ - Version: "v1.0.0", - PackageVersion: "1.0", - }, - }, - }, - expectError: true, - errorMsg: "engine name is empty", - }, - { - name: "no images", - manifest: &PackageManifest{ - ManifestVersion: "1.0", - Package: &EngineVersionPackage{ - Metadata: &PackageMetadata{ - EngineName: "test", - Version: "v1.0.0", - PackageVersion: "1.0", - }, - Images: []*ImageSpec{}, - EngineVersion: &v1.EngineVersion{}, - }, - }, - expectError: true, - errorMsg: "no images specified", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := parser.validateManifest(tt.manifest) - if tt.expectError { - assert.Error(t, err) - if tt.errorMsg != "" { - assert.Contains(t, err.Error(), tt.errorMsg) - } - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestParserGetImagePath(t *testing.T) { - parser := NewParser() - - extractedPath := "/tmp/extracted" - imageFile := "images/test.tar" - - result := parser.GetImagePath(extractedPath, imageFile) - expected := filepath.Join(extractedPath, imageFile) - - assert.Equal(t, expected, result) -} - -func TestImagePusherBuildTargetImage(t *testing.T) { - pusher, err := NewImagePusher(nil) // No API client needed for testing buildTargetImage - require.NoError(t, err, "Failed to create ImagePusher") - - tests := []struct { - name string - imagePrefix string - imgSpec *ImageSpec - expected string - }{ - { - name: "with prefix", - imagePrefix: "registry.example.com/neutree", - imgSpec: &ImageSpec{ - ImageName: "vllm-cuda", - Tag: "v0.5.0", - }, - expected: "registry.example.com/neutree/vllm-cuda:v0.5.0", - }, - { - name: "without prefix", - imagePrefix: "registry.example.com", - imgSpec: &ImageSpec{ - ImageName: "vllm-cuda", - Tag: "v0.5.0", - }, - expected: "registry.example.com/vllm-cuda:v0.5.0", - }, - { - name: "remove existing registry", - imagePrefix: "new-registry.com/neutree", - imgSpec: &ImageSpec{ - ImageName: "old-registry.com/vllm-cuda", - Tag: "v0.5.0", - }, - expected: "new-registry.com/neutree/vllm-cuda:v0.5.0", - }, - { - name: "remove existing registry with port", - imagePrefix: "new-registry.com/neutree", - imgSpec: &ImageSpec{ - ImageName: "old-registry.com:5000/vllm-cuda", - Tag: "v0.5.0", - }, - expected: "new-registry.com/neutree/vllm-cuda:v0.5.0", - }, - { - name: "keep organization name without dots", - imagePrefix: "registry.example.com/neutree", - imgSpec: &ImageSpec{ - ImageName: "myorg/vllm-cuda", - Tag: "v0.5.0", - }, - expected: "registry.example.com/neutree/myorg/vllm-cuda:v0.5.0", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := pusher.buildTargetImage(tt.imagePrefix, tt.imgSpec) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestImportOptionsValidation(t *testing.T) { - importer, err := NewImporter(nil) - require.NoError(t, err, "Failed to create Importer") - - tests := []struct { - name string - opts *ImportOptions - setupFunc func() string // Returns temp file path - cleanupFunc func(string) - expectError bool - errorMsg string - }{ - { - name: "valid options with skip image push", - setupFunc: func() string { - tmpFile, _ := os.CreateTemp("", "test-*.tar.gz") - tmpFile.Close() - return tmpFile.Name() - }, - cleanupFunc: func(path string) { - os.Remove(path) - }, - opts: &ImportOptions{ - PackagePath: "", // Will be set by setupFunc - SkipImagePush: true, - }, - expectError: false, - }, - { - name: "missing package path", - opts: &ImportOptions{ - PackagePath: "", - }, - expectError: true, - errorMsg: "package path is required", - }, - { - name: "package file not found", - opts: &ImportOptions{ - PackagePath: "/nonexistent/package.tar.gz", - }, - expectError: true, - errorMsg: "package file not found", - }, - { - name: "missing registry when not skipping push", - setupFunc: func() string { - tmpFile, _ := os.CreateTemp("", "test-*.tar.gz") - tmpFile.Close() - return tmpFile.Name() - }, - cleanupFunc: func(path string) { - os.Remove(path) - }, - opts: &ImportOptions{ - PackagePath: "", // Will be set by setupFunc - SkipImagePush: false, - ImageRegistry: "", - }, - expectError: true, - errorMsg: "image registry is required", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.setupFunc != nil { - tmpPath := tt.setupFunc() - tt.opts.PackagePath = tmpPath - if tt.cleanupFunc != nil { - defer tt.cleanupFunc(tmpPath) - } - } - - err := importer.validateOptions(tt.opts) - if tt.expectError { - require.Error(t, err) - if tt.errorMsg != "" { - assert.Contains(t, err.Error(), tt.errorMsg) - } - } else { - assert.NoError(t, err) - } - }) - } -} - -func TestImagePusherExtractImageNameWithoutRegistry(t *testing.T) { - pusher, err := NewImagePusher(nil) // No API client needed for testing - require.NoError(t, err, "Failed to create ImagePusher") - - tests := []struct { - name string - imageName string - expected string - }{ - { - name: "simple image name", - imageName: "vllm-cuda", - expected: "vllm-cuda", - }, - { - name: "image with organization", - imageName: "myorg/vllm-cuda", - expected: "myorg/vllm-cuda", - }, - { - name: "image with registry domain", - imageName: "registry.example.com/vllm-cuda", - expected: "vllm-cuda", - }, - { - name: "image with registry and org", - imageName: "registry.example.com/myorg/vllm-cuda", - expected: "myorg/vllm-cuda", - }, - { - name: "image with registry port", - imageName: "registry.example.com:5000/vllm-cuda", - expected: "vllm-cuda", - }, - { - name: "image with registry port and org", - imageName: "registry.example.com:5000/myorg/vllm-cuda", - expected: "myorg/vllm-cuda", - }, - { - name: "dockerhub official image", - imageName: "nginx", - expected: "nginx", - }, - { - name: "dockerhub user image", - imageName: "username/image", - expected: "username/image", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := pusher.extractImageNameWithoutRegistry(tt.imageName) - assert.Equal(t, tt.expected, result) - }) - } -} diff --git a/pkg/engine/importer.go b/pkg/engine/importer.go deleted file mode 100644 index 322b1d09..00000000 --- a/pkg/engine/importer.go +++ /dev/null @@ -1,259 +0,0 @@ -package engine - -import ( - "context" - "os" - "path/filepath" - - "github.com/pkg/errors" - "k8s.io/klog/v2" - - v1 "github.com/neutree-ai/neutree/api/v1" - internalutil "github.com/neutree-ai/neutree/internal/util" - "github.com/neutree-ai/neutree/pkg/client" -) - -// Importer handles importing engine version packages -type Importer struct { - apiClient *client.Client - extractor *Extractor - parser *Parser - imagePusher *ImagePusher -} - -// NewImporter creates a new Importer -func NewImporter(apiClient *client.Client) (*Importer, error) { - imagePusher, err := NewImagePusher(apiClient) - if err != nil { - return nil, errors.Wrap(err, "failed to create image pusher") - } - - return &Importer{ - apiClient: apiClient, - extractor: NewExtractor(), - parser: NewParser(), - imagePusher: imagePusher, - }, nil -} - -// Import imports an engine version package -func (i *Importer) Import(ctx context.Context, opts *ImportOptions) (*ImportResult, error) { - result := &ImportResult{ - ImagesImported: []string{}, - Errors: []error{}, - } - - // Validate options - if err := i.validateOptions(opts); err != nil { - return nil, errors.Wrap(err, "invalid import options") - } - - // Create temporary directory if not specified - if opts.ExtractPath == "" { - tempDir, err := os.MkdirTemp("", "engine-version-*") - if err != nil { - return nil, errors.Wrap(err, "failed to create temporary directory") - } - - opts.ExtractPath = tempDir - - defer os.RemoveAll(tempDir) - } - - klog.Infof("Extracting package to %s", opts.ExtractPath) - - // Extract the package - if err := i.extractor.Extract(opts.PackagePath, opts.ExtractPath); err != nil { - return nil, errors.Wrap(err, "failed to extract package") - } - - // Parse the manifest - klog.Info("Parsing manifest") - - manifest, err := i.parser.ParseManifest(opts.ExtractPath) - if err != nil { - return nil, errors.Wrap(err, "failed to parse manifest") - } - - result.EngineName = manifest.Package.Metadata.EngineName - result.Version = manifest.Package.Metadata.Version - - // Check if engine exists - engineList, err := i.apiClient.Engines.List(client.ListOptions{ - Workspace: opts.Workspace, - Name: manifest.Package.Metadata.EngineName, - }) - if err != nil { - return nil, errors.Wrap(err, "failed to check if engine exists") - } - - var engine *v1.Engine - if len(engineList) > 0 { - engine = &engineList[0] - klog.Infof("Found existing engine: %s", engine.Metadata.Name) - - // Check if version already exists - if !opts.Force { - for _, ver := range engine.Spec.Versions { - if ver.Version == manifest.Package.Metadata.Version { - return nil, errors.Errorf("engine version %s already exists for engine %s (use --force to overwrite)", - manifest.Package.Metadata.Version, manifest.Package.Metadata.EngineName) - } - } - } - } - - // Push images to registry if not skipped - if !opts.SkipImagePush { - klog.Info("Loading and pushing images to registry") - - pushedImages, err := i.imagePusher.LoadAndPushImages( - ctx, - opts.Workspace, - opts.ImageRegistry, - manifest, - opts.ExtractPath, - ) - if err != nil { - result.Errors = append(result.Errors, err) - return result, errors.Wrap(err, "failed to push images") - } - - result.ImagesImported = pushedImages - } - - // Update or create engine - klog.Info("Updating engine definition") - - if err := i.updateEngine(ctx, engine, manifest, opts); err != nil { - result.Errors = append(result.Errors, err) - return result, errors.Wrap(err, "failed to update engine") - } - - result.EngineUpdated = true - klog.Infof("Successfully imported engine version %s:%s", result.EngineName, result.Version) - - return result, nil -} - -// validateOptions validates the import options -func (i *Importer) validateOptions(opts *ImportOptions) error { - if opts.PackagePath == "" { - return errors.New("package path is required") - } - - if _, err := os.Stat(opts.PackagePath); os.IsNotExist(err) { - return errors.Errorf("package file not found: %s", opts.PackagePath) - } - - if !opts.SkipImagePush { - if opts.ImageRegistry == "" { - return errors.New("image registry is required when not skipping image push") - } - } - - return nil -} - -// updateEngine updates the engine with the new version -func (i *Importer) updateEngine(_ context.Context, engine *v1.Engine, manifest *PackageManifest, opts *ImportOptions) error { - newVersion := manifest.Package.EngineVersion - - // Ensure the version field matches the metadata - newVersion.Version = manifest.Package.Metadata.Version - - if engine == nil { - // Create new engine - engine = &v1.Engine{ - APIVersion: "v1", - Kind: "Engine", - Metadata: &v1.Metadata{ - Name: manifest.Package.Metadata.EngineName, - Workspace: opts.Workspace, - }, - Spec: &v1.EngineSpec{ - Versions: []*v1.EngineVersion{newVersion}, - SupportedTasks: []string{}, // Will be populated from manifest if available - }, - } - - return i.apiClient.Engines.Create(opts.Workspace, engine) - } - - // Update existing engine - // Check if version already exists and remove it if force is enabled - - var oldVersion *v1.EngineVersion - - for idx, ver := range engine.Spec.Versions { - if ver.Version == manifest.Package.Metadata.Version { - // Remove the old version - engine.Spec.Versions = append(engine.Spec.Versions[:idx], engine.Spec.Versions[idx+1:]...) - oldVersion = ver - - break - } - } - - if oldVersion == nil || opts.Force { - engine.Spec.Versions = append(engine.Spec.Versions, newVersion) - } else { - // merge oldVersion with newVersion - engine.Spec.Versions = append(engine.Spec.Versions, internalutil.MergeEngineVersion(oldVersion, newVersion)) - } - - return i.apiClient.Engines.Update(opts.Workspace, engine.GetID(), engine) -} - -// Validator handles validation of engine version packages -type Validator struct { - extractor *Extractor - parser *Parser -} - -// NewValidator creates a new Validator -func NewValidator() *Validator { - return &Validator{ - extractor: NewExtractor(), - parser: NewParser(), - } -} - -// ValidatePackage validates an engine version package without importing it -func (v *Validator) ValidatePackage(packagePath string) error { - // Create temporary directory - tempDir, err := os.MkdirTemp("", "engine-version-validate-*") - if err != nil { - return errors.Wrap(err, "failed to create temporary directory") - } - defer os.RemoveAll(tempDir) - - // Extract the package - if err := v.extractor.Extract(packagePath, tempDir); err != nil { - return errors.Wrap(err, "failed to extract package") - } - - // Parse the manifest - manifest, err := v.parser.ParseManifest(tempDir) - if err != nil { - return errors.Wrap(err, "failed to parse manifest") - } - - // Validate that all image files exist - for _, imgSpec := range manifest.Package.Images { - imagePath := filepath.Join(tempDir, imgSpec.ImageFile) - if _, err := os.Stat(imagePath); os.IsNotExist(err) { - return errors.Errorf("image file not found: %s", imgSpec.ImageFile) - } - } - - klog.Info("Package validation successful") - - return nil -} - -// ValidatePackage is a convenience function that creates a validator and validates a package -func ValidatePackage(packagePath string) error { - v := NewValidator() - return v.ValidatePackage(packagePath) -} diff --git a/pkg/engine/parser.go b/pkg/engine/parser.go deleted file mode 100644 index d0d3de41..00000000 --- a/pkg/engine/parser.go +++ /dev/null @@ -1,127 +0,0 @@ -package engine - -import ( - "encoding/base64" - "encoding/json" - "os" - "path/filepath" - - "github.com/pkg/errors" - "gopkg.in/yaml.v3" -) - -const ( - // ManifestFileName is the name of the manifest file in the package - ManifestFileName = "manifest.yaml" -) - -// Parser handles parsing of engine version package manifests -type Parser struct{} - -// NewParser creates a new Parser -func NewParser() *Parser { - return &Parser{} -} - -// ParseManifest parses the manifest file from the extracted package directory -func (p *Parser) ParseManifest(extractedPath string) (*PackageManifest, error) { - manifestPath := filepath.Join(extractedPath, ManifestFileName) - if _, err := os.Stat(manifestPath); err == nil { - return p.parseYAMLManifest(manifestPath) - } - - return nil, errors.New("manifest file not found") -} - -// parseYAMLManifest parses a YAML manifest file -func (p *Parser) parseYAMLManifest(path string) (*PackageManifest, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, errors.Wrap(err, "failed to read manifest file") - } - - var manifest PackageManifest - if err := yaml.Unmarshal(data, &manifest); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal YAML manifest") - } - - // Decode base64-encoded values schema if present - valueSchemaStr, ok := manifest.Package.EngineVersion.ValuesSchema["values_schema_base64"] - if ok { - valueSchemaBase64Str, ok := valueSchemaStr.(string) - if !ok { - return nil, errors.New("invalid values_schema_base64 format in manifest") - } - - valueSchemaJson, err := base64.StdEncoding.DecodeString(valueSchemaBase64Str) - if err != nil { - return nil, errors.Wrap(err, "failed to decode values schema from base64") - } - - var decodedSchema map[string]interface{} - if err := json.Unmarshal(valueSchemaJson, &decodedSchema); err != nil { - return nil, errors.Wrap(err, "failed to unmarshal values schema JSON") - } - - manifest.Package.EngineVersion.ValuesSchema = decodedSchema - } - - if err := p.validateManifest(&manifest); err != nil { - return nil, errors.Wrap(err, "manifest validation failed") - } - - return &manifest, nil -} - -// validateManifest validates the parsed manifest -func (p *Parser) validateManifest(manifest *PackageManifest) error { - if manifest.Package == nil { - return errors.New("package is nil") - } - - if manifest.Package.Metadata == nil { - return errors.New("package metadata is nil") - } - - if manifest.Package.Metadata.EngineName == "" { - return errors.New("engine name is empty") - } - - if manifest.Package.Metadata.Version == "" { - return errors.New("version is empty") - } - - if manifest.Package.EngineVersion == nil { - return errors.New("engine version is nil") - } - - if len(manifest.Package.Images) == 0 { - return errors.New("no images specified") - } - - // Validate each image spec - for i, img := range manifest.Package.Images { - if img.Accelerator == "" { - return errors.Errorf("image %d: accelerator is empty", i) - } - - if img.ImageName == "" { - return errors.Errorf("image %d: image name is empty", i) - } - - if img.Tag == "" { - return errors.Errorf("image %d: tag is empty", i) - } - - if img.ImageFile == "" { - return errors.Errorf("image %d: image file is empty", i) - } - } - - return nil -} - -// GetImagePath returns the full path to an image file in the extracted package -func (p *Parser) GetImagePath(extractedPath string, imageFile string) string { - return filepath.Join(extractedPath, imageFile) -} diff --git a/scripts/builder/build-engine-package.sh b/scripts/builder/build-engine-package.sh index 00e9b3e7..7ae99aa6 100755 --- a/scripts/builder/build-engine-package.sh +++ b/scripts/builder/build-engine-package.sh @@ -5,6 +5,9 @@ set -e +VERSION=$(git describe --tags --always --dirty) +OUTPUT_DIR="./dist" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -464,22 +467,22 @@ ${DEPLOY_TEMPLATE_CONTENT}" cat > "$PACKAGE_DIR/manifest.yaml" << EOF manifest_version: "1.0" -package: - metadata: - engine_name: "$ENGINE_NAME" - version: "$ENGINE_VERSION" - description: "${DESCRIPTION:-Engine version $ENGINE_VERSION}" - author: "$(whoami)" - created_at: "$CREATED_AT" - package_version: "1.0" - tags: - - "llm" - - "inference" - - images:$IMAGE_ENTRIES - - engine_version: - version: "$ENGINE_VERSION" +metadata: + description: "${DESCRIPTION:-Engine version $ENGINE_VERSION}" + author: "Neutree Team" + created_at: "$CREATED_AT" + version: $VERSION + tags: + - "engine" + - "$ENGINE_NAME" + - "$ENGINE_VERSION" + +images:$IMAGE_ENTRIES + +engines: +- name: $ENGINE_NAME + engine_versions: + - version: "$ENGINE_VERSION" ${VALUES_SCHEMA_SECTION} $DEPLOY_TEMPLATE_SECTION @@ -496,15 +499,17 @@ tar -I "pigz -p 16" -cf "$OUTPUT_FILE" * cd - > /dev/null # Move to final location -if [ -f "$OUTPUT_FILE" ]; then - : # File is already in the right place +mv -f "$PACKAGE_DIR/$OUTPUT_FILE" "$OUTPUT_DIR/$OUTPUT_FILE" +# Calculate checksum +log_info "Calculating checksum..." +if command -v md5sum &> /dev/null; then + md5sum "$OUTPUT_DIR/$OUTPUT_FILE" > "${OUTPUT_DIR}/${OUTPUT_FILE}.md5" else - mv "$PACKAGE_DIR/$OUTPUT_FILE" "./$OUTPUT_FILE" + md5 "$OUTPUT_DIR/$OUTPUT_FILE" | awk '{print $4}' > "${OUTPUT_DIR}/${OUTPUT_FILE}.md5" fi # Get package size -PACKAGE_SIZE=$(stat -f%z "$OUTPUT_FILE" 2>/dev/null || stat -c%s "$OUTPUT_FILE" 2>/dev/null) - +PACKAGE_SIZE=$(stat -f%z "$OUTPUT_DIR/$OUTPUT_FILE" 2>/dev/null || stat -c%s "$OUTPUT_DIR/$OUTPUT_FILE" 2>/dev/null) print_info "Package created successfully!" echo "" echo "================================================" @@ -517,7 +522,7 @@ echo "Size: $(numfmt --to=iec-i --suffix=B $PACKAGE_SIZE 2>/dev/null || e echo "================================================" echo "" print_info "You can now validate the package with:" -echo " neutree-cli engine validate --package $OUTPUT_FILE" +echo " neutree-cli import validate --package $OUTPUT_FILE" echo "" print_info "Or import it with:" -echo " neutree-cli engine import --package $OUTPUT_FILE --registry --workspace " +echo " neutree-cli import engine --package $OUTPUT_FILE --registry --workspace " diff --git a/scripts/builder/build-package.sh b/scripts/builder/build-package.sh new file mode 100755 index 00000000..a50e93ee --- /dev/null +++ b/scripts/builder/build-package.sh @@ -0,0 +1,370 @@ +#!/bin/bash + +set -e + +VERSION="${VERSION:-latest}" +PACKAGE_TYPE="" +CLUSTER_TYPE="" +ACCELERATOR="" +ARCH="${ARCH:-amd64}" +OUTPUT_DIR="./dist" +MIRROR_REGISTRY="" +TEMP_DIR=$(mktemp -d) + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Options: + --type Package type: controlplane, cluster + --version Version tag (default: latest) + --arch Architecture: amd64, arm64 (default: amd64) + --cluster-type Cluster type: k8s or ssh (required if type=cluster) + --accelerator Accelerator type: nvidia_gpu, amd_gpu (for ssh cluster) + --mirror-registry Mirror registry URL to pull images from (e.g., registry.example.com) + --output-dir Output directory (default: ./dist) + -h, --help Show this help message + +Examples: + # Build control plane package for amd64 + $0 --type controlplane --version v1.0.0 --arch amd64 + + # Build K8s cluster package for arm64 + $0 --type cluster --cluster-type k8s --version v1.0.0 --arch arm64 + + # Build SSH cluster package with NVIDIA for amd64 + $0 --type cluster --cluster-type ssh --accelerator nvidia_gpu --version v1.0.0 --arch amd64 + + # Build with mirror registry + $0 --type controlplane --version v1.0.0 --arch amd64 --mirror-registry registry.example.com +EOF + exit 1 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --type) + PACKAGE_TYPE="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --arch) + ARCH="$2" + shift 2 + ;; + --cluster-type) + CLUSTER_TYPE="$2" + shift 2 + ;; + --accelerator) + ACCELERATOR="$2" + shift 2 + ;; + --mirror-registry) + MIRROR_REGISTRY="$2" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + -h|--help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac +done + +# Validation +if [[ -z "$PACKAGE_TYPE" ]]; then + log_error "Package type is required" + usage +fi + +# Validate architecture +case "$ARCH" in + amd64|arm64) + ;; + *) + log_error "Unsupported architecture: $ARCH. Supported: amd64, arm64" + exit 1 + ;; +esac + +# Ensure image list files based on package type +IMAGE_LIST_FILES=() +PACKAGE_NAME="" + +case "$PACKAGE_TYPE" in + controlplane) + IMAGE_LIST_FILES+=("image-lists/controlplane/images.txt") + PACKAGE_NAME="neutree-control-plane-${VERSION}-${ARCH}" + ;; + cluster) + if [[ -z "$CLUSTER_TYPE" ]]; then + log_error "Cluster type is required for cluster package" + usage + fi + + case "$CLUSTER_TYPE" in + k8s) + IMAGE_LIST_FILES+=("image-lists/cluster/kubernetes/images.txt") + PACKAGE_NAME="neutree-cluster-k8s-${VERSION}-${ARCH}" + ;; + ssh) + PACKAGE_NAME="neutree-cluster-ssh" + + if [[ -n "$ACCELERATOR" ]]; then + IMAGE_LIST_FILES+=("image-lists/cluster/ssh/${ACCELERATOR}-images.txt") + PACKAGE_NAME="${PACKAGE_NAME}-${ACCELERATOR}" + fi + PACKAGE_NAME="${PACKAGE_NAME}-${VERSION}-${ARCH}" + ;; + *) + log_error "Unknown cluster type: $CLUSTER_TYPE" + usage + ;; + esac + ;; + engine) + log_error "Engine packages should use build-engine-package.sh" + exit 1 + ;; + *) + log_error "Unknown package type: $PACKAGE_TYPE" + usage + ;; +esac + +log_info "Building package: $PACKAGE_NAME" +log_info "Version: $VERSION" +log_info "Architecture: $ARCH" +log_info "Image list files: ${IMAGE_LIST_FILES[*]}" +if [[ -n "$MIRROR_REGISTRY" ]]; then + log_info "Mirror registry: $MIRROR_REGISTRY" +fi + +# Create package directory +PACKAGE_DIR="${TEMP_DIR}" +mkdir -p "${PACKAGE_DIR}/images" + +# Merge image lists +MERGED_IMAGE_LIST="${TEMP_DIR}/images.txt" +> "$MERGED_IMAGE_LIST" + +for list_file in "${IMAGE_LIST_FILES[@]}"; do + if [[ ! -f "$list_file" ]]; then + log_error "Image list file not found: $list_file" + exit 1 + fi + + log_info "Processing: $list_file" + # Process image list + while IFS= read -r line; do + # Skip comments and empty lines + [[ "$line" =~ ^#.*$ ]] && continue + [[ -z "$line" ]] && continue + + # If the image contains neutree + if [[ "$line" =~ neutree ]]; then + # Extract image name and tag + if [[ "$line" =~ ^([^:]+):(.+)$ ]]; then + image_name="${BASH_REMATCH[1]}" + image_tag="${BASH_REMATCH[2]}" + + # Replace "latest" in tag with version + new_tag="${image_tag//latest/${VERSION}}" + echo "${image_name}:${new_tag}" >> "$MERGED_IMAGE_LIST" + elif [[ "$line" =~ ^[^:]+$ ]]; then + # If no tag, default to version + echo "${line}:${VERSION}" >> "$MERGED_IMAGE_LIST" + else + echo "$line" >> "$MERGED_IMAGE_LIST" + fi + else + # Non-neutree images remain unchanged + echo "$line" >> "$MERGED_IMAGE_LIST" + fi + done < "$list_file" +done + + +# Deduplicate +sort -u "$MERGED_IMAGE_LIST" -o "$MERGED_IMAGE_LIST" + +log_info "Total images to package: $(wc -l < "$MERGED_IMAGE_LIST")" + +# Pull and save images +IMAGES_TO_PULL=() +while IFS= read -r image; do + [[ -z "$image" ]] && continue + + # Determine the actual image address to pull + pull_image="$image" + if [[ -n "$MIRROR_REGISTRY" ]]; then + # Pull from mirror registry + # Remove original registry (if any) + if [[ "$image" =~ ^([^/]*[.:][^/]*)/(.+)$ ]]; then + image_without_registry="${BASH_REMATCH[2]}" + else + image_without_registry="$image" + fi + pull_image="${MIRROR_REGISTRY}/${image_without_registry}" + log_info "Pulling image from mirror: $pull_image (original: $image)" + else + log_info "Pulling image: $image" + fi + + # Pull image with specified platform + if ! docker pull --platform "linux/${ARCH}" "$pull_image"; then + log_error "Failed to pull image: $pull_image for platform linux/${ARCH}" + exit 1 + fi + + # If using mirror registry, retag to original image name + if [[ -n "$MIRROR_REGISTRY" && "$pull_image" != "$image" ]]; then + log_info "Retagging to original image name: $image" + if ! docker tag "$pull_image" "$image"; then + log_error "Failed to tag image: $pull_image -> $image" + exit 1 + fi + fi + + IMAGES_TO_PULL+=("$image") +done < "$MERGED_IMAGE_LIST" + +log_info "Saving all images to single archive..." +ALL_IMAGES_FILE="${PACKAGE_DIR}/images/all-images.tar" + +if ! docker save -o "$ALL_IMAGES_FILE" "${IMAGES_TO_PULL[@]}"; then + log_error "Failed to save images" + exit 1 +fi + +log_info "All images saved successfully" + +# Generate manifest.yaml +log_info "Generating manifest..." +cat > "${PACKAGE_DIR}/manifest.yaml" << EOF +manifest_version: "1.0" +metadata: + version: "${VERSION}" + author: "Neutree Team" + created_at: "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + tags: + - "${PACKAGE_TYPE}" + - "${ARCH}" +$(if [[ -n "$CLUSTER_TYPE" ]]; then echo " - \"${CLUSTER_TYPE}\""; fi) +$(if [[ -n "$ACCELERATOR" ]]; then echo " - \"${ACCELERATOR}\""; fi) + +images: +EOF + +# Add image information to manifest +for image in "${IMAGES_TO_PULL[@]}"; do + IFS=':' read -r image_name image_tag <<< "$image" + + # Get image information + digest=$(docker inspect --format='{{.Id}}' "$image" 2>/dev/null || echo "") + size=$(docker inspect --format='{{.Size}}' "$image" 2>/dev/null || echo "0") + platform="linux/${ARCH}" + + cat >> "${PACKAGE_DIR}/manifest.yaml" << EOF + - image_name: "${image_name}" + tag: "${image_tag}" + image_file: "images/all-images.tar" + platform: "${platform}" + size: ${size} + digest: "${digest}" +EOF +done + +log_info "Manifest generated successfully" + +# Create README +cat > "${PACKAGE_DIR}/README.md" << EOF +# Neutree ${PACKAGE_TYPE^} Package + +Version: ${VERSION} +Architecture: ${ARCH} +Created: $(date -u +"%Y-%m-%d %H:%M:%S UTC") + +## Package Contents + +- Total Images: $(wc -l < "$MERGED_IMAGE_LIST") +- Package Type: ${PACKAGE_TYPE} +- Architecture: ${ARCH} +$(if [[ -n "$CLUSTER_TYPE" ]]; then echo "- Cluster Type: ${CLUSTER_TYPE}"; fi) +$(if [[ -n "$ACCELERATOR" ]]; then echo "- Accelerator: ${ACCELERATOR}"; fi) + +## Import Instructions + +\`\`\`bash +neutree package import \\ + --package ${PACKAGE_NAME}.tar.gz \\ + --registry your-registry.com +\`\`\` + +## Image List + +\`\`\` +$(cat "$MERGED_IMAGE_LIST") +\`\`\` +EOF + +# Package +mkdir -p "$OUTPUT_DIR" +PACKAGE_FILE="${OUTPUT_DIR}/${PACKAGE_NAME}.tar.gz" + +log_info "Creating package: $PACKAGE_FILE" + +CURRENT_DIR=$(pwd) + +cd "$TEMP_DIR" || exit 1 + +if command -v pigz &> /dev/null; then + tar -I "pigz -p 16" -cf "${CURRENT_DIR}/${PACKAGE_FILE}" * +else + tar -czf "${CURRENT_DIR}/${PACKAGE_FILE}" * +fi + +cd "$CURRENT_DIR" || exit 1 + +# Calculate checksum +log_info "Calculating checksum..." +if command -v md5sum &> /dev/null; then + md5sum "$PACKAGE_FILE" > "${PACKAGE_FILE}.md5" +else + md5 "$PACKAGE_FILE" | awk '{print $4}' > "${PACKAGE_FILE}.md5" +fi + +# Clean up +rm -rf "$TEMP_DIR" + +log_info "Package created successfully: $PACKAGE_FILE" +log_info "Package size: $(du -h "$PACKAGE_FILE" | cut -f1)" +log_info "Checksum: $(cat "${PACKAGE_FILE}.md5")" \ No newline at end of file diff --git a/scripts/builder/image-lists/cluster/kubernetes/images.txt b/scripts/builder/image-lists/cluster/kubernetes/images.txt new file mode 100644 index 00000000..ea45120d --- /dev/null +++ b/scripts/builder/image-lists/cluster/kubernetes/images.txt @@ -0,0 +1,2 @@ +neutree/router:latest +neutree/vmagent:v1.115.0 diff --git a/scripts/builder/image-lists/cluster/ssh/amd_gpu-images.txt b/scripts/builder/image-lists/cluster/ssh/amd_gpu-images.txt new file mode 100644 index 00000000..4bf387c6 --- /dev/null +++ b/scripts/builder/image-lists/cluster/ssh/amd_gpu-images.txt @@ -0,0 +1 @@ +neutree/neutree-serve:latest-rocm \ No newline at end of file diff --git a/scripts/builder/image-lists/cluster/ssh/nvidia_gpu-images.txt b/scripts/builder/image-lists/cluster/ssh/nvidia_gpu-images.txt new file mode 100644 index 00000000..4067aa98 --- /dev/null +++ b/scripts/builder/image-lists/cluster/ssh/nvidia_gpu-images.txt @@ -0,0 +1 @@ +neutree/neutree-serve:latest \ No newline at end of file diff --git a/scripts/builder/image-lists/controlplane/images.txt b/scripts/builder/image-lists/controlplane/images.txt new file mode 100644 index 00000000..276e536a --- /dev/null +++ b/scripts/builder/image-lists/controlplane/images.txt @@ -0,0 +1,17 @@ +bitnami/jwt-cli:6.2.0 +docker.io/grafana/grafana:11.5.3 +ghcr.io/groundnuty/k8s-wait-for:v2.0 +kong/kong:3.9 +migrate/migrate:v4.18.3 +neutree/neutree-api:latest +neutree/neutree-core:latest +neutree/neutree-db-scripts:latest +postgres:13 +postgrest/postgrest:v14.0 +supabase/gotrue:v2.170.0 +supabase/postgres-meta:v0.86.0 +timberio/vector:0.47.0-debian +victoriametrics/vmagent:v1.115.0 +victoriametrics/vminsert:v1.115.0-cluster +victoriametrics/vmselect:v1.115.0-cluster +victoriametrics/vmstorage:v1.115.0-cluster