Skip to content

Commit 27631d8

Browse files
EyalDelareaEyalDelarea
committed
Merge remote-tracking branch 'upstream/dev'
2 parents b382374 + 5c18251 commit 27631d8

File tree

12 files changed

+259
-37
lines changed

12 files changed

+259
-37
lines changed

‎artifactory/commands/utils/testdata/transferconfig/virtual_repo_a.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@
3131
"priorityResolution": false,
3232
"projectKey": "default",
3333
"environments": [],
34-
"repositories": ["b-virtual"],
34+
"repositories": ["b-virtual","a-local"],
35+
"defaultDeploymentRepo": "a-local",
3536
"hideUnauthorizedResources": false,
3637
"artifactoryRequestsCanRetrieveRemoteArtifacts": false,
3738
"resolveDockerTagsByTimestamp": false,

‎artifactory/commands/utils/testdata/transferconfig/virtual_repo_b.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
"priorityResolution": false,
3232
"projectKey": "default",
3333
"environments": [],
34+
"repositories": ["b-local"],
35+
"defaultDeploymentRepo": "b-local",
3436
"hideUnauthorizedResources": false,
3537
"artifactoryRequestsCanRetrieveRemoteArtifacts": false,
3638
"resolveDockerTagsByTimestamp": false,

‎artifactory/commands/utils/transferconfigbase.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -224,17 +224,22 @@ func (tcb *TransferConfigBase) transferVirtualRepositoriesToTarget(reposToTransf
224224
return
225225
}
226226

227-
// Create virtual repository without included repositories
227+
// Create virtual repository without included repositories and default deployment repo
228228
repositories := singleRepoParamsMap["repositories"]
229229
delete(singleRepoParamsMap, "repositories")
230+
defaultDeploymentRepo := singleRepoParamsMap["defaultDeploymentRepo"]
231+
delete(singleRepoParamsMap, "defaultDeploymentRepo")
230232
if err = tcb.createRepositoryAndAssignToProject(singleRepoParamsMap, repoToTransfer); err != nil {
231233
return
232234
}
233235

234-
// Restore included repositories to set them later on
236+
// Restore included repositories and default deployment repo to set them later on
235237
if repositories != nil {
236238
singleRepoParamsMap["repositories"] = repositories
237239
}
240+
if defaultDeploymentRepo != nil {
241+
singleRepoParamsMap["defaultDeploymentRepo"] = defaultDeploymentRepo
242+
}
238243
}
239244

240245
// Step 2 - Update all virtual repositories with the included repositories

‎artifactory/commands/utils/transferconfigbase_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,8 @@ func TestTransferVirtualRepositoriesToTarget(t *testing.T) {
223223
if r.Method == http.MethodPut {
224224
delete(expectedVirtualRepoAParamsMap, "repositories")
225225
delete(expectedVirtualRepoBParamsMap, "repositories")
226+
delete(expectedVirtualRepoAParamsMap, "defaultDeploymentRepo")
227+
delete(expectedVirtualRepoBParamsMap, "defaultDeploymentRepo")
226228
}
227229

228230
switch r.RequestURI {

‎artifactory/utils/container/buildinfo.go‎

Lines changed: 159 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ package container
22

33
import (
44
"encoding/json"
5-
ioutils "github.com/jfrog/gofrog/io"
5+
"fmt"
6+
"net/http"
67
"os"
78
"path"
89
"strings"
910

11+
ioutils "github.com/jfrog/gofrog/io"
12+
1013
buildinfo "github.com/jfrog/build-info-go/entities"
1114

1215
artutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
@@ -52,15 +55,28 @@ type buildInfoBuilder struct {
5255
imageLayers []utils.ResultItem
5356
}
5457

58+
type RepositoryDetails struct {
59+
key string
60+
isRemote bool
61+
repoType string
62+
}
63+
5564
// Create instance of docker build info builder.
5665
func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, project string, serviceManager artifactory.ArtifactoryServicesManager) (*buildInfoBuilder, error) {
5766
var err error
5867
builder := &buildInfoBuilder{}
5968
builder.repositoryDetails.key = repository
60-
builder.repositoryDetails.isRemote, err = artutils.IsRemoteRepo(repository, serviceManager)
69+
70+
// Get repository details in one API call to determine both isRemote and repoType
71+
repoDetails := &services.RepositoryDetails{}
72+
err = serviceManager.GetRepository(repository, &repoDetails)
6173
if err != nil {
62-
return nil, err
74+
return nil, errorutils.CheckErrorf("failed to get details for repository '" + repository + "'. Error:\n" + err.Error())
6375
}
76+
77+
builder.repositoryDetails.isRemote = repoDetails.GetRepoType() == "remote"
78+
builder.repositoryDetails.repoType = repoDetails.GetRepoType()
79+
6480
builder.image = image
6581
builder.buildName = buildName
6682
builder.buildNumber = buildNumber
@@ -69,11 +85,6 @@ func newBuildInfoBuilder(image *Image, repository, buildName, buildNumber, proje
6985
return builder, nil
7086
}
7187

72-
type RepositoryDetails struct {
73-
key string
74-
isRemote bool
75-
}
76-
7788
func (builder *buildInfoBuilder) setImageSha2(imageSha2 string) {
7889
builder.imageSha2 = imageSha2
7990
}
@@ -90,12 +101,34 @@ func (builder *buildInfoBuilder) getSearchableRepo() string {
90101
}
91102

92103
// Set build properties on image layers in Artifactory.
93-
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager) (err error) {
104+
func setBuildProperties(buildName, buildNumber, project string, imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) (err error) {
105+
if buildName == "" || buildNumber == "" {
106+
log.Debug("Skipping setting properties - build name and build number are required")
107+
return nil
108+
}
109+
94110
props, err := build.CreateBuildProperties(buildName, buildNumber, project)
95111
if err != nil {
96112
return
97113
}
98-
pathToFile, err := writeLayersToFile(imageLayers)
114+
115+
if len(props) == 0 {
116+
log.Debug("Skipping setting properties - no properties created")
117+
return nil
118+
}
119+
120+
filteredLayers, err := filterLayersForVirtualRepository(imageLayers, serviceManager, originalRepo, repoDetails)
121+
if err != nil {
122+
log.Debug("Failed to filter layers for virtual repository, proceeding with all layers:", err.Error())
123+
filteredLayers = imageLayers
124+
}
125+
126+
if len(filteredLayers) == 0 {
127+
log.Debug("No layers to set properties on, skipping property setting")
128+
return nil
129+
}
130+
131+
pathToFile, err := writeLayersToFile(filteredLayers)
99132
if err != nil {
100133
return
101134
}
@@ -105,6 +138,120 @@ func setBuildProperties(buildName, buildNumber, project string, imageLayers []ut
105138
return
106139
}
107140

141+
// filterLayersForVirtualRepository filters image layers to only include those from the default deployment repository
142+
// when dealing with virtual repositories. For non-virtual repositories, it returns all layers unchanged.
143+
func filterLayersForVirtualRepository(imageLayers []utils.ResultItem, serviceManager artifactory.ArtifactoryServicesManager, originalRepo string, repoDetails *RepositoryDetails) ([]utils.ResultItem, error) {
144+
if len(imageLayers) == 0 {
145+
return imageLayers, nil
146+
}
147+
148+
// Optimization: If we already know the repo type and it's not virtual, skip the API call
149+
if repoDetails != nil && repoDetails.repoType != "" && repoDetails.repoType != "virtual" {
150+
log.Debug("Repository ", originalRepo, "is not virtual (type:", repoDetails.repoType+"), skipping determining default deployment config")
151+
return imageLayers, nil
152+
}
153+
154+
// For backwards compatibility or when repoDetails is not available, fall back to API call
155+
if repoDetails == nil || repoDetails.repoType == "" {
156+
log.Debug("Repository type not cached, making API call to determine repository configuration")
157+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
158+
if err != nil {
159+
return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for '%s': %w", originalRepo, err)
160+
}
161+
162+
// If it's not a virtual repository, return all layers unchanged
163+
if repoConfig == nil || repoConfig.Rclass != "virtual" {
164+
log.Debug("Repository", originalRepo, "is not virtual, proceeding with all layers")
165+
return imageLayers, nil
166+
}
167+
168+
// If it's a virtual repository but has no default deployment repo, return all layers
169+
if repoConfig.DefaultDeploymentRepo == "" {
170+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
171+
return imageLayers, nil
172+
}
173+
174+
// Filter layers to only include those from the default deployment repository
175+
var filteredLayers []utils.ResultItem
176+
for _, layer := range imageLayers {
177+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
178+
filteredLayers = append(filteredLayers, layer)
179+
}
180+
}
181+
182+
if len(filteredLayers) == 0 {
183+
log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'.
184+
This may indicate that image layers exist in other repositories but not in the default deployment repository.
185+
Properties will not be set to maintain consistency with virtual repository configuration.
186+
To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
187+
return []utils.ResultItem{}, nil
188+
}
189+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
190+
191+
return filteredLayers, nil
192+
}
193+
194+
log.Info("Determining virtual repository", originalRepo, "config to determine default deployment repository")
195+
repoConfig, err := getRepositoryConfiguration(originalRepo, serviceManager)
196+
if err != nil {
197+
return imageLayers, errorutils.CheckErrorf("failed to get repository configuration for virtual repository '%s': %w", originalRepo, err)
198+
}
199+
200+
// If it's a virtual repository but has no default deployment repo, return all layers
201+
if repoConfig.DefaultDeploymentRepo == "" {
202+
log.Debug("Virtual repository", originalRepo, "has no default deployment repository, proceeding with all layers")
203+
return imageLayers, nil
204+
}
205+
206+
// Filter layers to only include those from the default deployment repository
207+
var filteredLayers []utils.ResultItem
208+
for _, layer := range imageLayers {
209+
if layer.Repo == repoConfig.DefaultDeploymentRepo {
210+
filteredLayers = append(filteredLayers, layer)
211+
}
212+
}
213+
214+
if len(filteredLayers) == 0 {
215+
log.Warn(fmt.Sprintf(`No layers found in default deployment repository '%s' for virtual repository '%s'.
216+
This may indicate that image layers exist in other repositories but not in the default deployment repository.
217+
Properties will not be set to maintain consistency with virtual repository configuration.
218+
To fix this, consider pushing the image directly to the virtual repository to ensure it lands in the default deployment repository.`, repoConfig.DefaultDeploymentRepo, originalRepo))
219+
return []utils.ResultItem{}, nil
220+
}
221+
log.Info("Filtered", len(imageLayers), "layers to", len(filteredLayers), "layers from default deployment repository:", repoConfig.DefaultDeploymentRepo)
222+
223+
return filteredLayers, nil
224+
}
225+
226+
// repositoryConfig represents the virtual repository configuration
227+
type repositoryConfig struct {
228+
Key string `json:"key"`
229+
Rclass string `json:"rclass"`
230+
DefaultDeploymentRepo string `json:"defaultDeploymentRepo"`
231+
}
232+
233+
// getRepositoryConfiguration fetches the repository configuration from Artifactory
234+
func getRepositoryConfiguration(repoKey string, serviceManager artifactory.ArtifactoryServicesManager) (*repositoryConfig, error) {
235+
httpClientDetails := serviceManager.GetConfig().GetServiceDetails().CreateHttpClientDetails()
236+
237+
baseUrl := serviceManager.GetConfig().GetServiceDetails().GetUrl()
238+
endpoint := "api/repositories/" + repoKey
239+
url := baseUrl + endpoint
240+
resp, body, _, err := serviceManager.Client().SendGet(url, true, &httpClientDetails)
241+
if err != nil {
242+
return nil, err
243+
}
244+
if resp.StatusCode != http.StatusOK {
245+
return nil, fmt.Errorf("failed to get repository configuration: HTTP %d", resp.StatusCode)
246+
}
247+
var config repositoryConfig
248+
if err := json.Unmarshal(body, &config); err != nil {
249+
return nil, fmt.Errorf("failed to parse repository configuration: %v", err)
250+
}
251+
252+
return &config, nil
253+
}
254+
108255
// Download the content of layer search result.
109256
func downloadLayer(searchResult utils.ResultItem, result interface{}, serviceManager artifactory.ArtifactoryServicesManager, repo string) error {
110257
// Search results may include artifacts from the remote-cache repository.
@@ -326,7 +473,7 @@ func (builder *buildInfoBuilder) createBuildInfo(commandType CommandType, manife
326473
return nil, err
327474
}
328475
if !builder.skipTaggingLayers {
329-
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager); err != nil {
476+
if err := setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails); err != nil {
330477
return nil, err
331478
}
332479
}
@@ -385,7 +532,7 @@ func (builder *buildInfoBuilder) createMultiPlatformBuildInfo(fatManifest *FatMa
385532
Parent: imageLongNameWithoutRepo,
386533
})
387534
}
388-
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager)
535+
return buildInfo, setBuildProperties(builder.buildName, builder.buildNumber, builder.project, builder.imageLayers, builder.serviceManager, builder.repositoryDetails.key, &builder.repositoryDetails)
389536
}
390537

391538
// Construct the manifest's module ID by its type (attestation) or its platform.

‎artifactory/utils/container/remoteagent.go‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,16 +53,14 @@ func (rabib *RemoteAgentBuildInfoBuilder) Build(module string) (*buildinfo.Build
5353
// Search for image manifest and layers in Artifactory.
5454
func (rabib *RemoteAgentBuildInfoBuilder) handleManifest(resultMap map[string]*utils.ResultItem) (map[string]*utils.ResultItem, *manifest, error) {
5555
if manifest, ok := resultMap["manifest.json"]; ok {
56-
err := rabib.isVerifiedManifest(manifest)
57-
if err != nil {
58-
return nil, nil, err
59-
56+
if !rabib.isVerifiedManifest(manifest) {
57+
log.Debug("Manifest verification failed, continuing with SHA-based validation...")
6058
}
6159
manifest, err := getManifest(resultMap, rabib.buildInfoBuilder.serviceManager, rabib.buildInfoBuilder.repositoryDetails.key)
6260
if err != nil {
6361
return nil, nil, err
6462
}
65-
// // Manifest may hold 'empty layers'. As a result, promotion will fail to promote the same layer more than once.
63+
// Manifest may hold 'empty layers'. As a result, promotion will fail to promote the same layer more than once.
6664
rabib.buildInfoBuilder.imageSha2 = manifest.Config.Digest
6765
log.Debug("Found manifest.json. Proceeding to create build-info.")
6866
return resultMap, manifest, nil
@@ -95,6 +93,8 @@ func (rabib *RemoteAgentBuildInfoBuilder) searchImage() (resultMap map[string]*u
9593
// Search image's manifest.
9694
manifestPathsCandidates := getManifestPaths(imagePath, rabib.buildInfoBuilder.getSearchableRepo(), Push)
9795
log.Debug("Start searching for image manifest.json")
96+
97+
// First try standard tag-based search
9898
for _, path := range manifestPathsCandidates {
9999
log.Debug(`Searching in:"` + path + `"`)
100100
resultMap, err = performSearch(path, rabib.buildInfoBuilder.serviceManager)
@@ -108,15 +108,39 @@ func (rabib *RemoteAgentBuildInfoBuilder) searchImage() (resultMap map[string]*u
108108
return resultMap, nil
109109
}
110110
}
111+
112+
// If tag-based search failed and we have a SHA, try SHA-based search
113+
if rabib.manifestSha2 != "" {
114+
log.Debug("Tag-based search failed. Trying SHA-based search with: " + rabib.manifestSha2)
115+
// Extract repository path without tag
116+
repoPath := imagePath[:strings.LastIndex(imagePath, "/")]
117+
// Convert SHA format from sha256:xxx to sha256__xxx for Artifactory path format
118+
shaPath := strings.Replace(rabib.manifestSha2, ":", "__", 1)
119+
// Search for the image using SHA path
120+
shaSearchPath := repoPath + "/" + shaPath + "/*"
121+
log.Debug(`Searching by SHA in:"` + shaSearchPath + `"`)
122+
resultMap, err = performSearch(shaSearchPath, rabib.buildInfoBuilder.serviceManager)
123+
if err != nil {
124+
return nil, err
125+
}
126+
if resultMap != nil && (resultMap["list.manifest.json"] != nil || resultMap["manifest.json"] != nil) {
127+
log.Info("Found image by SHA digest in repository")
128+
return resultMap, nil
129+
}
130+
}
131+
111132
return nil, errorutils.CheckErrorf(imageNotFoundErrorMessage, rabib.buildInfoBuilder.image.name)
112133
}
113134

114-
// Verify manifest's sha256. If there is no match, return nil.
115-
func (rabib *RemoteAgentBuildInfoBuilder) isVerifiedManifest(imageManifest *utils.ResultItem) error {
135+
// Verify manifest's sha256. Returns true if manifest is verified, false otherwise.
136+
func (rabib *RemoteAgentBuildInfoBuilder) isVerifiedManifest(imageManifest *utils.ResultItem) bool {
116137
if imageManifest.GetProperty("docker.manifest.digest") != rabib.manifestSha2 {
117-
return errorutils.CheckErrorf(`Found incorrect manifest.json file. Expects digest "` + rabib.manifestSha2 + `" found "` + imageManifest.GetProperty("docker.manifest.digest"))
138+
manifestDigest := imageManifest.GetProperty("docker.manifest.digest")
139+
log.Warn("Manifest digest mismatch detected. Local image digest: " + rabib.manifestSha2 + ", Repository digest: " + manifestDigest)
140+
log.Info("Proceeding with SHA-based validation to ensure correct image identification...")
141+
return false
118142
}
119-
return nil
143+
return true
120144
}
121145

122146
func getFatManifestRoot(fatManifestPath string) string {

‎artifactory/utils/utils.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/jfrog/jfrog-client-go/jfconnect"
3030
"github.com/jfrog/jfrog-client-go/lifecycle"
3131
"github.com/jfrog/jfrog-client-go/metadata"
32+
"github.com/jfrog/jfrog-client-go/onemodel"
3233
clientUtils "github.com/jfrog/jfrog-client-go/utils"
3334
"github.com/jfrog/jfrog-client-go/utils/errorutils"
3435
ioUtils "github.com/jfrog/jfrog-client-go/utils/io"
@@ -263,6 +264,27 @@ func CreateMetadataServiceManager(serviceDetails *config.ServerDetails, isDryRun
263264
return metadata.NewManager(serviceConfig)
264265
}
265266

267+
func CreateOnemodelServiceManager(serviceDetails *config.ServerDetails, isDryRun bool) (onemodel.Manager, error) {
268+
certsPath, err := coreutils.GetJfrogCertsDir()
269+
if err != nil {
270+
return nil, err
271+
}
272+
mdAuth, err := serviceDetails.CreateOnemodelAuthConfig()
273+
if err != nil {
274+
return nil, err
275+
}
276+
serviceConfig, err := clientConfig.NewConfigBuilder().
277+
SetServiceDetails(mdAuth).
278+
SetCertificatesPath(certsPath).
279+
SetInsecureTls(serviceDetails.InsecureTls).
280+
SetDryRun(isDryRun).
281+
Build()
282+
if err != nil {
283+
return nil, err
284+
}
285+
return onemodel.NewManager(serviceConfig)
286+
}
287+
266288
func CreateJfConnectServiceManager(serverDetails *config.ServerDetails, httpRetries, httpRetryWaitMilliSecs int) (jfconnect.Manager, error) {
267289
certsPath, err := coreutils.GetJfrogCertsDir()
268290
if err != nil {

0 commit comments

Comments
 (0)