Create LinuxApplicationLayerDetector (#1551)

* Create `LinuxApplicationLayerDetector`

* Code formatting

* Include npm and pip detectors in experiment

* Increment LinuxContainerDetector version

* Add support for filtering by component type

* Fix LinuxContainerDetectorTests

Author: JamieMagee

.editorconfig

@@ -463,6 +463,9 @@ dotnet_diagnostic.SA1623.severity = none
 # https://github.com/DotNetAnalyzers/StyleCopAnalyzers
 ##########################################
 
+dotnet_diagnostic.SA1009.severity = none
+dotnet_diagnostic.SA1111.severity = none
+
 # https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/SA1600.md
 # Elements should be documented
 dotnet_diagnostic.SA1600.severity = suggestion

src/Microsoft.ComponentDetection.Detectors/linux/ILinuxScanner.cs

@@ -5,6 +5,7 @@ namespace Microsoft.ComponentDetection.Detectors.Linux;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.ComponentDetection.Contracts.BcdeModels;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
 
 /// <summary>
 /// Interface for scanning Linux container layers to identify components.
@@ -17,7 +18,14 @@ public interface ILinuxScanner
     /// <param name="imageHash">The hash identifier of the container image to scan.</param>
     /// <param name="containerLayers">The collection of Docker layers that make up the container image.</param>
     /// <param name="baseImageLayerCount">The number of layers that belong to the base image, used to distinguish base image layers from application layers.</param>
+    /// <param name="enabledComponentTypes">The set of component types to include in the scan results. Only components matching these types will be returned.</param>
     /// <param name="cancellationToken">A token to monitor for cancellation requests. The default value is <see cref="CancellationToken.None"/>.</param>
     /// <returns>A task that represents the asynchronous operation. The task result contains a collection of <see cref="LayerMappedLinuxComponents"/> representing the components found in the image and their associated layers.</returns>
-    public Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string imageHash, IEnumerable<DockerLayer> containerLayers, int baseImageLayerCount, CancellationToken cancellationToken = default);
+    public Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(
+        string imageHash,
+        IEnumerable<DockerLayer> containerLayers,
+        int baseImageLayerCount,
+        ISet<ComponentType> enabledComponentTypes,
+        CancellationToken cancellationToken = default
+    );
 }

src/Microsoft.ComponentDetection.Detectors/linux/LinuxApplicationLayerDetector.cs

@@ -0,0 +1,42 @@
+#nullable disable
+namespace Microsoft.ComponentDetection.Detectors.Linux;
+
+using System;
+using System.Collections.Generic;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+using Microsoft.Extensions.Logging;
+
+/// <summary>
+/// Experimental detector for Linux container images that captures application-level packages (npm and pip)
+/// in addition to system packages. This detector runs as an experiment to compare results with the base
+/// Linux detector (which only scans system packages).
+/// </summary>
+/// <param name="linuxScanner">The Linux scanner service.</param>
+/// <param name="dockerService">The Docker service.</param>
+/// <param name="logger">The logger.</param>
+public class LinuxApplicationLayerDetector(
+    ILinuxScanner linuxScanner,
+    IDockerService dockerService,
+    ILogger<LinuxApplicationLayerDetector> logger
+) : LinuxContainerDetector(linuxScanner, dockerService, logger), IExperimentalDetector
+{
+    /// <inheritdoc/>
+    public new string Id => "LinuxApplicationLayer";
+
+    /// <inheritdoc/>
+    public new IEnumerable<string> Categories =>
+        [
+            Enum.GetName(typeof(DetectorClass), DetectorClass.Linux),
+            Enum.GetName(typeof(DetectorClass), DetectorClass.Npm),
+            Enum.GetName(typeof(DetectorClass), DetectorClass.Pip),
+        ];
+
+    /// <inheritdoc/>
+    public new IEnumerable<ComponentType> SupportedComponentTypes =>
+        [ComponentType.Linux, ComponentType.Npm, ComponentType.Pip];
+
+    /// <inheritdoc/>
+    protected override ISet<ComponentType> GetEnabledComponentTypes() =>
+        new HashSet<ComponentType> { ComponentType.Linux, ComponentType.Npm, ComponentType.Pip };
+}

src/Microsoft.ComponentDetection.Detectors/linux/LinuxContainerDetector.cs

@@ -21,7 +21,8 @@ namespace Microsoft.ComponentDetection.Detectors.Linux;
 /// </summary>
 public class LinuxScanner : ILinuxScanner
 {
-    private const string ScannerImage = "governancecontainerregistry.azurecr.io/syft:v1.37.0@sha256:48d679480c6d272c1801cf30460556959c01d4826795be31d4fd8b53750b7d91";
+    private const string ScannerImage =
+        "governancecontainerregistry.azurecr.io/syft:v1.37.0@sha256:48d679480c6d272c1801cf30460556959c01d4826795be31d4fd8b53750b7d91";
 
     private static readonly IList<string> CmdParameters =
     [
@@ -34,13 +35,19 @@ public class LinuxScanner : ILinuxScanner
 
     private static readonly SemaphoreSlim ContainerSemaphore = new SemaphoreSlim(2);
 
-    private static readonly int SemaphoreTimeout = Convert.ToInt32(TimeSpan.FromHours(1).TotalMilliseconds);
+    private static readonly int SemaphoreTimeout = Convert.ToInt32(
+        TimeSpan.FromHours(1).TotalMilliseconds
+    );
 
     private readonly IDockerService dockerService;
     private readonly ILogger<LinuxScanner> logger;
     private readonly IEnumerable<IArtifactComponentFactory> componentFactories;
     private readonly IEnumerable<IArtifactFilter> artifactFilters;
     private readonly Dictionary<string, IArtifactComponentFactory> factoryLookup;
+    private readonly Dictionary<
+        ComponentType,
+        IArtifactComponentFactory
+    > componentTypeToFactoryLookup;
 
     /// <summary>
     /// Initializes a new instance of the <see cref="LinuxScanner"/> class.
@@ -53,7 +60,8 @@ public LinuxScanner(
         IDockerService dockerService,
         ILogger<LinuxScanner> logger,
         IEnumerable<IArtifactComponentFactory> componentFactories,
-        IEnumerable<IArtifactFilter> artifactFilters)
+        IEnumerable<IArtifactFilter> artifactFilters
+    )
     {
         this.dockerService = dockerService;
         this.logger = logger;
@@ -69,10 +77,27 @@ public LinuxScanner(
                 this.factoryLookup[artifactType] = factory;
             }
         }
+
+        // Build a lookup dictionary for component type to factory mapping
+        this.componentTypeToFactoryLookup = new Dictionary<ComponentType, IArtifactComponentFactory>
+        {
+            {
+                ComponentType.Linux,
+                componentFactories.FirstOrDefault(f => f is LinuxComponentFactory)
+            },
+            { ComponentType.Npm, componentFactories.FirstOrDefault(f => f is NpmComponentFactory) },
+            { ComponentType.Pip, componentFactories.FirstOrDefault(f => f is PipComponentFactory) },
+        };
     }
 
     /// <inheritdoc/>
-    public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string imageHash, IEnumerable<DockerLayer> containerLayers, int baseImageLayerCount, CancellationToken cancellationToken = default)
+    public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(
+        string imageHash,
+        IEnumerable<DockerLayer> containerLayers,
+        int baseImageLayerCount,
+        ISet<ComponentType> enabledComponentTypes,
+        CancellationToken cancellationToken = default
+    )
     {
         using var record = new LinuxScannerTelemetryRecord
         {
@@ -93,8 +118,14 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
             {
                 try
                 {
-                    var command = new List<string> { imageHash }.Concat(CmdParameters).ToList();
-                    (stdout, stderr) = await this.dockerService.CreateAndRunContainerAsync(ScannerImage, command, cancellationToken);
+                    var command = new List<string> { imageHash }
+                        .Concat(CmdParameters)
+                        .ToList();
+                    (stdout, stderr) = await this.dockerService.CreateAndRunContainerAsync(
+                        ScannerImage,
+                        command,
+                        cancellationToken
+                    );
                 }
                 catch (Exception e)
                 {
@@ -106,7 +137,10 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
             else
             {
                 record.SemaphoreFailure = true;
-                this.logger.LogWarning("Failed to enter the container semaphore for image {ImageHash}", imageHash);
+                this.logger.LogWarning(
+                    "Failed to enter the container semaphore for image {ImageHash}",
+                    imageHash
+                );
             }
         }
         finally
@@ -123,14 +157,13 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
         if (string.IsNullOrWhiteSpace(stdout) || !string.IsNullOrWhiteSpace(stderr))
         {
             throw new InvalidOperationException(
-                $"Scan failed with exit info: {stdout}{System.Environment.NewLine}{stderr}");
+                $"Scan failed with exit info: {stdout}{System.Environment.NewLine}{stderr}"
+            );
         }
 
         var layerDictionary = containerLayers
             .DistinctBy(layer => layer.DiffId)
-            .ToDictionary(
-                layer => layer.DiffId,
-                _ => new List<TypedComponent>());
+            .ToDictionary(layer => layer.DiffId, _ => new List<TypedComponent>());
 
         try
         {
@@ -143,10 +176,25 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
                 validArtifacts = filter.Filter(validArtifacts, syftOutput.Distro);
             }
 
-            // Create components using factories
+            // Build a set of enabled factories based on requested component types
+            var enabledFactories = new HashSet<IArtifactComponentFactory>();
+            foreach (var componentType in enabledComponentTypes)
+            {
+                if (
+                    this.componentTypeToFactoryLookup.TryGetValue(componentType, out var factory)
+                    && factory != null
+                )
+                {
+                    enabledFactories.Add(factory);
+                }
+            }
+
+            // Create components using only enabled factories
             var componentsWithLayers = validArtifacts
                 .DistinctBy(artifact => (artifact.Name, artifact.Version, artifact.Type))
-                .Select(artifact => this.CreateComponentWithLayers(artifact, syftOutput.Distro))
+                .Select(artifact =>
+                    this.CreateComponentWithLayers(artifact, syftOutput.Distro, enabledFactories)
+                )
                 .Where(result => result.Component != null)
                 .ToList();
 
@@ -159,7 +207,10 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
 
             if (unsupportedTypes.Count > 0)
             {
-                this.logger.LogDebug("Encountered unsupported artifact types: {UnsupportedTypes}", string.Join(", ", unsupportedTypes));
+                this.logger.LogDebug(
+                    "Encountered unsupported artifact types: {UnsupportedTypes}",
+                    string.Join(", ", unsupportedTypes)
+                );
             }
 
             // Map components to layers
@@ -179,7 +230,9 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
             });
 
             // Track detected components in telemetry
-            syftTelemetryRecord.Components = JsonConvert.SerializeObject(componentsWithLayers.Select(c => c.Component.Id));
+            syftTelemetryRecord.Components = JsonConvert.SerializeObject(
+                componentsWithLayers.Select(c => c.Component.Id)
+            );
 
             return layerMappedLinuxComponents;
         }
@@ -190,13 +243,23 @@ public async Task<IEnumerable<LayerMappedLinuxComponents>> ScanLinuxAsync(string
         }
     }
 
-    private (TypedComponent Component, IEnumerable<string> LayerIds) CreateComponentWithLayers(ArtifactElement artifact, Distro distro)
+    private (TypedComponent Component, IEnumerable<string> LayerIds) CreateComponentWithLayers(
+        ArtifactElement artifact,
+        Distro distro,
+        HashSet<IArtifactComponentFactory> enabledFactories
+    )
     {
         if (!this.factoryLookup.TryGetValue(artifact.Type, out var factory))
         {
             return (null, []);
         }
 
+        // Skip this artifact if its factory is not in the enabled set
+        if (!enabledFactories.Contains(factory))
+        {
+            return (null, []);
+        }
+
         var component = factory.CreateComponent(artifact, distro);
         if (component == null)
         {

src/Microsoft.ComponentDetection.Detectors/linux/LinuxScanner.cs

@@ -0,0 +1,33 @@
+namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs;
+
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Detectors.Linux;
+using Microsoft.ComponentDetection.Detectors.Npm;
+using Microsoft.ComponentDetection.Detectors.Pip;
+
+/// <summary>
+/// Experiment to validate the <see cref="LinuxApplicationLayerDetector"/> which captures application-level
+/// packages in addition to system packages from Linux containers.
+/// Control group includes the standard file-based npm and pip detectors plus the Linux system package detector.
+/// Experiment group uses container-based detection for all package types together.
+/// </summary>
+public class LinuxApplicationLayerExperiment : IExperimentConfiguration
+{
+    /// <inheritdoc />
+    public string Name => "LinuxApplicationLayer";
+
+    /// <inheritdoc />
+    public bool IsInControlGroup(IComponentDetector componentDetector) =>
+        componentDetector
+            is (LinuxContainerDetector and not LinuxApplicationLayerDetector)
+                or NpmComponentDetector
+                or NpmLockfileDetectorBase
+                or PipReportComponentDetector;
+
+    /// <inheritdoc />
+    public bool IsInExperimentGroup(IComponentDetector componentDetector) =>
+        componentDetector is LinuxApplicationLayerDetector;
+
+    /// <inheritdoc />
+    public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true;
+}

src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/LinuxApplicationLayerExperiment.cs

@@ -72,6 +72,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
         services.AddSingleton<IExperimentProcessor, DefaultExperimentProcessor>();
         services.AddSingleton<IExperimentConfiguration, SimplePipExperiment>();
         services.AddSingleton<IExperimentConfiguration, UvLockDetectorExperiment>();
+        services.AddSingleton<IExperimentConfiguration, LinuxApplicationLayerExperiment>();
 
         // Detectors
         // CocoaPods
@@ -106,6 +107,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s
         services.AddSingleton<IArtifactComponentFactory, PipComponentFactory>();
         services.AddSingleton<IArtifactFilter, Mariner2ArtifactFilter>();
         services.AddSingleton<IComponentDetector, LinuxContainerDetector>();
+        services.AddSingleton<IComponentDetector, LinuxApplicationLayerDetector>();
 
         // Maven
         services.AddSingleton<IMavenCommandService, MavenCommandService>();