Allow multiple LogStorage with primary and secondaries

README.md

@@ -11,6 +11,17 @@ Plugin that defines Pipeline API.
 A component of [Pipeline
 Plugin](https://plugins.jenkins.io/workflow-aggregator).
 
+# JEP-210: External log storage for Pipeline
+## Implementation
+This plugin provides APIs for [https://github.com/jenkinsci/jep/tree/master/jep/210](JEP-210), which allow plugins to take over Pipeline build logging.
+The default logging implementation is the `@Extension LogStorageFactoryDescriptor` with the highest `ordinal` value that implements `LogStorageFactoryDescriptor.getDefaultInstance`.
+You can override the default implementation by configuring a logger explicitly under "Pipeline logger" on the Jenkins system configuration page.
+
+## Multiple loggers
+In some cases, you may want to use a logging implementation that sends logs to an external system, while also preserving logs in Jenkins for other use cases.
+You can accomplish by configuring the "Multiple loggers" implementation in the "Pipeline logger" section on the Jenkins system configuration page.
+This implementation allows you to select a "Primary" logger that handles reads and writes, as well as a "Secondary" logger which receives copies of all writes, similarly to the Unix `tee` command.
+
 # Changelog
 
 * For new versions, see [GitHub Releases](https://github.com/jenkinsci/workflow-api-plugin/releases)

pom.xml

@@ -138,5 +138,10 @@
           <artifactId>apache-httpcomponents-client-4-api</artifactId>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>io.jenkins.configuration-as-code</groupId>
+          <artifactId>test-harness</artifactId>
+          <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

src/main/java/org/jenkinsci/plugins/workflow/log/FileLogStorageFactory.java

@@ -0,0 +1,37 @@
+package org.jenkinsci.plugins.workflow.log;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.Extension;
+import hudson.model.Descriptor;
+import java.io.File;
+import org.jenkinsci.Symbol;
+import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+import org.kohsuke.stapler.DataBoundConstructor;
+
+@Restricted(Beta.class)
+public class FileLogStorageFactory implements LogStorageFactory {
+
+    @DataBoundConstructor
+    public FileLogStorageFactory() {}
+
+    @Override
+    public LogStorage forBuild(@NonNull FlowExecutionOwner b) {
+        try {
+            return FileLogStorage.forFile(new File(b.getRootDir(), "log"));
+        } catch (Exception x) {
+            return new BrokenLogStorage(x);
+        }
+    }
+
+    @Extension
+    @Symbol("file")
+    public static final class DescriptorImpl extends LogStorageFactoryDescriptor<FileLogStorageFactory> {
+        @NonNull
+        @Override
+        public String getDisplayName() {
+            return "Standard file logger";
+        }
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorage.java

@@ -25,7 +25,6 @@
 package org.jenkinsci.plugins.workflow.log;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import hudson.ExtensionList;
 import hudson.console.AnnotatedLargeText;
 import hudson.console.ConsoleAnnotationOutputStream;
 import hudson.model.BuildListener;
@@ -39,6 +38,7 @@
 import java.util.logging.Logger;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import org.jenkinsci.plugins.workflow.actions.LogAction;
+import org.jenkinsci.plugins.workflow.log.configuration.PipelineLoggingGlobalConfiguration;
 import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
 import org.jenkinsci.plugins.workflow.graph.FlowNode;
 import org.jenkinsci.plugins.workflow.steps.StepContext;
@@ -160,8 +160,9 @@ public interface LogStorage {
      */
     static @NonNull LogStorage of(@NonNull FlowExecutionOwner b) {
         try {
-            for (LogStorageFactory factory : ExtensionList.lookup(LogStorageFactory.class)) {
-                LogStorage storage = factory.forBuild(b);
+            PipelineLoggingGlobalConfiguration config = PipelineLoggingGlobalConfiguration.get();
+            if (config.getFactoryOrDefault() != null) {
+                LogStorage storage = config.getFactoryOrDefault().forBuild(b);
                 if (storage != null) {
                     // Pending integration with JEP-207 / JEP-212, this choice is not persisted.
                     return storage;

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactory.java

@@ -24,9 +24,11 @@
 
 package org.jenkinsci.plugins.workflow.log;
 
-import hudson.ExtensionPoint;
 import edu.umd.cs.findbugs.annotations.CheckForNull;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.model.Describable;
+import java.util.List;
+import jenkins.model.Jenkins;
 import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
 import org.kohsuke.accmod.Restricted;
 import org.kohsuke.accmod.restrictions.Beta;
@@ -35,13 +37,33 @@
  * Factory interface for {@link LogStorage}.
  */
 @Restricted(Beta.class)
-public interface LogStorageFactory extends ExtensionPoint {
+public interface LogStorageFactory extends Describable<LogStorageFactory> {
 
     /**
-     * Checks whether we should handle a given build.
+     * When the current factory has been configured or is considered a default factory {@link #getDefaultFactory()}, returns the expected log storage instance to handle the build.
      * @param b a build about to start
-     * @return a mechanism for handling this build, or null to fall back to the next implementation or the default
+     * @return a mechanism for handling this build, see {@link LogStorage#of(FlowExecutionOwner)}
      */
     @CheckForNull LogStorage forBuild(@NonNull FlowExecutionOwner b);
 
+    default LogStorageFactoryDescriptor<?> getDescriptor() {
+        return (LogStorageFactoryDescriptor<?>) Jenkins.get().getDescriptorOrDie(this.getClass());
+    }
+
+    static List<LogStorageFactoryDescriptor<?>> all() {
+        return Jenkins.get().getDescriptorList(LogStorageFactory.class);
+    }
+
+    /**
+     * Returns the default {@link LogStorageFactory} based on the descriptor {@code @Extension#ordinal} order and the {@link LogStorageFactoryDescriptor#getDefaultInstance()} implmentations.
+     */
+    static LogStorageFactory getDefaultFactory() {
+        for (LogStorageFactoryDescriptor<?> descriptor : all()) {
+            var instance = descriptor.getDefaultInstance();
+            if (instance != null) {
+                return instance;
+            }
+        }
+        return new FileLogStorageFactory();
+    }
 }

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactoryDescriptor.java

@@ -0,0 +1,29 @@
+package org.jenkinsci.plugins.workflow.log;
+
+import hudson.model.Descriptor;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+
+@Restricted(Beta.class)
+public abstract class LogStorageFactoryDescriptor<T extends LogStorageFactory> extends Descriptor<LogStorageFactory> {
+
+    /**
+     * Indicates whether the factory supports being used in read/write mode (e.g. as a top-level logger, or as a primary for {@link org.jenkinsci.plugins.workflow.log.tee.TeeLogStorageFactory})
+     */
+    public boolean isReadWrite() {
+        return true;
+    }
+    /**
+     * Indicates whether the factory supports being used in write-only mode (as a secondary for {@link org.jenkinsci.plugins.workflow.log.tee.TeeLogStorageFactory}).
+     */
+    public boolean isWriteOnly() {
+        return true;
+    }
+
+    /**
+     * Allow to define the default factory instance to use if no configuration exists
+     */
+    public LogStorageFactory getDefaultInstance() {
+        return null;
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/configuration/PipelineLoggingGlobalConfiguration.java

@@ -0,0 +1,84 @@
+package org.jenkinsci.plugins.workflow.log.configuration;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.Extension;
+import hudson.ExtensionList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+import jenkins.model.GlobalConfiguration;
+import jenkins.model.Jenkins;
+import net.sf.json.JSONObject;
+import org.jenkinsci.Symbol;
+import org.jenkinsci.plugins.workflow.log.LogStorageFactory;
+import org.jenkinsci.plugins.workflow.log.LogStorageFactoryDescriptor;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.DataBoundSetter;
+import org.kohsuke.stapler.StaplerRequest2;
+
+@Extension
+@Symbol("pipelineLogging")
+@Restricted(Beta.class)
+public class PipelineLoggingGlobalConfiguration extends GlobalConfiguration {
+    private static final Logger LOGGER = Logger.getLogger(PipelineLoggingGlobalConfiguration.class.getName());
+    private LogStorageFactory factory;
+
+    public PipelineLoggingGlobalConfiguration() {
+        load();
+    }
+
+    /**
+     * For configuration only. Use {@link #getFactoryOrDefault()} instead.
+     */
+    @Restricted(NoExternalUse.class)
+    public LogStorageFactory getFactory() {
+        return factory;
+    }
+
+    @DataBoundSetter
+    public void setFactory(LogStorageFactory factory) {
+        this.factory = factory;
+        save();
+    }
+
+    @Override
+    public boolean configure(StaplerRequest2 req, JSONObject json) throws FormException {
+        this.factory = null;
+        return super.configure(req, json);
+    }
+
+    public LogStorageFactory getFactoryOrDefault() {
+        if (factory == null) {
+            return LogStorageFactory.getDefaultFactory();
+        }
+        return factory;
+    }
+
+    public List<LogStorageFactoryDescriptor<?>> getLogStorageFactoryDescriptors() {
+        List<LogStorageFactoryDescriptor<?>> result = new ArrayList<>();
+        result.add(null); // offer the option to use the default factory without any explicit configuration
+        result.addAll(getFilteredLogStorageFactoryDescriptors());
+        return result;
+    }
+
+    private List<LogStorageFactoryDescriptor<?>> getFilteredLogStorageFactoryDescriptors() {
+        return LogStorageFactory.all().stream()
+                                .filter(LogStorageFactoryDescriptor::isReadWrite)
+                                .toList();
+    }
+
+    public LogStorageFactoryDescriptor<?> getDefaultFactoryDescriptor() {
+        return LogStorageFactory.getDefaultFactory().getDescriptor();
+    }
+
+    public String getDefaultFactoryPlugin() {
+        var pluginWrapper = Jenkins.get().getPluginManager().whichPlugin(LogStorageFactory.getDefaultFactory().getClass());
+        return pluginWrapper != null ? pluginWrapper.getShortName() : "unknown";
+    }
+
+    public static PipelineLoggingGlobalConfiguration get() {
+        return ExtensionList.lookupSingleton(PipelineLoggingGlobalConfiguration.class);
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/tee/TeeBuildListener.java

@@ -0,0 +1,69 @@
+package org.jenkinsci.plugins.workflow.log.tee;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.model.BuildListener;
+import hudson.model.TaskListener;
+import java.io.OutputStream;
+import java.io.Serial;
+import java.util.List;
+import org.jenkinsci.plugins.workflow.log.OutputStreamTaskListener;
+
+class TeeBuildListener extends OutputStreamTaskListener.Default
+        implements BuildListener, OutputStreamTaskListener, AutoCloseable {
+
+    @Serial
+    private static final long serialVersionUID = 1L;
+
+    private final TaskListener primary;
+
+    private final List<TaskListener> secondaries;
+
+    private transient OutputStream outputStream;
+
+    TeeBuildListener(TaskListener primary, TaskListener... secondaries) {
+        this.primary = primary;
+        this.secondaries = List.of(secondaries);
+    }
+
+    @NonNull
+    @Override
+    public synchronized OutputStream getOutputStream() {
+        if (outputStream == null) {
+            outputStream = new TeeOutputStream(
+                    OutputStreamTaskListener.getOutputStream(primary),
+                    secondaries.stream()
+                            .map(OutputStreamTaskListener::getOutputStream)
+                            .toArray(OutputStream[]::new));
+        }
+        return outputStream;
+    }
+
+    @Override
+    public void close() throws Exception {
+        getLogger().close();
+        Exception exception = null;
+        if (primary instanceof AutoCloseable) {
+            try {
+                ((AutoCloseable) primary).close();
+            } catch (Exception e) {
+                exception = e;
+            }
+        }
+        for (TaskListener secondary : secondaries) {
+            if (secondary instanceof AutoCloseable) {
+                try {
+                    ((AutoCloseable) secondary).close();
+                } catch (Exception e) {
+                    if (exception == null) {
+                        exception = e;
+                    } else {
+                        exception.addSuppressed(e);
+                    }
+                }
+            }
+        }
+        if (exception != null) {
+            throw exception;
+        }
+    }
+}