diff --git a/.gitignore b/.gitignore index 3b6abb0..e04b7e8 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ cookies.txt # Vim # .*.sw* +*.tif diff --git a/src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java b/src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java index 12e6561..d63c9f4 100644 --- a/src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java +++ b/src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java @@ -181,6 +181,7 @@ public static void writeN5File(final AbstractSequenceDescription seq, int numCompletedTasks = 0; final ExecutorService executorService = Executors.newFixedThreadPool( numCellCreatorThreads ); + try (N5Writer n5 = writerSupplier.get()) { // write image data for all views final int numTimepoints = timepointIds.size(); diff --git a/src/main/java/cz/it4i/fiji/legacy/CloneEmptyDatasetFromN5toZarr.java b/src/main/java/cz/it4i/fiji/legacy/CloneEmptyDatasetFromN5toZarr.java new file mode 100644 index 0000000..5466936 --- /dev/null +++ b/src/main/java/cz/it4i/fiji/legacy/CloneEmptyDatasetFromN5toZarr.java @@ -0,0 +1,160 @@ +package cz.it4i.fiji.legacy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import cz.it4i.fiji.legacy.util.GuiResolutionLevelParams; +import cz.it4i.fiji.rest.util.DatasetInfo; +import net.imagej.Dataset; +import org.scijava.ItemIO; +import org.scijava.ItemVisibility; +import org.scijava.command.Command; +import org.scijava.command.CommandInfo; +import org.scijava.command.CommandModule; +import org.scijava.command.CommandService; +import org.scijava.log.LogLevel; +import org.scijava.log.LogService; +import org.scijava.log.Logger; +import org.scijava.module.Module; +import org.scijava.module.ModuleException; +import org.scijava.module.process.PostprocessorPlugin; +import org.scijava.module.process.PreprocessorPlugin; +import org.scijava.plugin.Parameter; +import org.scijava.plugin.Plugin; +import org.scijava.plugin.PluginService; +import org.scijava.prefs.PrefService; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +@Plugin(type = Command.class, headless = true, menuPath = "Plugins>HPC DataStore>ZARR Convert>Create new dataset from existing N5") + +public class CloneEmptyDatasetFromN5toZarr implements Command { + @Parameter(label = "URL of a reference DatasetsRegisterService:", persistKey = "datasetserverurl") + public String src_url = "someHostname:9080"; + + @Parameter(label = "UUID of a reference N5 dataset on that service:", persistKey = "datasetdatasetid") + public String src_uuid = "someDatasetUUID"; + + @Parameter(label = "URL of a target DatasetsRegisterService:", persistKey = "datasetserverurl") + public String tgt_url = "someHostname:9080"; + + @Parameter(label = "Modify params before creating the new dataset:") + public boolean shallUpdateParams = false; + + @Parameter + public PrefService prefs; + + @Parameter + public CommandService cs; + @Parameter + public PluginService ps; + + @Parameter + public LogService mainLogger; + protected Logger myLogger; + + @Parameter(label = "Report corresponding macro command:", required = false) + public boolean showRunCmd = false; + + @Parameter(type = ItemIO.OUTPUT, label="UUID of the created Zarr dataset:") + public String newDatasetUUID; + @Parameter(type = ItemIO.OUTPUT, label="Label of the created Zarr dataset:") + public String newDatasetLabel; + @Override + public void run() { + //logging facility + myLogger = mainLogger.subLogger("HPC CloneDataset", LogLevel.INFO); + + try { + //retrieve reference data + myLogger.info("Reading "+src_uuid+" from "+src_url); + DatasetInfo di = DatasetInfo.createFrom(src_url, src_uuid); + myLogger.info(di.toString()); + + if (!shallUpdateParams) { + final String json = new ObjectMapper().writeValueAsString(di); + myLogger.info("CREATED JSON:\n"+json); + + final Future subcall = cs.run(CreateNewDatasetFromJSON.class, true, + "url",tgt_url, "json",json, "showRunCmd",showRunCmd); + newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID"); + newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel"); + } else { + //we gonna preset the prefs store so that the "params of resolution level" dialog(s) + //will get preloaded with the reference values + final GuiResolutionLevelParams prefsHandler = new GuiResolutionLevelParams(); + prefsHandler.prefs = prefs; + for (int level = 0; level < di.resolutionLevels.size(); ++level) { + final DatasetInfo.ResolutionLevel l = di.resolutionLevels.get(level); + prefsHandler.resLevelNumber=level+1; + prefsHandler.down_x = l.resolutions.get(0); + prefsHandler.down_y = l.resolutions.get(1); + prefsHandler.down_z = l.resolutions.get(2); + prefsHandler.block_x = l.blockDimensions.get(0); + prefsHandler.block_y = l.blockDimensions.get(1); + prefsHandler.block_z = l.blockDimensions.get(2); + prefsHandler.storeIntoPrefs(); + } + + final CommandModule cm = new CommandModule( new CommandInfo(CreateNewDataset.class) ); + cm.setInput("datasetType","Zarr"); + cm.setInput("url", tgt_url); + cm.setInput("label", di.label); + cm.setInput("voxelType", di.voxelType); + cm.setInput("fullResSizeX", di.dimensions.get(0)); + cm.setInput("fullResSizeY", di.dimensions.get(1)); + cm.setInput("fullResSizeZ", di.dimensions.get(2)); + cm.setInput("timepoints", di.timepoints); + cm.setInput("channels", di.channels); + cm.setInput("angles", di.angles); + cm.setInput("res_x", di.voxelResolution.get(0)); + cm.setInput("res_y", di.voxelResolution.get(1)); + cm.setInput("res_z", di.voxelResolution.get(2)); + cm.setInput("res_unit", di.voxelUnit); + + if (di.timepointResolution != null) { + cm.setInput("time_res", di.timepointResolution.value); + cm.setInput("time_unit", di.timepointResolution.unit); + } + if (di.channelResolution != null) { + cm.setInput("channel_res", di.channelResolution.value); + cm.setInput("channel_unit", di.channelResolution.unit); + } + if (di.angleResolution != null) { + cm.setInput("angle_res", di.angleResolution.value); + cm.setInput("angle_unit", di.angleResolution.unit); + } + cm.setInput("numberOfAllResLevels", di.resolutionLevels.size()); + cm.setInput("compression", di.compression); + cm.setInput("showRunCmd", showRunCmd); + + // 1) this is a standard list of pre- and post-processors, + // this is how ModuleService.java creates them for every call of run() + final List pre = ps.createInstancesOfType(PreprocessorPlugin.class); + final List post = ps.createInstancesOfType(PostprocessorPlugin.class); + + // 2) but we find and remove one particular service... + for (int i = 0; i < pre.size(); ++i) + if (pre.get(i).getClass().getSimpleName().startsWith("LoadInputsPreprocessor")) { + pre.remove(i); + break; + } + + // 3) finally, the command (module actually) is started + final Future subcall = cs.moduleService().run(cm,pre,post); + + newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID"); + newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel"); + } + } catch (IOException | ModuleException e) { + myLogger.error("Some error accessing the reference service and dataset: "+e.getMessage()); + } catch (ExecutionException e) { + myLogger.error(e.getMessage()); + + } catch (InterruptedException e) { + //dialog interrupted, do nothing special... + myLogger.error("eee!? "+e.getMessage()); + } + } +} diff --git a/src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java b/src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java index d7b542a..5080f31 100644 --- a/src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java +++ b/src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java @@ -97,6 +97,13 @@ public class CreateNewDataset implements Command { @Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:") public String newDatasetLabel; + @Parameter(label = "DatasetType:", choices = { "N5", "Zarr" }) + public String datasetType; + protected String getDatasetType() + { + return this.datasetType; + } + @Override public void run() { //logging facility @@ -115,9 +122,18 @@ public void run() { di.timepointResolution = new DatasetInfo.ResolutionWithOwnUnit(time_res, time_unit); di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit); di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit); - - di.compression = this.compression.equals("none") ? "raw" : this.compression; - + //This solution was chosen so that there was no need to change the DatasetDTO object, + // so the information about the dataset type is placed after the compression with the delimiter '/', + // according to which it is then divided in the registerServiceEndpoint and only the actual value of the compression is stored in the compression. + // datasetType is then normally stored in the Dataset class and is used to select a suitable handler. + // The most likely solution would be to add the datasetType attribute to the DatasetDTO class, but unfortunately this was not entirely within my power. + if(this.datasetType.equals("Zarr")) { + di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr"; + } + else + { + di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5"; + } di.versions = Collections.emptyList(); di.resolutionLevels = new ArrayList<>(numberOfAllResLevels); di.label = label; @@ -162,7 +178,7 @@ public void run() { myLogger.info("CREATED JSON:\n"+json); final Future subcall = cs.run(CreateNewDatasetFromJSON.class, true, - "url",url, "json",json, "showRunCmd",showRunCmd); + "url",url, "json",json, "showRunCmd",showRunCmd,"datasetType",getDatasetType()); newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID"); newDatasetLabel = di.getLabel(); } catch (ExecutionException e) { diff --git a/src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java b/src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java index 3720c2b..c87a334 100644 --- a/src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java +++ b/src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java @@ -115,6 +115,9 @@ void readParams() { @Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:") public String newDatasetLabel; + @Parameter(label = "DatasetType:", choices = { "N5", "Zarr" }) + public String datasetType; + @Override public void run() { //logging facility @@ -134,7 +137,13 @@ public void run() { di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit); di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit); - di.compression = this.compression.equals("none") ? "raw" : this.compression; + if(this.datasetType.equals("Zarr")) { + di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr"; + } + else + { + di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5"; + } di.versions = Collections.emptyList(); di.resolutionLevels = new ArrayList<>(numberOfAllResLevels); @@ -179,7 +188,7 @@ public void run() { myLogger.info("CREATED JSON:\n"+json); final Future subcall = cs.run(CreateNewDatasetFromJSON.class, true, - "url",url, "json",json, "showRunCmd",showRunCmd); + "url",url, "json",json, "showRunCmd",showRunCmd,"datasetType"); newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID"); newDatasetLabel = di.getLabel(); } catch (ExecutionException e) {