Skip to content

Zarr New #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Zarr New #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,4 @@ cookies.txt

# Vim #
.*.sw*
*.tif
1 change: 1 addition & 0 deletions src/main/java/cz/it4i/fiji/datastore/rest_client/WriteSequenceToN5.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ public static void writeN5File(final AbstractSequenceDescription<?, ?, ?> seq,
int numCompletedTasks = 0;

final ExecutorService executorService = Executors.newFixedThreadPool( numCellCreatorThreads );

try (N5Writer n5 = writerSupplier.get()) {
// write image data for all views
final int numTimepoints = timepointIds.size();
Expand Down
160 changes: 160 additions & 0 deletions src/main/java/cz/it4i/fiji/legacy/CloneEmptyDatasetFromN5toZarr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
package cz.it4i.fiji.legacy;

import com.fasterxml.jackson.databind.ObjectMapper;
import cz.it4i.fiji.legacy.util.GuiResolutionLevelParams;
import cz.it4i.fiji.rest.util.DatasetInfo;
import net.imagej.Dataset;
import org.scijava.ItemIO;
import org.scijava.ItemVisibility;
import org.scijava.command.Command;
import org.scijava.command.CommandInfo;
import org.scijava.command.CommandModule;
import org.scijava.command.CommandService;
import org.scijava.log.LogLevel;
import org.scijava.log.LogService;
import org.scijava.log.Logger;
import org.scijava.module.Module;
import org.scijava.module.ModuleException;
import org.scijava.module.process.PostprocessorPlugin;
import org.scijava.module.process.PreprocessorPlugin;
import org.scijava.plugin.Parameter;
import org.scijava.plugin.Plugin;
import org.scijava.plugin.PluginService;
import org.scijava.prefs.PrefService;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

@Plugin(type = Command.class, headless = true, menuPath = "Plugins>HPC DataStore>ZARR Convert>Create new dataset from existing N5")

public class CloneEmptyDatasetFromN5toZarr implements Command {
@Parameter(label = "URL of a reference DatasetsRegisterService:", persistKey = "datasetserverurl")
public String src_url = "someHostname:9080";

@Parameter(label = "UUID of a reference N5 dataset on that service:", persistKey = "datasetdatasetid")
public String src_uuid = "someDatasetUUID";

@Parameter(label = "URL of a target DatasetsRegisterService:", persistKey = "datasetserverurl")
public String tgt_url = "someHostname:9080";

@Parameter(label = "Modify params before creating the new dataset:")
public boolean shallUpdateParams = false;

@Parameter
public PrefService prefs;

@Parameter
public CommandService cs;
@Parameter
public PluginService ps;

@Parameter
public LogService mainLogger;
protected Logger myLogger;

@Parameter(label = "Report corresponding macro command:", required = false)
public boolean showRunCmd = false;

@Parameter(type = ItemIO.OUTPUT, label="UUID of the created Zarr dataset:")
public String newDatasetUUID;
@Parameter(type = ItemIO.OUTPUT, label="Label of the created Zarr dataset:")
public String newDatasetLabel;
@Override
public void run() {
//logging facility
myLogger = mainLogger.subLogger("HPC CloneDataset", LogLevel.INFO);

try {
//retrieve reference data
myLogger.info("Reading "+src_uuid+" from "+src_url);
DatasetInfo di = DatasetInfo.createFrom(src_url, src_uuid);
myLogger.info(di.toString());

if (!shallUpdateParams) {
final String json = new ObjectMapper().writeValueAsString(di);
myLogger.info("CREATED JSON:\n"+json);

final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
"url",tgt_url, "json",json, "showRunCmd",showRunCmd);
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel");
} else {
//we gonna preset the prefs store so that the "params of resolution level" dialog(s)
//will get preloaded with the reference values
final GuiResolutionLevelParams prefsHandler = new GuiResolutionLevelParams();
prefsHandler.prefs = prefs;
for (int level = 0; level < di.resolutionLevels.size(); ++level) {
final DatasetInfo.ResolutionLevel l = di.resolutionLevels.get(level);
prefsHandler.resLevelNumber=level+1;
prefsHandler.down_x = l.resolutions.get(0);
prefsHandler.down_y = l.resolutions.get(1);
prefsHandler.down_z = l.resolutions.get(2);
prefsHandler.block_x = l.blockDimensions.get(0);
prefsHandler.block_y = l.blockDimensions.get(1);
prefsHandler.block_z = l.blockDimensions.get(2);
prefsHandler.storeIntoPrefs();
}

final CommandModule cm = new CommandModule( new CommandInfo(CreateNewDataset.class) );
cm.setInput("datasetType","Zarr");
cm.setInput("url", tgt_url);
cm.setInput("label", di.label);
cm.setInput("voxelType", di.voxelType);
cm.setInput("fullResSizeX", di.dimensions.get(0));
cm.setInput("fullResSizeY", di.dimensions.get(1));
cm.setInput("fullResSizeZ", di.dimensions.get(2));
cm.setInput("timepoints", di.timepoints);
cm.setInput("channels", di.channels);
cm.setInput("angles", di.angles);
cm.setInput("res_x", di.voxelResolution.get(0));
cm.setInput("res_y", di.voxelResolution.get(1));
cm.setInput("res_z", di.voxelResolution.get(2));
cm.setInput("res_unit", di.voxelUnit);

if (di.timepointResolution != null) {
cm.setInput("time_res", di.timepointResolution.value);
cm.setInput("time_unit", di.timepointResolution.unit);
}
if (di.channelResolution != null) {
cm.setInput("channel_res", di.channelResolution.value);
cm.setInput("channel_unit", di.channelResolution.unit);
}
if (di.angleResolution != null) {
cm.setInput("angle_res", di.angleResolution.value);
cm.setInput("angle_unit", di.angleResolution.unit);
}
cm.setInput("numberOfAllResLevels", di.resolutionLevels.size());
cm.setInput("compression", di.compression);
cm.setInput("showRunCmd", showRunCmd);

// 1) this is a standard list of pre- and post-processors,
// this is how ModuleService.java creates them for every call of run()
final List<PreprocessorPlugin> pre = ps.createInstancesOfType(PreprocessorPlugin.class);
final List<PostprocessorPlugin> post = ps.createInstancesOfType(PostprocessorPlugin.class);

// 2) but we find and remove one particular service...
for (int i = 0; i < pre.size(); ++i)
if (pre.get(i).getClass().getSimpleName().startsWith("LoadInputsPreprocessor")) {
pre.remove(i);
break;
}

// 3) finally, the command (module actually) is started
final Future<Module> subcall = cs.moduleService().run(cm,pre,post);

newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
newDatasetLabel = (String)subcall.get().getOutput("newDatasetLabel");
}
} catch (IOException | ModuleException e) {
myLogger.error("Some error accessing the reference service and dataset: "+e.getMessage());
} catch (ExecutionException e) {
myLogger.error(e.getMessage());

} catch (InterruptedException e) {
//dialog interrupted, do nothing special...
myLogger.error("eee!? "+e.getMessage());
}
}
}
24 changes: 20 additions & 4 deletions src/main/java/cz/it4i/fiji/legacy/CreateNewDataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ public class CreateNewDataset implements Command {
@Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:")
public String newDatasetLabel;

@Parameter(label = "DatasetType:", choices = { "N5", "Zarr" })
public String datasetType;
protected String getDatasetType()
{
return this.datasetType;
}

@Override
public void run() {
//logging facility
Expand All @@ -115,9 +122,18 @@ public void run() {
di.timepointResolution = new DatasetInfo.ResolutionWithOwnUnit(time_res, time_unit);
di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit);
di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit);

di.compression = this.compression.equals("none") ? "raw" : this.compression;

//This solution was chosen so that there was no need to change the DatasetDTO object,
// so the information about the dataset type is placed after the compression with the delimiter '/',
// according to which it is then divided in the registerServiceEndpoint and only the actual value of the compression is stored in the compression.
// datasetType is then normally stored in the Dataset class and is used to select a suitable handler.
// The most likely solution would be to add the datasetType attribute to the DatasetDTO class, but unfortunately this was not entirely within my power.
if(this.datasetType.equals("Zarr")) {
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr";
}
else
{
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5";
}
di.versions = Collections.emptyList();
di.resolutionLevels = new ArrayList<>(numberOfAllResLevels);
di.label = label;
Expand Down Expand Up @@ -162,7 +178,7 @@ public void run() {
myLogger.info("CREATED JSON:\n"+json);

final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
"url",url, "json",json, "showRunCmd",showRunCmd);
"url",url, "json",json, "showRunCmd",showRunCmd,"datasetType",getDatasetType());
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
newDatasetLabel = di.getLabel();
} catch (ExecutionException e) {
Expand Down
13 changes: 11 additions & 2 deletions src/main/java/cz/it4i/fiji/legacy/CreateNewDatasetFromImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ void readParams() {
@Parameter(type = ItemIO.OUTPUT, label="Label of the created dataset:")
public String newDatasetLabel;

@Parameter(label = "DatasetType:", choices = { "N5", "Zarr" })
public String datasetType;

@Override
public void run() {
//logging facility
Expand All @@ -134,7 +137,13 @@ public void run() {
di.channelResolution = new DatasetInfo.ResolutionWithOwnUnit(channel_res, channel_unit);
di.angleResolution = new DatasetInfo.ResolutionWithOwnUnit(angle_res, angle_unit);

di.compression = this.compression.equals("none") ? "raw" : this.compression;
if(this.datasetType.equals("Zarr")) {
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/Zarr";
}
else
{
di.compression = this.compression.equals("none") ? "raw" : this.compression + "/N5";
}

di.versions = Collections.emptyList();
di.resolutionLevels = new ArrayList<>(numberOfAllResLevels);
Expand Down Expand Up @@ -179,7 +188,7 @@ public void run() {
myLogger.info("CREATED JSON:\n"+json);

final Future<CommandModule> subcall = cs.run(CreateNewDatasetFromJSON.class, true,
"url",url, "json",json, "showRunCmd",showRunCmd);
"url",url, "json",json, "showRunCmd",showRunCmd,"datasetType");
newDatasetUUID = (String)subcall.get().getOutput("newDatasetUUID");
newDatasetLabel = di.getLabel();
} catch (ExecutionException e) {
Expand Down