diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/BrowseProductReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/BrowseProductReader.java index 53654a063..874b49c7f 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/BrowseProductReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/BrowseProductReader.java @@ -124,11 +124,11 @@ public Product createProduct() throws ProductIOException { // } // @Override - protected Map addBrsBands(Product product, List variables) { + protected Map addBrsBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band; - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); String description = "Level-1A Browse data"; String units = "Relative Reflectance units"; @@ -179,7 +179,7 @@ protected Map addBrsBands(Product product, List var } } } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); } } else if (variable.getRank() == 3) { final int[] dimensions = variable.getShape(); @@ -206,7 +206,7 @@ protected Map addBrsBands(Product product, List var e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AHawkeyeFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AHawkeyeFileReader.java index 1309d5fb1..de720225a 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AHawkeyeFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AHawkeyeFileReader.java @@ -135,14 +135,14 @@ public void addGeocoding(final Product product) throws ProductIOException { } - public Map addHawkeyeBands(Product product, + public Map addHawkeyeBands(Product product, List variables) { - final Map bandToVariableMap = new HashMap(); + final Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { Band band = addNewBand(product, variable); if (band != null) { - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); } if (spectralBandIndex < 8) { final float wavelength = Float.valueOf(HAWKEYE_WVL[spectralBandIndex]); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AOctsFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AOctsFileReader.java index 9155d53fd..f783a4113 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AOctsFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1AOctsFileReader.java @@ -128,11 +128,11 @@ public void addGeocoding(final Product product) throws ProductIOException { } } - private Map addOctsBands(Product product, List variables) { + private Map addOctsBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { int variableRank = variable.getRank(); @@ -168,7 +168,7 @@ private Map addOctsBands(Product product, List variabl } catch (InvalidRangeException e) { e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1ASeawifsFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1ASeawifsFileReader.java index 30c878bb2..3b68beace 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1ASeawifsFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1ASeawifsFileReader.java @@ -105,11 +105,11 @@ public Product createProduct() throws ProductIOException { } - private Map addSeawifsBands(Product product, List variables) { + private Map addSeawifsBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { int variableRank = variable.getRank(); @@ -145,7 +145,7 @@ private Map addSeawifsBands(Product product, List vari } catch (InvalidRangeException e) { e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BHicoFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BHicoFileReader.java index b59587bff..4a66a177d 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BHicoFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BHicoFileReader.java @@ -46,6 +46,7 @@ public class L1BHicoFileReader extends SeadasFileReader { L1BHicoFileReader(SeadasProductReader productReader) { super(productReader); + } Array wavlengths = ncFile.findVariable("products/Lt").findAttribute("wavelengths").getValues(); @@ -222,12 +223,12 @@ public void addMetadata(Product product, String groupname, String meta_element) } } - private Map addHicoBands(Product product, List variables) { + private Map addHicoBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band; - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { if ((variable.getShortName().contains("latitude")) || (variable.getShortName().contains("longitude")) @@ -265,7 +266,7 @@ private Map addHicoBands(Product product, List variabl band.setNoDataValueUsed(true); } } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); band.setUnit(units); band.setDescription(name); } @@ -320,7 +321,7 @@ private Map addHicoBands(Product product, List variabl band.setNoDataValueUsed(true); } } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BModisFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BModisFileReader.java index a471a759e..8d9b541d4 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BModisFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BModisFileReader.java @@ -211,12 +211,12 @@ private float getScaleFactor(Variable variable) { return scale_factor; } - private Map addModisBands(Product product, List variables) { + private Map addModisBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { final int variableRank = variable.getRank(); @@ -282,7 +282,7 @@ private Map addModisBands(Product product, List variab } catch (InvalidRangeException e) { e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); if (slope != null) { diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReader.java index 6ecee9cd1..4b1d53134 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReader.java @@ -16,27 +16,39 @@ package gov.nasa.gsfc.seadas.dataio; +import eu.esa.snap.core.dataio.cache.CacheDataProvider; +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; import org.esa.snap.core.dataio.ProductIOException; import org.esa.snap.core.dataio.geocoding.ComponentGeoCoding; import org.esa.snap.core.dataio.geocoding.GeoCodingFactory; import org.esa.snap.core.datamodel.*; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; +import org.esa.snap.runtime.Config; +import org.jspecify.annotations.NonNull; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; import ucar.nc2.Group; import ucar.nc2.Variable; -import ucar.nc2.Dimension; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.prefs.Preferences; + public class L1BPaceOciFileReader extends SeadasFileReader { + L1BPaceOciFileReader(SeadasProductReader productReader) { super(productReader); + + final Preferences preferences = Config.instance("seadas").preferences(); + wantsCaching = preferences.getBoolean("seadas.reader.enable.cache", true); + applyScaling = preferences.getBoolean("seadas.reader.apply.scaling", true); } enum WvlType { @@ -44,9 +56,9 @@ enum WvlType { BLUE("blue_wavelengths"), SWIR("swir_wavelengths"); - private String name; + private final String name; - private WvlType(String nm) { + WvlType(String nm) { name = nm; } @@ -62,9 +74,9 @@ public String toString() { @Override public Product createProduct() throws ProductIOException { - + int[] shape; - int sceneWidth, sceneHeight ; + int sceneWidth, sceneHeight; String productName; try { @@ -81,18 +93,7 @@ public Product createProduct() throws ProductIOException { productName = productReader.getInputFile().getName(); } - Variable blueWvl = ncFile.findVariable("sensor_band_parameters/blue_wavelength"); - Variable redWvl = ncFile.findVariable("sensor_band_parameters/red_wavelength"); - Variable swirWvl = ncFile.findVariable("sensor_band_parameters/SWIR_wavelength"); - try { - blue_wavlengths = blueWvl.read(); - red_wavlengths = redWvl.read(); - swir_wavlengths = swirWvl.read(); - } catch (IOException e) { - throw new ProductIOException(e.getMessage(), e); - } -// mustFlipY = true; -// mustFlipX = false; + readWvlVariables(); if (SeadasReaderDefaults.FlIP_YES.equals(getBandFlipXL1BPace())) { mustFlipX = true; @@ -102,39 +103,26 @@ public Product createProduct() throws ProductIOException { mustFlipX = false; // default flipX } + // default flipY if (SeadasReaderDefaults.FlIP_YES.equals(getBandFlipYL1BPace())) { mustFlipY = true; - } else if (SeadasReaderDefaults.FlIP_NO.equals(getBandFlipYL1BPace())) { - mustFlipY = false; - } else { - mustFlipY = true; // default flipY - } - - + } else mustFlipY = !SeadasReaderDefaults.FlIP_NO.equals(getBandFlipYL1BPace()); SeadasProductReader.ProductType productType = productReader.getProductType(); Product product = new Product(productName, productType.toString(), sceneWidth, sceneHeight); product.setDescription(productName); - // @todo 3 this is obviously never used again - remove? tb 2026-01-13 - //Attribute startTime = findAttribute("time_coverage_start"); - //ProductData.UTC utcStart = getUTCAttribute("time_coverage_start"); - //ProductData.UTC utcEnd = getUTCAttribute("time_coverage_end"); - //if (startTime == null) { - // utcStart = getUTCAttribute("Start_Time"); - // utcEnd = getUTCAttribute("End_Time"); - //} - // only needed as a stop-gap to handle an intermediate version of l2gen metadata - //if (utcEnd == null) { - // utcEnd = getUTCAttribute("time_coverage_stop"); - //} - product.setFileLocation(productReader.getInputFile()); product.setProductReader(productReader); addGlobalMetadata(product); - variableMap = addOciBands(product, ncFile.getVariables()); + List variables = ncFile.getVariables(); + if (wantsCaching) { + variableMap = addCachedOciBands(product, variables); + } else { + variableMap = addOciBands(product, variables); + } addGeocoding(product); addMetadata(product, "products", "Band_Metadata"); @@ -146,6 +134,19 @@ public Product createProduct() throws ProductIOException { return product; } + private void readWvlVariables() throws ProductIOException { + Variable blueWvl = ncFile.findVariable("sensor_band_parameters/blue_wavelength"); + Variable redWvl = ncFile.findVariable("sensor_band_parameters/red_wavelength"); + Variable swirWvl = ncFile.findVariable("sensor_band_parameters/SWIR_wavelength"); + try { + blue_wavlengths = blueWvl.read(); + red_wavlengths = redWvl.read(); + swir_wavlengths = swirWvl.read(); + } catch (IOException e) { + throw new ProductIOException(e.getMessage(), e); + } + } + public void addMetadata(Product product, String groupname, String meta_element) throws ProductIOException { Group group = ncFile.findGroup(groupname); @@ -172,13 +173,67 @@ public void addMetadata(Product product, String groupname, String meta_element) } } - private Map addOciBands(Product product, List variables) { + private Map addCachedOciBands(Product product, List variables) { + final Map bandToVariableMap = new HashMap(); final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); - Band band; - Map bandToVariableMap = new HashMap(); + for (Variable variable : variables) { + final Group parentGroup = variable.getParentGroup(); + final String groupName = parentGroup.getShortName(); + if ("sensor_band_parameters".equals(groupName) || "scan_line_attributes".equals(groupName)) { + continue; + } + final String variableName = variable.getShortName(); + if (("latitude".equals(variableName)) || ("longitude".equals(variableName))) { + continue; + } + + final int variableRank = variable.getRank(); + if (variableRank == 2) { + add2DBand(product, variable, sceneRasterHeight, sceneRasterWidth, bandToVariableMap); + } else if (variableRank == 3) { + add3DVariables_forCaching(product, variable, sceneRasterHeight, sceneRasterWidth, bandToVariableMap); + } + } + + return bandToVariableMap; + } + + private void add3DVariables_forCaching(Product product, Variable variable, int sceneRasterHeight, int sceneRasterWidth, Map bandToVariableMap) { + final int[] dimensions = variable.getShape(); + final int bands = dimensions[0]; + final int height = dimensions[1]; + final int width = dimensions[2]; + + if (height != sceneRasterHeight || width != sceneRasterWidth) { + return; + } + + final String ncVariableName = variable.getShortName(); + final WvlType wvlType = getWvlType(variable.getShortName()); int spectralBandIndex = 0; + for (int i = 0; i < bands; i++) { + final float wavelength = getOciWvl(i, wvlType); + final String variableName = getVariableName(ncVariableName, wavelength); + final int dataType = getProductDataType(variable); + final Band band = createBand(variableName, dataType, width, height); + + product.addBand(band); + + band.setSpectralWavelength(wavelength); + band.setSpectralBandIndex(spectralBandIndex++); + addAttributes(variable, band, applyScaling); + band.setDescription(ncVariableName); + } + bandToVariableMap.put(ncVariableName, variable); + } + + private Map addOciBands(Product product, List variables) { + final int sceneRasterWidth = product.getSceneRasterWidth(); + final int sceneRasterHeight = product.getSceneRasterHeight(); + + Map bandToVariableMap = new HashMap(); for (Variable variable : variables) { if (variable.getParentGroup().equals("sensor_band_parameters") || variable.getParentGroup().equals("scan_line_attributes")) continue; @@ -187,106 +242,123 @@ private Map addOciBands(Product product, List variable int variableRank = variable.getRank(); if (variableRank == 2) { - final int[] dimensions = variable.getShape(); - final int height = dimensions[0]; - final int width = dimensions[1]; - - if (height == sceneRasterHeight && width == sceneRasterWidth) { - // final List list = variable.getAttributes(); - - String units = variable.getUnitsString(); - String name = variable.getShortName(); - final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); - product.addBand(band); - - final List list = variable.getAttributes(); - for (Attribute hdfAttribute : list) { - final String attribName = hdfAttribute.getShortName(); - if ("units".equals(attribName)) { - band.setUnit(hdfAttribute.getStringValue()); - } else if ("long_name".equals(attribName)) { - band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { - band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { - band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { - band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { - band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { - band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); - band.setNoDataValueUsed(true); - } - } - bandToVariableMap.put(band, variable); - band.setUnit(units); - band.setDescription(variable.getDescription()); - } + add2DBand(product, variable, sceneRasterHeight, sceneRasterWidth, bandToVariableMap); } else if (variableRank == 3) { - final int[] dimensions = variable.getShape(); - final int bands = dimensions[0]; - final int height = dimensions[1]; - final int width = dimensions[2]; - - if (height == sceneRasterHeight && width == sceneRasterWidth) { - // final List list = variable.getAttributes(); - - String units = variable.getUnitsString(); - String description = variable.getShortName(); - - for (int i = 0; i < bands; i++) { - final float wavelength = getOciWvl(i, getWvlType(variable.getShortName())); - StringBuilder longname = new StringBuilder(description); - longname.append("_"); - longname.append(wavelength); - String name = longname.toString(); - final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); - product.addBand(band); - - band.setSpectralWavelength(wavelength); - band.setSpectralBandIndex(spectralBandIndex++); - - Variable sliced = null; - try { - sliced = variable.slice(0, i); - } catch (InvalidRangeException e) { - e.printStackTrace(); //Todo change body of catch statement. - } - - final List list = variable.getAttributes(); - for (Attribute hdfAttribute : list) { - final String attribName = hdfAttribute.getShortName(); - if ("units".equals(attribName)) { - band.setUnit(hdfAttribute.getStringValue()); - } else if ("long_name".equals(attribName)) { - band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { - band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { - band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { - band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { - band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { - band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); - band.setNoDataValueUsed(true); - } - } - bandToVariableMap.put(band, sliced); - band.setUnit(units); - band.setDescription(description); - - } - } + add3DBands(product, variable, sceneRasterHeight, sceneRasterWidth, bandToVariableMap); } + } + return bandToVariableMap; + } + private void add3DBands(Product product, Variable variable, int sceneRasterHeight, int sceneRasterWidth, Map bandToVariableMap) { + int spectralBandIndex = 0; + final int[] dimensions = variable.getShape(); + final int bands = dimensions[0]; + final int height = dimensions[1]; + final int width = dimensions[2]; + if (height != sceneRasterHeight || width != sceneRasterWidth) { + return; } - return bandToVariableMap; + + final String units = variable.getUnitsString(); + final String description = variable.getShortName(); + + for (int i = 0; i < bands; i++) { + final float wavelength = getOciWvl(i, getWvlType(variable.getShortName())); + final String variableName = getVariableName(description, wavelength); + final int dataType = getProductDataType(variable); + final Band band = createBand(variableName, dataType, width, height); + product.addBand(band); + + band.setSpectralWavelength(wavelength); + band.setSpectralBandIndex(spectralBandIndex++); + + Variable sliced = null; + try { + sliced = variable.slice(0, i); + } catch (InvalidRangeException e) { + e.printStackTrace(); //Todo change body of catch statement. + } + + addAttributes(variable, band, applyScaling); + bandToVariableMap.put(band.getName(), sliced); + // the following two are duplicated, already covered by addAttributes() tb 2026-04-28 + band.setUnit(units); + band.setDescription(description); + } + } + + // package access for testing only tb 2026-04-28 + static void addAttributes(Variable variable, Band band, boolean applyScaling) { + Attribute attribute = variable.findAttribute("units"); + if (attribute != null) { + band.setUnit(attribute.getStringValue()); + } + attribute = variable.findAttribute("long_name"); + if (attribute != null) { + band.setDescription(attribute.getStringValue()); + } + if (applyScaling) { + attribute = variable.findAttribute("slope"); + if (attribute != null) { + band.setScalingFactor(attribute.getNumericValue().doubleValue()); + } + attribute = variable.findAttribute("scale_factor"); + if (attribute != null) { + band.setScalingFactor(attribute.getNumericValue().doubleValue()); + } + attribute = variable.findAttribute("intercept"); + if (attribute != null) { + band.setScalingOffset(attribute.getNumericValue().doubleValue()); + } + attribute = variable.findAttribute("add_offset"); + if (attribute != null) { + band.setScalingOffset(attribute.getNumericValue().doubleValue()); + } + attribute = variable.findAttribute("bad_value_scaled"); + if (attribute != null) { + band.setNoDataValue(attribute.getNumericValue().doubleValue()); + band.setNoDataValueUsed(true); + } + } + } + + // package access for testing only tb 2026-04-28 + static @NonNull String getVariableName(String description, float wavelength) { + return description + "_" + wavelength; + } + + @Override + String getNcVariableName(String variableName) { + return stripWavelengthFromName(variableName); + } + + static @NonNull String stripWavelengthFromName(String variableName) { + final int underscoreIdx = variableName.lastIndexOf("_"); + return variableName.substring(0, underscoreIdx); + } + + private void add2DBand(Product product, Variable variable, int sceneRasterHeight, int sceneRasterWidth, Map bandToVariableMap) { + final int[] dimensions = variable.getShape(); + final int height = dimensions[0]; + final int width = dimensions[1]; + + if (height != sceneRasterHeight || width != sceneRasterWidth) { + return; + } + + final String units = variable.getUnitsString(); + final String name = variable.getShortName(); + final int dataType = getProductDataType(variable); + final Band band = createBand(name, dataType, width, height); + product.addBand(band); + + addAttributes(variable, band, applyScaling); + + bandToVariableMap.put(band.getName(), variable); + band.setUnit(units); + band.setDescription(variable.getDescription()); } private WvlType getWvlType(String productName) { @@ -368,13 +440,23 @@ private void addGeocoding(final Product product) throws ProductIOException { } } } - private int getDimension(String dimensionName) { - final List dimensions = ncFile.getDimensions(); - for (Dimension dimension : dimensions) { - if (dimension.getShortName().equals(dimensionName)) { - return dimension.getLength(); - } + + @Override + public VariableDescriptor getVariableDescriptor(String variableName) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + return SeadasCacheUtils.getDefaultVariableDescriptor(netcdfVariable, variableName); + } + + @Override + public DataBuffer readCacheBlock(String variableName, int[] offsets, int[] shapes, ProductData targetData) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + final int rasterDataType = DataTypeUtils.getRasterDataType(netcdfVariable); + + Array rawBuffer; + synchronized (ncFile) { + rawBuffer = SeadasCacheUtils.readArray(netcdfVariable, offsets, shapes); } - return -1; + + return SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, targetData, rasterDataType); } } \ No newline at end of file diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1CPaceFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1CPaceFileReader.java index 9a1ff360b..8d48d51d3 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1CPaceFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L1CPaceFileReader.java @@ -16,11 +16,17 @@ package gov.nasa.gsfc.seadas.dataio; +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; import org.esa.snap.core.dataio.ProductIOException; import org.esa.snap.core.dataio.geocoding.ComponentGeoCoding; import org.esa.snap.core.dataio.geocoding.GeoCodingFactory; import org.esa.snap.core.datamodel.*; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; +import org.esa.snap.runtime.Config; +import org.jspecify.annotations.NonNull; import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; import ucar.nc2.Group; @@ -28,6 +34,7 @@ import java.io.IOException; import java.util.*; +import java.util.prefs.Preferences; import static java.lang.String.format; @@ -35,6 +42,10 @@ public class L1CPaceFileReader extends SeadasFileReader { L1CPaceFileReader(SeadasProductReader productReader) { super(productReader); + + final Preferences preferences = Config.instance("seadas").preferences(); + wantsCaching = preferences.getBoolean("seadas.reader.enable.cache", true); + applyScaling = preferences.getBoolean("seadas.reader.apply.scaling", true); } @Override @@ -216,12 +227,12 @@ public void addMetadata(Product product, String groupname, String meta_element) } } - private Map addHarpBands(Product product, List variables, Array view_angles, Array wavelengths, boolean correctDim) { + private Map addHarpBands(Product product, List variables, Array view_angles, Array wavelengths, boolean correctDim) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band; - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { if (variable.getParentGroup().equals("sensor_views_bands")) @@ -242,7 +253,7 @@ private Map addHarpBands(Product product, List variabl String units = variable.getUnitsString(); String name = variable.getShortName(); final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); final List list = variable.getAttributes(); @@ -253,15 +264,15 @@ private Map addHarpBands(Product product, List variabl band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -308,7 +319,7 @@ private Map addHarpBands(Product product, List variabl } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); band.setUnit(units); band.setDescription(variable.getDescription()); } @@ -344,7 +355,7 @@ private Map addHarpBands(Product product, List variabl String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); band.setSpectralWavelength(wavelengths.getFloat(i)); @@ -377,20 +388,20 @@ private Map addHarpBands(Product product, List variabl band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { - band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); - band.setNoDataValueUsed(true); + band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); + band.setNoDataValueUsed(true); } else if (attribName.startsWith("valid_")) { if ("valid_min".equals(attribName)) { if (hdfAttribute.getDataType().isUnsigned()) { @@ -432,7 +443,7 @@ private Map addHarpBands(Product product, List variabl } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); } @@ -472,7 +483,7 @@ private Map addHarpBands(Product product, List variabl String name = longname.toString(); String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); band.setSpectralWavelength(wavelengths.getFloat(j)); @@ -510,15 +521,15 @@ private Map addHarpBands(Product product, List variabl band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -565,7 +576,7 @@ private Map addHarpBands(Product product, List variabl } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); // } @@ -577,12 +588,12 @@ private Map addHarpBands(Product product, List variabl return bandToVariableMap; } - private Map addOciBands(Product product, List variables, Array view_angles, Array wavelengths) { + private Map addOciBands(Product product, List variables, Array view_angles, Array wavelengths) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band; - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); int spectralBandIndex = 0; for (Variable variable : variables) { if (variable.getParentGroup().equals("sensor_views_bands")) @@ -603,7 +614,7 @@ private Map addOciBands(Product product, List variable String units = variable.getUnitsString(); String name = variable.getShortName(); final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); final List list = variable.getAttributes(); @@ -614,15 +625,15 @@ private Map addOciBands(Product product, List variable band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -669,7 +680,7 @@ private Map addOciBands(Product product, List variable } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); band.setUnit(units); band.setDescription(variable.getDescription()); } @@ -696,7 +707,7 @@ private Map addOciBands(Product product, List variable String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); band.setAngularValue(view_angles.getFloat(i)); @@ -717,15 +728,15 @@ private Map addOciBands(Product product, List variable band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -772,7 +783,7 @@ private Map addOciBands(Product product, List variable } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); } @@ -803,7 +814,7 @@ private Map addOciBands(Product product, List variable String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); band.setSpectralWavelength(wavelengths.getFloat(j)); @@ -829,15 +840,15 @@ private Map addOciBands(Product product, List variable band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -884,7 +895,7 @@ private Map addOciBands(Product product, List variable } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); } @@ -898,7 +909,7 @@ private Map addOciBands(Product product, List variable return bandToVariableMap; } - private Map addSPEXoneBands(Product product, List variables, Array view_angles, Array wavelengths, Array wavelengths_pol) { + private Map addSPEXoneBands(Product product, List variables, Array view_angles, Array wavelengths, Array wavelengths_pol) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band; @@ -907,7 +918,7 @@ private Map addSPEXoneBands(Product product, List vari // Array wavelengths_pol = null; // Array view_angles = null; - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); for (Variable variable : variables) { if (variable.getParentGroup().equals("sensor_views_bands")) continue; @@ -927,7 +938,7 @@ private Map addSPEXoneBands(Product product, List vari String units = variable.getUnitsString(); String name = variable.getShortName(); final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); final List list = variable.getAttributes(); @@ -938,15 +949,15 @@ private Map addSPEXoneBands(Product product, List vari band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -993,7 +1004,7 @@ private Map addSPEXoneBands(Product product, List vari } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); band.setUnit(units); band.setDescription(variable.getDescription()); } @@ -1031,7 +1042,7 @@ private Map addSPEXoneBands(Product product, List vari String name = longname.toString(); String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); if ((i > views / 2) && (view_angles.getInt(i) == view_angles.getInt(views - 1 - i))) { @@ -1056,15 +1067,15 @@ private Map addSPEXoneBands(Product product, List vari band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -1111,7 +1122,7 @@ private Map addSPEXoneBands(Product product, List vari } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); } @@ -1169,7 +1180,7 @@ private Map addSPEXoneBands(Product product, List vari String name = longname.toString(); String safeName = (name != null && name.contains("-")) ? "'" + name + "'" : name; final int dataType = getProductDataType(variable); - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); if (bands == 400) { @@ -1203,15 +1214,15 @@ private Map addSPEXoneBands(Product product, List vari band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); - } else if ("bad_value_scaled".equals(attribName)) { + } else if (applyScaling && "bad_value_scaled".equals(attribName)) { band.setNoDataValue(hdfAttribute.getNumericValue(0).doubleValue()); band.setNoDataValueUsed(true); } else if ("_FillValue".equals(attribName)) { @@ -1258,7 +1269,7 @@ private Map addSPEXoneBands(Product product, List vari } band.setValidPixelExpression(validExp);//.format(name, validMinMax[0], name, validMinMax[1])); } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); band.setUnit(units); band.setDescription(description); } @@ -1339,4 +1350,25 @@ public void addGeocoding(final Product product) throws ProductIOException { } } + + + @Override + public VariableDescriptor getVariableDescriptor(String variableName) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + return SeadasCacheUtils.getDefaultVariableDescriptor(netcdfVariable, variableName); + } + + + @Override + public DataBuffer readCacheBlock(String variableName, int[] offsets, int[] shapes, ProductData targetData) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + final int rasterDataType = DataTypeUtils.getRasterDataType(netcdfVariable); + + Array rawBuffer; + synchronized (ncFile) { + rawBuffer = SeadasCacheUtils.readArray(netcdfVariable, offsets, shapes); + } + + return SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, targetData, rasterDataType); + } } \ No newline at end of file diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L2FileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L2FileReader.java index 7786d10ff..a00ecd3bb 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L2FileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L2FileReader.java @@ -16,6 +16,8 @@ package gov.nasa.gsfc.seadas.dataio; +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; import org.esa.snap.core.dataio.ProductIOException; import org.esa.snap.core.dataio.geocoding.ComponentGeoCoding; import org.esa.snap.core.dataio.geocoding.GeoCodingFactory; @@ -23,13 +25,14 @@ import org.esa.snap.core.datamodel.Product; import org.esa.snap.core.datamodel.ProductData; import org.esa.snap.core.datamodel.TiePointGrid; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; +import org.esa.snap.runtime.Config; import ucar.ma2.Array; import ucar.nc2.Attribute; -import ucar.nc2.Dimension; import ucar.nc2.Variable; import java.io.IOException; -import java.util.List; +import java.util.prefs.Preferences; /** * Created by IntelliJ IDEA. @@ -44,6 +47,10 @@ public class L2FileReader extends SeadasFileReader { L2FileReader(SeadasProductReader productReader) { super(productReader); + + final Preferences preferences = Config.instance("seadas").preferences(); + wantsCaching = preferences.getBoolean("seadas.reader.enable.cache", true); + applyScaling = preferences.getBoolean("seadas.reader.apply.scaling", true); } @Override @@ -414,4 +421,24 @@ private void addPixelGeocoding(final Product product) throws ProductIOException } } } + + @Override + public VariableDescriptor getVariableDescriptor(String variableName) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + return SeadasCacheUtils.getDefaultVariableDescriptor(netcdfVariable, variableName); + } + + + @Override + public DataBuffer readCacheBlock(String variableName, int[] offsets, int[] shapes, ProductData targetData) throws IOException { + final Variable netcdfVariable = variableMap.get(variableName); + final int rasterDataType = DataTypeUtils.getRasterDataType(netcdfVariable); + + Array rawBuffer; + synchronized (ncFile) { + rawBuffer = SeadasCacheUtils.readArray(netcdfVariable, offsets, shapes); + } + + return SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, targetData, rasterDataType); + } } \ No newline at end of file diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L3BinFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L3BinFileReader.java index 0db43286c..eda651d65 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L3BinFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/L3BinFileReader.java @@ -360,20 +360,20 @@ public RowInfo(int offset, int length) { } } - public Map addBands(Product product, Variable idxVariable, List l3ProdVars) { + public Map addBands(Product product, Variable idxVariable, List l3ProdVars) { final Structure binListStruc = (Structure) idxVariable; - final Map bandToVariableMap = new HashMap(); + final Map bandToVariableMap = new HashMap(); // bandToVariableMap.put(addBand(product, "bin_num", ProductData.TYPE_UINT32), binListStruc.select("bin_num").findVariable("bin_num")); - bandToVariableMap.put(addBand(product, "weights", ProductData.TYPE_FLOAT32), binListStruc.select("weights").findVariable("weights")); - bandToVariableMap.put(addBand(product, "nobs", ProductData.TYPE_UINT16), binListStruc.select("nobs").findVariable("nobs")); - bandToVariableMap.put(addBand(product, "nscenes", ProductData.TYPE_UINT16), binListStruc.select("nscenes").findVariable("nscenes")); + bandToVariableMap.put(addBand(product, "weights", ProductData.TYPE_FLOAT32).getName(), binListStruc.select("weights").findVariable("weights")); + bandToVariableMap.put(addBand(product, "nobs", ProductData.TYPE_UINT16).getName(), binListStruc.select("nobs").findVariable("nobs")); + bandToVariableMap.put(addBand(product, "nscenes", ProductData.TYPE_UINT16).getName(), binListStruc.select("nscenes").findVariable("nscenes")); // ncFile.getRootGroup().findGroup("Level-3 Binned Data").findVariable("BinList"); if (ncFile.getRootGroup().findGroup("Level-3_Binned_Data").findVariable("qual_l3") != null){ - bandToVariableMap.put(addBand(product, "qual_l3", ProductData.TYPE_UINT8), ncFile.getRootGroup().findGroup("Level-3_Binned_Data").findVariable("qual_l3")); + bandToVariableMap.put(addBand(product, "qual_l3", ProductData.TYPE_UINT8).getName(), ncFile.getRootGroup().findGroup("Level-3_Binned_Data").findVariable("qual_l3")); } String groupnames = ""; for (Variable l3Var : l3ProdVars) { @@ -393,7 +393,7 @@ public Map addBands(Product product, Variable idxVariable, List< List vnames = binStruc.getVariableNames(); for (String bandvar : vnames) { - bandToVariableMap.put(addBand(product, bandvar, dataType), binStruc.select(bandvar).findVariable(bandvar)); + bandToVariableMap.put(addBand(product, bandvar, dataType).getName(), binStruc.select(bandvar).findVariable(bandvar)); } // Add virtual band for product mean StringBuilder prodname = new StringBuilder(varName); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/MeasuresL3BinFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/MeasuresL3BinFileReader.java index 29087bf88..03753dd22 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/MeasuresL3BinFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/MeasuresL3BinFileReader.java @@ -247,9 +247,9 @@ public RowInfo(int offset, int length) { } } - public Map addBands(Product product, List l3ProdVars) { + public Map addBands(Product product, List l3ProdVars) { - final Map bandToVariableMap = new HashMap(); + final Map bandToVariableMap = new HashMap(); for (Variable l3Var : l3ProdVars) { String varName = l3Var.getShortName(); @@ -263,7 +263,7 @@ public Map addBands(Product product, List l3ProdVars) if (!varName.contains("Indexes") && (!varName.equalsIgnoreCase("n"))) { - bandToVariableMap.put(addBand(product, varName, dataType), l3Var); + bandToVariableMap.put(addBand(product, varName, dataType).getName(), l3Var); } } return bandToVariableMap; diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/NSIDCSeaIceFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/NSIDCSeaIceFileReader.java index d84bf7247..b8cfe130a 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/NSIDCSeaIceFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/NSIDCSeaIceFileReader.java @@ -90,9 +90,9 @@ public Product createProduct() throws ProductIOException { } - public Map addNSIDCBands(Product product, + public Map addNSIDCBands(Product product, List variables) { - Map bandToVariableMap = new HashMap(); + Map bandToVariableMap = new HashMap(); for (Variable variable : variables) { int variableRank = variable.getRank(); if (variableRank == 3) { @@ -131,7 +131,7 @@ protected void setDateBand(Product product) { } } - protected Map add3DNSIDCBands(Product product, Variable variable, Map bandToVariableMap) { + protected Map add3DNSIDCBands(Product product, Variable variable, Map bandToVariableMap) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); @@ -179,7 +179,7 @@ protected Map add3DNSIDCBands(Product product, Variable variable } catch (InvalidRangeException e) { e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); try { Attribute fillValue = variable.findAttribute("_FillValue"); diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SMIFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SMIFileReader.java index 209630d00..53c1b7ea2 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SMIFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SMIFileReader.java @@ -116,10 +116,10 @@ public Product createProduct() throws ProductIOException { return product; } - protected Map addSmiBands(Product product, List variables) { + protected Map addSmiBands(Product product, List variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); - Map bandToVariableMap = new HashMap<>(); + Map bandToVariableMap = new HashMap<>(); for (Variable variable : variables) { int variableRank = variable.getRank(); if (variableRank == 2) { @@ -154,7 +154,7 @@ protected Map addSmiBands(Product product, List variab } catch (Exception ignored) { } - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); // Set units, if defined try { band.setUnit(getStringAttribute("Units")); @@ -227,7 +227,7 @@ protected Map addSmiBands(Product product, List variab e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); product.addBand(band); try { diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtils.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtils.java new file mode 100644 index 000000000..6640c0807 --- /dev/null +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtils.java @@ -0,0 +1,112 @@ +package gov.nasa.gsfc.seadas.dataio; + +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; +import org.esa.snap.core.datamodel.ProductData; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; +import org.jspecify.annotations.NonNull; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.IOException; + + +public class SeadasCacheUtils { + + + public static VariableDescriptor getDefaultVariableDescriptor(Variable netcdfVariable, String variableName) throws IOException { + if (netcdfVariable == null) { + throw new IOException("Variable not known: " + variableName); + } + + final VariableDescriptor variableDescriptor = new VariableDescriptor(); + variableDescriptor.name = variableName; + variableDescriptor.dataType = DataTypeUtils.getRasterDataType(netcdfVariable.getDataType(), false); + + final int[] shape = netcdfVariable.getShape(); + final Array chunkSizesValues; + final Attribute chunkSizes = netcdfVariable.findAttribute("_ChunkSizes"); + if (chunkSizes != null) { + chunkSizesValues = chunkSizes.getValues(); + } else { + chunkSizesValues = Array.factory(DataType.INT, new int[]{shape.length}, shape); + } + + if (shape.length == 2) { + variableDescriptor.width = shape[1]; + variableDescriptor.height = shape[0]; + variableDescriptor.layers = -1; + + variableDescriptor.tileWidth = chunkSizesValues.getInt(1); + variableDescriptor.tileHeight = chunkSizesValues.getInt(0); + variableDescriptor.tileLayers = -1; + } else if (shape.length == 3) { + variableDescriptor.width = shape[2]; + variableDescriptor.height = shape[1]; + variableDescriptor.layers = shape[0]; + + variableDescriptor.tileWidth = chunkSizesValues.getInt(2); + variableDescriptor.tileHeight = chunkSizesValues.getInt(1); + variableDescriptor.tileLayers = chunkSizesValues.getInt(0); + } else { + throw new IOException("Unsupported variable rank for caching: " + shape.length); + } + + return variableDescriptor; + } + + public static Array readArray(Variable netcdfVariable, int[] offsets, int[] shapes) throws IOException { + try { + return netcdfVariable.read(offsets, shapes); + } catch (InvalidRangeException e) { + throw new IOException(e); + } + } + + public static DataBuffer constructDataBuffer(Array rawBuffer, int[] offsets, int[] shapes, ProductData targetData, int rasterDataType) throws IOException { + if (targetData == null) { + targetData = createTargetDataBuffer(shapes, rasterDataType); + } + + switch (rasterDataType) { + case ProductData.TYPE_INT8: + case ProductData.TYPE_UINT8: + targetData.setElems(rawBuffer.get1DJavaArray(DataType.BYTE)); + break; + case ProductData.TYPE_INT16: + case ProductData.TYPE_UINT16: + targetData.setElems(rawBuffer.get1DJavaArray(DataType.SHORT)); + break; + case ProductData.TYPE_INT32: + case ProductData.TYPE_UINT32: + targetData.setElems(rawBuffer.get1DJavaArray(DataType.INT)); + break; + case ProductData.TYPE_FLOAT32: + targetData.setElems(rawBuffer.get1DJavaArray(DataType.FLOAT)); + break; + case ProductData.TYPE_FLOAT64: + targetData.setElems(rawBuffer.get1DJavaArray(DataType.DOUBLE)); + break; + default: + throw new IOException("Unknown data type: " + rasterDataType); + } + + return new DataBuffer(targetData, offsets, shapes); + } + + + private static @NonNull ProductData createTargetDataBuffer(int[] shapes, int rasterDataType) throws IOException { + ProductData targetData; + if (shapes.length == 2) { + targetData = ProductData.createInstance(rasterDataType, shapes[0] * shapes[1]); + } else if (shapes.length == 3) { + targetData = ProductData.createInstance(rasterDataType, shapes[0] * shapes[1] * shapes[2]); + } else { + throw new IOException("Illegally shaped variable"); + } + return targetData; + } +} diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasFileReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasFileReader.java index 983d829fd..4be914d02 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasFileReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasFileReader.java @@ -16,13 +16,16 @@ package gov.nasa.gsfc.seadas.dataio; import com.bc.ceres.core.ProgressMonitor; +import eu.esa.snap.core.dataio.cache.CacheDataProvider; +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.ProductCache; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; +import eu.esa.snap.core.datamodel.band.BandUsingReaderDirectly; import org.esa.snap.core.dataio.ProductIOException; import org.esa.snap.core.datamodel.*; -import org.esa.snap.core.util.ProductUtils; import org.esa.snap.core.util.PropertyMap; -import org.esa.snap.core.util.ResourceInstaller; -import org.esa.snap.core.util.SystemUtils; import org.esa.snap.core.util.io.CsvReader; +import org.esa.snap.dataio.netcdf.util.DataTypeUtils; import org.esa.snap.rcp.SnapApp; import ucar.ma2.Array; import ucar.ma2.DataType; @@ -32,16 +35,12 @@ import ucar.nc2.Dimension; import java.awt.*; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.BufferedReader; -import java.io.FileReader; import java.math.BigDecimal; import java.math.MathContext; import java.math.RoundingMode; -import java.nio.file.Path; import java.text.ParseException; import java.util.*; import java.util.List; @@ -49,7 +48,6 @@ import java.util.logging.Logger; import static gov.nasa.gsfc.seadas.dataio.SeadasReaderDefaults.*; -import static java.lang.String.format; import static java.lang.System.arraycopy; /** @@ -63,12 +61,14 @@ //AUG 2024 - Daniel Knowles - added PACE OCI actual center wavelengths lookup in resources -public abstract class SeadasFileReader { +public abstract class SeadasFileReader implements CacheDataProvider { protected boolean mustFlipX; protected boolean mustFlipY; + protected boolean wantsCaching; + protected boolean applyScaling; protected List globalAttributes; - protected Map variableMap; + protected Map variableMap; protected NetcdfFile ncFile; protected SeadasProductReader productReader; protected int[] start = new int[2]; @@ -88,20 +88,30 @@ public abstract class SeadasFileReader { private static final String FLAG_MEANINGS = "flag_meanings"; protected Logger logger = Logger.getLogger(getClass().getSimpleName()); - private boolean isHeadless; + private final boolean isHeadless; + + private ProductCache productCache; protected static final SkipBadNav LAT_SKIP_BAD_NAV = new SkipBadNav() { @Override - public final boolean isBadNav(double value) { + public boolean isBadNav(double value) { return Double.isNaN(value) || value > 90.0 || value < -90.0; } }; public SeadasFileReader(SeadasProductReader productReader) { this.productReader = productReader; + ncFile = productReader.getNcfile(); globalAttributes = ncFile.getGlobalAttributes(); - this.isHeadless = GraphicsEnvironment.isHeadless(); + isHeadless = GraphicsEnvironment.isHeadless(); + + wantsCaching = false; + applyScaling = true; + } + + public void setProductCache(ProductCache productCache) { + this.productCache = productCache; } public abstract Product createProduct() throws IOException; @@ -116,58 +126,110 @@ public synchronized void readBandData(Band destBand, int sourceOffsetX, int sour if (mustFlipX) { sourceOffsetX = destBand.getRasterWidth() - (sourceOffsetX + sourceWidth); } + sourceOffsetY += leadLineSkip; int widthRemainder = destBand.getRasterWidth() - (sourceOffsetX + sourceWidth); if (widthRemainder < 0) { sourceWidth += widthRemainder; } - start[0] = sourceOffsetY; - start[1] = sourceOffsetX; - stride[0] = sourceStepY; - stride[1] = sourceStepX; - count[0] = sourceHeight; - count[1] = sourceWidth; - Object buffer = destBuffer.getElems(); - Variable variable = variableMap.get(destBand); - pm.beginTask("Reading band '" + variable.getShortName() + "'...", sourceHeight); - try { - Section section = new Section(start, count, stride); + if (wantsCaching) { + final int[] offsets; + final int[] shapes; - Array array; - int[] newshape = {sourceHeight, sourceWidth}; + String cacheVariableName = destBand.getName(); + Variable cacheVariable = variableMap.get(cacheVariableName); - array = variable.read(section); - if (array.getRank() > 2) { - array = array.reshapeNoCopy(newshape); + final int spectralBandIndex = destBand.getSpectralBandIndex(); + if (spectralBandIndex >= 0) { + final String ncVariableName = getNcVariableName(cacheVariableName); + final Variable ncVariable = variableMap.get(ncVariableName); + if (ncVariable != null) { + cacheVariableName = ncVariableName; + cacheVariable = ncVariable; + } } - Object storage; - - if (mustFlipX && !mustFlipY) { - storage = array.flip(1).copyTo1DJavaArray(); - } else if (!mustFlipX && mustFlipY) { - storage = array.flip(0).copyTo1DJavaArray(); - } else if (mustFlipX && mustFlipY) { - storage = array.flip(0).flip(1).copyTo1DJavaArray(); + + final int variableRank = cacheVariable.getRank(); + if (spectralBandIndex >= 0 && variableRank == 3) { + offsets = new int[]{spectralBandIndex, sourceOffsetY, sourceOffsetX}; + shapes = new int[]{1, sourceHeight, sourceWidth}; + } else if (variableRank == 2) { + offsets = new int[]{sourceOffsetY, sourceOffsetX}; + shapes = new int[]{sourceHeight, sourceWidth}; } else { - storage = array.copyTo1DJavaArray(); + throw new IOException("Unsupported cached variable rank " + variableRank + + " for variable '" + cacheVariableName + "'"); } - if (widthRemainder < 0) { - arraycopy(storage, 0, buffer, 0, destBuffer.getNumElems() + widthRemainder); - } else { - arraycopy(storage, 0, buffer, 0, destBuffer.getNumElems()); + final int[] targetOffsets = {sourceOffsetY, sourceOffsetX}; + final int[] targetShapes = {sourceHeight, sourceWidth}; + final DataBuffer targetBuffer = new DataBuffer(destBuffer, targetOffsets, targetShapes); + productCache.read(cacheVariableName, offsets, shapes, targetBuffer); + + if (mustFlipX || mustFlipY) { + DataType netcdfDataType = DataTypeUtils.getNetcdfDataType(destBuffer.getType()); + Array resultArray = Array.factory(netcdfDataType, targetShapes, destBuffer.getElems()); + Array flipped = null; + if (mustFlipX && !mustFlipY) { + flipped = resultArray.flip(1); + } else if (!mustFlipX && mustFlipY) { + flipped = resultArray.flip(0); + } else if (mustFlipX && mustFlipY) { + flipped = resultArray.flip(0).flip(1); + } + destBuffer.setElems(flipped.copyTo1DJavaArray()); + } + } else { + + start[0] = sourceOffsetY; + start[1] = sourceOffsetX; + stride[0] = sourceStepY; + stride[1] = sourceStepX; + count[0] = sourceHeight; + count[1] = sourceWidth; + Object buffer = destBuffer.getElems(); + Variable variable = variableMap.get(destBand.getName()); + + pm.beginTask("Reading band '" + variable.getShortName() + "'...", sourceHeight); + try { + Section section = new Section(start, count, stride); + Array array; + int[] newshape = {sourceHeight, sourceWidth}; + + array = variable.read(section); + if (array.getRank() > 2) { + array = array.reshapeNoCopy(newshape); + } + Object storage; + + if (mustFlipX && !mustFlipY) { + storage = array.flip(1).copyTo1DJavaArray(); + } else if (!mustFlipX && mustFlipY) { + storage = array.flip(0).copyTo1DJavaArray(); + } else if (mustFlipX && mustFlipY) { + storage = array.flip(0).flip(1).copyTo1DJavaArray(); + } else { + storage = array.copyTo1DJavaArray(); + } + + if (widthRemainder < 0) { + arraycopy(storage, 0, buffer, 0, destBuffer.getNumElems() + widthRemainder); + } else { + arraycopy(storage, 0, buffer, 0, destBuffer.getNumElems()); + + } + } finally { + pm.done(); } - } finally { - pm.done(); } } - public FlagCoding readFlagCoding(Product product, Band bandName) { - Variable variable = variableMap.get(bandName); + public FlagCoding readFlagCoding(Product product, Band band) { + Variable variable = variableMap.get(band.getName()); if (variable.getFullName().contains("flag")) { final String codingName = variable.getShortName() + "_coding"; return readFlagCoding(variable, codingName); @@ -239,7 +301,6 @@ static String replaceNonWordCharacters(String flagName) { final static Color Cornflower = new Color(38, 115, 245); - static String getFlagDescription(String flagName) { if (flagName == null) { return null; @@ -686,12 +747,12 @@ protected void addFlagsAndMasks(Product product) { } else if (flagNames != null) { // this is the old way of doing this --- leaving in case possibly needed for heritage files. - int flagBits[] = {0x01,0x02,0x04,0x08,0x010,0x020,0x040,0x080,0x100,0x200,0x400,0x800,0x1000,0x2000,0x4000,0x8000,0x10000,0x20000,0x40000,0x80000,0x100000,0x200000,0x400000,0x800000,0x1000000,0x2000000,0x4000000,0x8000000,0x10000000,0x20000000,0x40000000,0x80000000}; //todo finish this list + int[] flagBits = {0x01, 0x02, 0x04, 0x08, 0x010, 0x020, 0x040, 0x080, 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000, 0x10000, 0x20000, 0x40000, 0x80000, 0x100000, 0x200000, 0x400000, 0x800000, 0x1000000, 0x2000000, 0x4000000, 0x8000000, 0x10000000, 0x20000000, 0x40000000, 0x80000000}; //todo finish this list for (int bit = 0; (bit < flagNames.length && bit < flagBits.length); bit++) { String flagName = flagNames[bit]; if (flagName.startsWith("SPARE")) { - flagName = flagName + Integer.toString(bit + 1); + flagName = flagName + (bit + 1); } flagNames[bit] = flagName; // System.out.println("flag=" + flagName); @@ -699,7 +760,7 @@ protected void addFlagsAndMasks(Product product) { // System.out.println("Adding flag=" + flagName); flagCoding.addFlag(flagName, flagBits[bit], getFlagDescription(flagName)); } else { - flagName = flagName + Integer.toString(bit + 1); + flagName = flagName + (bit + 1); // System.out.println("Adding flag=" + flagName); flagCoding.addFlag(flagName, flagBits[bit], getFlagDescription(flagName)); } @@ -716,7 +777,6 @@ protected void addFlagsAndMasks(Product product) { Mask Water_Mask = null; - String masksTopOfStack = getMasksTopOfStack(); String[] masksTopOfStackArray = masksTopOfStack.split("\\s+|,"); @@ -724,29 +784,26 @@ protected void addFlagsAndMasks(Product product) { String[] masksBottomOfStackArray = masksBottomOfStack.split("\\s+|,"); - if (masksTopOfStackArray.length > 0) { - - for (String flagName : masksTopOfStackArray) { - - if (isMaskEnabled(MaskType.COMPOSITE1_INCLUDE) && composite1Mask == null && flagName.equals(getComposite1MaskName())) { - composite1Mask = createMaskComposite1(product, composite1Description, flagCoding); - continue; - } - if (isMaskEnabled(MaskType.COMPOSITE2_INCLUDE) && composite2Mask == null && flagName.equals(getComposite2MaskName())) { - composite2Mask = createMaskComposite2(product, composite2Description, flagCoding); - continue; - } - if (isMaskEnabled(MaskType.COMPOSITE3_INCLUDE) && composite3Mask == null && flagName.equals(getComposite3MaskName())) { - composite3Mask = createMaskComposite3(product, composite3Description, flagCoding); - continue; - } - if (Water_Mask == null && flagName.equals("Water")) { - Water_Mask = createWaterMask(product, Water_Description, flagCoding); - continue; - } + for (String flagName : masksTopOfStackArray) { - addFlagMask(product, flagName, flagCoding); + if (isMaskEnabled(MaskType.COMPOSITE1_INCLUDE) && composite1Mask == null && flagName.equals(getComposite1MaskName())) { + composite1Mask = createMaskComposite1(product, composite1Description, flagCoding); + continue; } + if (isMaskEnabled(MaskType.COMPOSITE2_INCLUDE) && composite2Mask == null && flagName.equals(getComposite2MaskName())) { + composite2Mask = createMaskComposite2(product, composite2Description, flagCoding); + continue; + } + if (isMaskEnabled(MaskType.COMPOSITE3_INCLUDE) && composite3Mask == null && flagName.equals(getComposite3MaskName())) { + composite3Mask = createMaskComposite3(product, composite3Description, flagCoding); + continue; + } + if (Water_Mask == null && flagName.equals("Water")) { + Water_Mask = createWaterMask(product, Water_Description, flagCoding); + continue; + } + + addFlagMask(product, flagName, flagCoding); } String[] flagNames = flagCoding.getFlagNames(); @@ -770,9 +827,6 @@ protected void addFlagsAndMasks(Product product) { } - - - boolean bottomOfStackComposite1 = false; boolean bottomOfStackComposite2 = false; boolean bottomOfStackComposite3 = false; @@ -814,34 +868,28 @@ protected void addFlagsAndMasks(Product product) { } + for (String flagName : masksBottomOfStackArray) { - - if (masksBottomOfStackArray.length > 0) { - - for (String flagName : masksBottomOfStackArray) { - - if (isMaskEnabled(MaskType.COMPOSITE1_INCLUDE) && composite1Mask == null && flagName.equals(getComposite1MaskName())) { - composite1Mask = createMaskComposite1(product, composite1Description, flagCoding); - continue; - } - if (isMaskEnabled(MaskType.COMPOSITE2_INCLUDE) && composite2Mask == null && flagName.equals(getComposite2MaskName())) { - composite2Mask = createMaskComposite2(product, composite2Description, flagCoding); - continue; - } - if (isMaskEnabled(MaskType.COMPOSITE3_INCLUDE) && composite3Mask == null && flagName.equals(getComposite3MaskName())) { - composite3Mask = createMaskComposite3(product, composite3Description, flagCoding); - continue; - } - if (Water_Mask == null && flagName.equals("Water")) { - Water_Mask = createWaterMask(product, Water_Description, flagCoding); - continue; - } - - addFlagMask(product, flagName, flagCoding); + if (isMaskEnabled(MaskType.COMPOSITE1_INCLUDE) && composite1Mask == null && flagName.equals(getComposite1MaskName())) { + composite1Mask = createMaskComposite1(product, composite1Description, flagCoding); + continue; + } + if (isMaskEnabled(MaskType.COMPOSITE2_INCLUDE) && composite2Mask == null && flagName.equals(getComposite2MaskName())) { + composite2Mask = createMaskComposite2(product, composite2Description, flagCoding); + continue; + } + if (isMaskEnabled(MaskType.COMPOSITE3_INCLUDE) && composite3Mask == null && flagName.equals(getComposite3MaskName())) { + composite3Mask = createMaskComposite3(product, composite3Description, flagCoding); + continue; + } + if (Water_Mask == null && flagName.equals("Water")) { + Water_Mask = createWaterMask(product, Water_Description, flagCoding); + continue; } + + addFlagMask(product, flagName, flagCoding); } - - + String[] bandNames = product.getBandNames(); for (String bandName : bandNames) { @@ -968,14 +1016,21 @@ protected void addFlagsAndMasks(Product product) { } - if (composite1Mask != null && isMaskEnabled(MaskType.COMPOSITE1)) {raster.getOverlayMaskGroup().add(composite1Mask);} - if (composite2Mask != null && isMaskEnabled(MaskType.COMPOSITE2)) {raster.getOverlayMaskGroup().add(composite2Mask);} - if (composite3Mask != null && isMaskEnabled(MaskType.COMPOSITE3)) {raster.getOverlayMaskGroup().add(composite3Mask);} - if (isMaskEnabled(MaskType.WATER)) {raster.getOverlayMaskGroup().add(Water_Mask);} + if (composite1Mask != null && isMaskEnabled(MaskType.COMPOSITE1)) { + raster.getOverlayMaskGroup().add(composite1Mask); + } + if (composite2Mask != null && isMaskEnabled(MaskType.COMPOSITE2)) { + raster.getOverlayMaskGroup().add(composite2Mask); + } + if (composite3Mask != null && isMaskEnabled(MaskType.COMPOSITE3)) { + raster.getOverlayMaskGroup().add(composite3Mask); + } + if (isMaskEnabled(MaskType.WATER)) { + raster.getOverlayMaskGroup().add(Water_Mask); + } } - } Band QFBandSST = product.getBand("flags_sst"); if (QFBandSST != null) { @@ -1350,7 +1405,7 @@ private int getFlagBitPosition(int flags) { int bitPosition = -1; // not set if equals -1 // System.out.println("Checking flags=" + flags); - for (int currBitPosition=0; currBitPosition < 32; currBitPosition++) { + for (int currBitPosition = 0; currBitPosition < 32; currBitPosition++) { if (isBitSet(flags, currBitPosition)) { bitPosition = currBitPosition; // System.out.println("flags=" +flags + " currBitPosition=" + currBitPosition); @@ -1371,8 +1426,6 @@ public boolean isBitSet(int n, int i) { } - - private void setFlagMeaningsAndNames(Product product, MetadataElement metadataElementL2Flags) { final MetadataAttribute flagMeaningsAttribute = metadataElementL2Flags.getAttribute(FLAG_MEANINGS); final MetadataAttribute flagMasksAttribute = metadataElementL2Flags.getAttribute(FLAG_MASKS); @@ -1420,9 +1473,6 @@ private Mask createMask(Product product, String maskName, Color maskColor, doubl } - - - private Mask createMaskMisc(Product product, String flagName) { String flag = "l2_flags." + flagName; Mask Misc_Mask = Mask.BandMathsType.create(flagName, getFlagDescription(flagName), @@ -1435,7 +1485,6 @@ private Mask createMaskMisc(Product product, String flagName) { } - private Mask createMaskComposite1(Product product, String composite1Description, FlagCoding flagCoding) { Mask composite1Mask = null; @@ -1457,7 +1506,6 @@ private Mask createMaskComposite1(Product product, String composite1Description, } - private Mask createMaskComposite2(Product product, String composite2Description, FlagCoding flagCoding) { Mask composite2Mask = null; @@ -1501,7 +1549,6 @@ private Mask createMaskComposite3(Product product, String composite3Description, } - private Mask createWaterMask(Product product, String Water_Description, FlagCoding flagCoding) { if (flagCoding != null && flagCoding.containsAttribute("LAND")) { @@ -1517,8 +1564,6 @@ private Mask createWaterMask(Product product, String Water_Description, FlagCodi } - - private Mask createSPAREMask(Product product, String maskName) { String flagName = "l2_flags." + maskName; Mask SPARE_Mask = Mask.BandMathsType.create(maskName, getFlagDescription(maskName), @@ -1534,21 +1579,15 @@ private Mask createSPAREMask(Product product, String maskName) { } - - - - - - - public Map addBands(Product product, - List variables) { - Map bandToVariableMap = new HashMap(); + public Map addBands(Product product, + List variables) { + Map bandToVariableMap = new HashMap(); for (Variable variable : variables) { int variableRank = variable.getRank(); if (variableRank == 2) { Band band = addNewBand(product, variable); if (band != null) { - bandToVariableMap.put(band, variable); + bandToVariableMap.put(band.getName(), variable); } } else if (variableRank == 3) { add3DNewBands(product, variable, bandToVariableMap); @@ -1560,6 +1599,13 @@ public Map addBands(Product product, return bandToVariableMap; } + protected Band createBand(String name, int dataType, int width, int height) { + if (wantsCaching) { + return new BandUsingReaderDirectly(name, dataType, width, height); + } + return new Band(name, dataType, width, height); + } + protected void setSpectralBand(Product product) { // todo Later Possibly Delete: but for now commenting out code which used decimal wavelength lookup file @@ -1658,7 +1704,7 @@ protected void setSpectralBand(Product product) { // todo END todo block - protected Map add3DNewBands(Product product, Variable variable, Map bandToVariableMap) { + protected Map add3DNewBands(Product product, Variable variable, Map bandToVariableMap) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); @@ -1681,7 +1727,7 @@ protected Map add3DNewBands(Product product, Variable variable, if (height == sceneRasterHeight && width == sceneRasterWidth) { // final List list = variable.getAttributes(); List dims = ncFile.getDimensions(); - for (Dimension d: dims){ + for (Dimension d : dims) { if (d.getShortName().equalsIgnoreCase("wavelength_3d")) { dim = d.getLength(); } @@ -1750,7 +1796,7 @@ protected Map add3DNewBands(Product product, Variable variable, if (!product.containsBand(name)) { - final Band band = new Band(name, dataType, width, height); + final Band band = createBand(name, dataType, width, height); product.addBand(band); Variable sliced = null; @@ -1759,15 +1805,17 @@ protected Map add3DNewBands(Product product, Variable variable, } catch (InvalidRangeException e) { e.printStackTrace(); //Todo change body of catch statement. } - bandToVariableMap.put(band, sliced); + bandToVariableMap.put(band.getName(), sliced); try { Attribute fillValue = variable.findAttribute("_FillValue"); - if (fillValue == null) { + if (fillValue == null && applyScaling) { fillValue = variable.findAttribute("bad_value_scaled"); } - band.setNoDataValue((double) fillValue.getNumericValue().floatValue()); - band.setNoDataValueUsed(true); + if (fillValue != null) { + band.setNoDataValue(fillValue.getNumericValue().floatValue()); + band.setNoDataValueUsed(true); + } band.setSpectralWavelength(wavelengths.getFloat(i)); band.setSpectralBandIndex(1); } catch (Exception ignored) { @@ -1781,13 +1829,13 @@ protected Map add3DNewBands(Product product, Variable variable, band.setUnit(attribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(attribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(attribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(attribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(attribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(attribute.getNumericValue(0).doubleValue()); } else if (attribName.startsWith("valid_")) { if ("valid_min".equals(attribName)) { @@ -1869,17 +1917,19 @@ protected Band addNewBand(Product product, Variable variable) { if (!product.containsBand(name)) { - band = new Band(name, dataType, width, height); + band = createBand(name, dataType, width, height); product.addBand(band); try { Attribute fillValue = variable.findAttribute("_FillValue"); - if (fillValue == null) { + if (fillValue == null && applyScaling) { fillValue = variable.findAttribute("bad_value_scaled"); } - band.setNoDataValue((double) fillValue.getNumericValue().floatValue()); - band.setNoDataValueUsed(true); + if (fillValue != null) { + band.setNoDataValue(fillValue.getNumericValue().floatValue()); + band.setNoDataValueUsed(true); + } } catch (Exception ignored) { } @@ -1891,13 +1941,13 @@ protected Band addNewBand(Product product, Variable variable) { band.setUnit(attribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(attribute.getStringValue()); - } else if ("slope".equals(attribName)) { + } else if (applyScaling && "slope".equals(attribName)) { band.setScalingFactor(attribute.getNumericValue(0).doubleValue()); - } else if ("intercept".equals(attribName)) { + } else if (applyScaling && "intercept".equals(attribName)) { band.setScalingOffset(attribute.getNumericValue(0).doubleValue()); - } else if ("scale_factor".equals(attribName)) { + } else if (applyScaling && "scale_factor".equals(attribName)) { band.setScalingFactor(attribute.getNumericValue(0).doubleValue()); - } else if ("add_offset".equals(attribName)) { + } else if (applyScaling && "add_offset".equals(attribName)) { band.setScalingOffset(attribute.getNumericValue(0).doubleValue()); } else if (attribName.startsWith("valid_")) { if ("valid_min".equals(attribName)) { @@ -1964,8 +2014,6 @@ protected Band addNewBand(Product product, Variable variable) { } - - public String formatValidMinMax(double value, boolean isMax) { String valueStr; @@ -1978,7 +2026,6 @@ public String formatValidMinMax(double value, boolean isMax) { } - boolean roundBigNumbersToInt = false; // if do not round big numbers, then increase significant figure to length of integer (left side of decimal place) @@ -2028,13 +2075,10 @@ public String formatValidMinMax(double value, boolean isMax) { } - - - - private static String cleanUpPaddedDecimalZeros(String valueStr) { + private static String cleanUpPaddedDecimalZeros(String valueStr) { if (valueStr.contains(".") && !valueStr.toLowerCase().contains("e")) { while (valueStr.endsWith("0") && !valueStr.endsWith(".0")) { - valueStr = valueStr.substring(0,valueStr.length()-1); + valueStr = valueStr.substring(0, valueStr.length() - 1); } } @@ -2046,7 +2090,6 @@ private static String cleanUpPaddedDecimalZeros(String valueStr) { } - public void addGlobalMetadata(Product product) { final MetadataElement globalElement = new MetadataElement("Global_Attributes"); addAttributesToElement(globalAttributes, globalElement); @@ -2192,7 +2235,6 @@ public float computeGeoPixel(final float a, final float b, final double weight) } - public boolean getDefaultFlip() throws ProductIOException { return getDefaultFlip(false); } @@ -2228,7 +2270,6 @@ public boolean getDefaultFlip(boolean defaultFlip) throws ProductIOException { } - protected static HashMap readTwoColumnTable(String resourceName) { final InputStream stream = SeadasProductReader.class.getResourceAsStream(resourceName); if (stream != null) { @@ -2580,8 +2621,6 @@ private boolean isMaskSort() { } - - // Composite1 private String getComposite1MaskName() { @@ -2596,7 +2635,6 @@ private String getComposite1MaskName() { } - // Composite2 private String getComposite2MaskName() { @@ -2611,7 +2649,6 @@ private String getComposite2MaskName() { } - // Composite3 private String getComposite3MaskName() { @@ -2626,8 +2663,6 @@ private String getComposite3MaskName() { } - - private String validateCompositeFlagsName(String compositeMaskName, String propertyKey) { // don't let composite mask name be same as any of the flags if (compositeMaskName != null) { @@ -2684,7 +2719,7 @@ private String getCompositeFlagsExpression(String flags, FlagCoding flagCoding) String[] flagsArray = flags.split("\\s+|,"); - for (int i=0; i < flagsArray.length; i++) { + for (int i = 0; i < flagsArray.length; i++) { if (flagsArray[i].contains("l2_flags.")) { flagsArray[i] = flagsArray[i].replace("l2_flags.", ""); } @@ -2745,7 +2780,6 @@ private ArrayList getValidFlagsComposite(String[] flagsArray, FlagCoding } - private boolean isMaskEnabled(MaskType maskType) { if (isHeadless) { return getDefaultBool(maskType); @@ -2779,9 +2813,6 @@ private String getMaskComposite(MaskType maskType) { } - - - public String getBandGroupingLevel2() { if (this.isHeadless) { return PROPERTY_LEVEL2_BAND_GROUPING_DEFAULT; @@ -2832,7 +2863,6 @@ public String getBandFlipYL1BPace() { } - public String getBandGroupingL3Mapped() { if (this.isHeadless) { return SeadasReaderDefaults.PROPERTY_L3_MAPPED_BAND_GROUPING_DEFAULT; @@ -2858,8 +2888,6 @@ public String getBandFlipYL3Mapped() { } - - public String getBandGroupingL1CPace() { if (this.isHeadless) { return SeadasReaderDefaults.PROPERTY_L1C_PACE_OCI_BAND_GROUPING_DEFAULT; @@ -2952,7 +2980,6 @@ public String getBandFlipYL1B_VIIRS() { } - public int get_VALID_PIXEL_SIG_FIGS() { String sigFixStr; if (this.isHeadless) { @@ -2975,11 +3002,21 @@ public int get_VALID_PIXEL_SIG_FIGS() { } - - private interface SkipBadNav { - boolean isBadNav(double value); } + String getNcVariableName(String variableName) { + return variableName; + } + + @Override + public VariableDescriptor getVariableDescriptor(String variableName) throws IOException { + throw new RuntimeException("This method must be overridden if the reader wants to use caching"); + } + + @Override + public DataBuffer readCacheBlock(String variableName, int[] offsets, int[] shapes, ProductData targetData) throws IOException { + throw new RuntimeException("This method must be overridden if the reader wants to use caching"); + } } diff --git a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasProductReader.java b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasProductReader.java index 376d1fbbe..778a261ce 100644 --- a/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasProductReader.java +++ b/opttbx-seadas-reader/src/main/java/gov/nasa/gsfc/seadas/dataio/SeadasProductReader.java @@ -16,6 +16,7 @@ package gov.nasa.gsfc.seadas.dataio; import com.bc.ceres.core.ProgressMonitor; +import eu.esa.snap.core.dataio.cache.*; import org.esa.snap.core.dataio.AbstractProductReader; import org.esa.snap.core.dataio.ProductIOException; import org.esa.snap.core.dataio.ProductReaderPlugIn; @@ -33,21 +34,22 @@ import java.util.List; -// import org.opengis.filter.spatial.Equals; - -public class SeadasProductReader extends AbstractProductReader { +public class SeadasProductReader extends AbstractProductReader implements CacheDataProvider { private NetcdfFile ncfile; private ProductType productType; private SeadasFileReader seadasFileReader; + private ProductCache productCache; enum Mission { OCI("OCI"); - private String name; - private Mission(String nm) { + private final String name; + + Mission(String nm) { name = nm; } + public String toString() { return name; } @@ -61,17 +63,18 @@ enum ProcessingLevel { L3b("L3 Binned"), L3m("L3 Mapped"); - private String name; - private ProcessingLevel(String nm) { + private final String name; + + ProcessingLevel(String nm) { name = nm; } + public String toString() { return name; } } - enum ProductType { ANCNRT("SeaWiFS Near Real-Time Ancillary Data"), ANCNRT2("NCEP Reanalysis 2 Ancillary Data"), @@ -132,10 +135,15 @@ public String toString() { */ protected SeadasProductReader(ProductReaderPlugIn readerPlugIn) { super(readerPlugIn); + + productCache = null; + ncfile = null; } @Override protected Product readProductNodesImpl() throws IOException { + productCache = new ProductCache(this); + CacheManager.getInstance().register(productCache); try { // Product product; @@ -145,6 +153,8 @@ protected Product readProductNodesImpl() throws IOException { ncfile = NetcdfFileOpener.open(path); productType = findProductType(); seadasFileReader = getReaderFromProductType(productType, this); + // @todo 1 tb refactor 2026-04-22 + seadasFileReader.setProductCache(productCache); Product product = seadasFileReader.createProduct(); @@ -222,8 +232,14 @@ static SeadasFileReader getReaderFromProductType(ProductType productType, Seadas @Override public void close() throws IOException { + if (productCache != null) { + CacheManager.getInstance().remove(productCache); + productCache = null; + } + if (ncfile != null) { ncfile.close(); + ncfile = null; } } @@ -232,8 +248,6 @@ protected void readBandRasterDataImpl(int sourceOffsetX, int sourceOffsetY, int int sourceStepX, int sourceStepY, Band destBand, int destOffsetX, int destOffsetY, int destWidth, int destHeight, ProductData destBuffer, ProgressMonitor pm) throws IOException { - - try { seadasFileReader.readBandData(destBand, sourceOffsetX, sourceOffsetY, sourceWidth, sourceHeight, sourceStepX, sourceStepY, destBuffer, pm); @@ -262,12 +276,7 @@ static boolean checkEqcProjection(Attribute mapProjectionAttribute) { } try { String projection = mapProjectionAttribute.getStringValue(); - if (projection.contains("proj=eqc") || projection.contains("Equidistant Cylindrical")){ - return true; - } - else { - return false; - } + return projection.contains("proj=eqc") || projection.contains("Equidistant Cylindrical"); } catch (Exception e) { return false; } @@ -453,9 +462,9 @@ static ProductType findProductTypeWithoutFileAccess(String title, String process return ProductType.ANCCLIM; } else if (title.contains("Level-3") && title.contains("Mapped Image")) { return ProductType.Level3_SeadasMapped; - }else if (title.matches("(.*)Level-3 Standard Mapped Image") || title.matches("(.*)Level-3 Equidistant Cylindrical Mapped Image")) { + } else if (title.matches("(.*)Level-3 Standard Mapped Image") || title.matches("(.*)Level-3 Equidistant Cylindrical Mapped Image")) { return ProductType.SMI; - } else if (title.contains("Level-3 Binned Data") || title.contains("level-3_binned_data")) { + } else if (title.contains("Level-3 Binned Data") || title.contains("level-3_binned_data")) { return ProductType.Level3_Bin; } return ProductType.UNKNOWN; @@ -470,4 +479,14 @@ private static String getStringAttributeValue(Attribute processing_levelAttr) { } return null; } + + @Override + public VariableDescriptor getVariableDescriptor(String variableName) throws IOException { + return seadasFileReader.getVariableDescriptor(variableName); + } + + @Override + public DataBuffer readCacheBlock(String variableName, int[] offsets, int[] shapes, ProductData targetData) throws IOException { + return seadasFileReader.readCacheBlock(variableName, offsets, shapes, targetData); + } } diff --git a/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReaderTest.java b/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReaderTest.java new file mode 100644 index 000000000..c8f44873c --- /dev/null +++ b/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/L1BPaceOciFileReaderTest.java @@ -0,0 +1,86 @@ +package gov.nasa.gsfc.seadas.dataio; + +import com.bc.ceres.annotation.STTM; +import org.esa.snap.core.datamodel.Band; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.Test; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class L1BPaceOciFileReaderTest { + + @Test + @STTM("SNAP-4122") + public void testGetVariableName() { + assertEquals("werner_675.776", L1BPaceOciFileReader.getVariableName("werner",675.776f)); + assertEquals("heffalump_298.5502", L1BPaceOciFileReader.getVariableName("heffalump",298.5502f)); + } + + @Test + @STTM("SNAP-4122") + public void testAddAttributes_slope_intercept() { + final Variable variable = mock(Variable.class); + when(variable.findAttribute("units")).thenReturn(new Attribute("units", "m^2/kg")); + when(variable.findAttribute("long_name")).thenReturn(new Attribute("long_name", "Maria")); + when(variable.findAttribute("slope")).thenReturn(new Attribute("slope", 18.65)); + when(variable.findAttribute("intercept")).thenReturn(new Attribute("intercept", -0.887)); + when(variable.findAttribute("bad_value_scaled")).thenReturn(new Attribute("bad_value_scaled", 88.108)); + + final Band band = new Band("test", ProductData.TYPE_FLOAT64, 12, 14); + L1BPaceOciFileReader.addAttributes(variable, band, true); + + assertEquals("m^2/kg", band.getUnit()); + assertEquals("Maria", band.getDescription()); + assertEquals(18.65, band.getScalingFactor(), 1e-8); + assertEquals(-0.887, band.getScalingOffset(), 1e-8); + assertEquals(88.108, band.getNoDataValue(), 1e-8); + assertTrue(band.isNoDataValueUsed()); + } + + @Test + @STTM("SNAP-4122") + public void testAddAttributes_slope_intercept_applyScalingPath() { + final Variable variable = mock(Variable.class); + when(variable.findAttribute("units")).thenReturn(new Attribute("units", "m^2/kg")); + when(variable.findAttribute("long_name")).thenReturn(new Attribute("long_name", "Maria")); + when(variable.findAttribute("slope")).thenReturn(new Attribute("slope", 18.65)); + when(variable.findAttribute("intercept")).thenReturn(new Attribute("intercept", -0.887)); + when(variable.findAttribute("bad_value_scaled")).thenReturn(new Attribute("bad_value_scaled", 88.108)); + + final Band band = new Band("test", ProductData.TYPE_FLOAT64, 12, 14); + L1BPaceOciFileReader.addAttributes(variable, band, false); + + assertEquals("m^2/kg", band.getUnit()); + assertEquals("Maria", band.getDescription()); + assertEquals(1.0, band.getScalingFactor(), 1e-8); + assertEquals(0.0, band.getScalingOffset(), 1e-8); + assertEquals(0.0, band.getNoDataValue(), 1e-8); + assertFalse(band.isNoDataValueUsed()); + } + + @Test + @STTM("SNAP-4122") + public void testAddAttributes_scale_offset() { + final Variable variable = mock(Variable.class); + when(variable.findAttribute("slope")).thenReturn(new Attribute("scale_factor", 0.855)); + when(variable.findAttribute("intercept")).thenReturn(new Attribute("add_offset", 100.4)); + + final Band band = new Band("test", ProductData.TYPE_FLOAT64, 12, 14); + L1BPaceOciFileReader.addAttributes(variable, band, true); + + assertEquals(0.855, band.getScalingFactor(), 1e-8); + assertEquals(100.4, band.getScalingOffset(), 1e-8); + assertFalse(band.isNoDataValueUsed()); + } + + @Test + @STTM("SNAP-4122") + public void testStripWavelengthFromName() { + assertEquals("rhot_blue", L1BPaceOciFileReader.stripWavelengthFromName("rhot_blue_123.554")); + assertEquals("rhot_red", L1BPaceOciFileReader.stripWavelengthFromName("rhot_red_642.228")); + } +} diff --git a/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtilsTest.java b/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtilsTest.java new file mode 100644 index 000000000..1e74604c4 --- /dev/null +++ b/opttbx-seadas-reader/src/test/java/gov/nasa/gsfc/seadas/dataio/SeadasCacheUtilsTest.java @@ -0,0 +1,190 @@ +package gov.nasa.gsfc.seadas.dataio; + +import com.bc.ceres.annotation.STTM; +import eu.esa.snap.core.dataio.cache.DataBuffer; +import eu.esa.snap.core.dataio.cache.VariableDescriptor; +import org.esa.snap.core.datamodel.ProductData; +import org.junit.Test; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Variable; + +import java.io.IOException; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +public class SeadasCacheUtilsTest { + + + @Test + @STTM("SNAP-4122") + public void test_getDefaultVariableDescriptor_2DVariableWithoutChunkSizes() throws IOException { + final Variable variable = mock(Variable.class); + when(variable.getDataType()).thenReturn(DataType.FLOAT); + when(variable.getShape()).thenReturn(new int[]{17, 23}); + + final VariableDescriptor descriptor = SeadasCacheUtils.getDefaultVariableDescriptor(variable, "radiance"); + + assertEquals("radiance", descriptor.name); + assertEquals(ProductData.TYPE_FLOAT32, descriptor.dataType); + assertEquals(23, descriptor.width); + assertEquals(17, descriptor.height); + assertEquals(-1, descriptor.layers); + assertEquals(23, descriptor.tileWidth); + assertEquals(17, descriptor.tileHeight); + assertEquals(-1, descriptor.tileLayers); + } + + @Test + @STTM("SNAP-4122") + public void test_getDefaultVariableDescriptor_3DVariableWithChunkSizes() throws IOException { + final Variable variable = mock(Variable.class); + final Attribute chunkSizes = mock(Attribute.class); + when(chunkSizes.getValues()).thenReturn(Array.factory(DataType.INT, new int[]{3}, new int[]{2, 5, 7})); + when(variable.getDataType()).thenReturn(DataType.SHORT); + when(variable.getShape()).thenReturn(new int[]{11, 13, 19}); + when(variable.findAttribute("_ChunkSizes")).thenReturn(chunkSizes); + + final VariableDescriptor descriptor = SeadasCacheUtils.getDefaultVariableDescriptor(variable, "rhot_blue"); + + assertEquals("rhot_blue", descriptor.name); + assertEquals(ProductData.TYPE_INT16, descriptor.dataType); + assertEquals(19, descriptor.width); + assertEquals(13, descriptor.height); + assertEquals(11, descriptor.layers); + assertEquals(7, descriptor.tileWidth); + assertEquals(5, descriptor.tileHeight); + assertEquals(2, descriptor.tileLayers); + } + + @Test + @STTM("SNAP-4122") + public void test_getDefaultVariableDescriptor_nullVariableThrowsIOException() { + try { + SeadasCacheUtils.getDefaultVariableDescriptor(null, "missing"); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("Variable not known: missing", e.getMessage()); + } + } + + @Test + @STTM("SNAP-4122") + public void test_getDefaultVariableDescriptor_UnsupportedRankThrowsIOException() { + final Variable variable = mock(Variable.class); + when(variable.getDataType()).thenReturn(DataType.FLOAT); + when(variable.getShape()).thenReturn(new int[]{3, 5, 7, 11}); + + try { + SeadasCacheUtils.getDefaultVariableDescriptor(variable, "too_many_dims"); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("Unsupported variable rank for caching: 4", e.getMessage()); + } + } + + @Test + @STTM("SNAP-4122") + public void test_readArrayReadsRequestedSection() throws IOException { + final Variable variable = mock(Variable.class); + final int[] offsets = {3, 4}; + final int[] shapes = {2, 3}; + final Array expected = Array.factory(DataType.INT, shapes, new int[]{1, 2, 3, 4, 5, 6}); + try { + when(variable.read(offsets, shapes)).thenReturn(expected); + } catch (InvalidRangeException e) { + fail(e.getMessage()); + } + + final Array actual = SeadasCacheUtils.readArray(variable, offsets, shapes); + + assertSame(expected, actual); + } + + @Test + @STTM("SNAP-4122") + public void test_readArrayWrapsInvalidRangeException() throws InvalidRangeException, IOException { + final Variable variable = mock(Variable.class); + final int[] offsets = {3, 4}; + final int[] shapes = {2, 3}; + final InvalidRangeException invalidRangeException = new InvalidRangeException("bad range"); + when(variable.read(offsets, shapes)).thenThrow(invalidRangeException); + + try { + SeadasCacheUtils.readArray(variable, offsets, shapes); + fail("Expected IOException"); + } catch (IOException e) { + assertSame(invalidRangeException, e.getCause()); + } + } + + @Test + @STTM("SNAP-4122") + public void test_ConstructDataBufferCreatesFloat32TargetFor2DShape() throws IOException { + final int[] offsets = {4, 6}; + final int[] shapes = {2, 3}; + final float[] values = {1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f}; + final Array rawBuffer = Array.factory(DataType.FLOAT, shapes, values); + + final DataBuffer dataBuffer = SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, null, ProductData.TYPE_FLOAT32); + + assertArrayEquals(values, (float[]) dataBuffer.getData().getElems(), 1e-6f); + assertEquals(ProductData.TYPE_FLOAT32, dataBuffer.getData().getType()); + assertArrayEquals(new int[]{-1, 4, 6}, dataBuffer.getOffsets()); + assertArrayEquals(new int[]{-1, 2, 3}, dataBuffer.getShapes()); + assertEquals(6, dataBuffer.getSizeInBytes() / ProductData.getElemSize(ProductData.TYPE_FLOAT32)); + } + + @Test + @STTM("SNAP-4122") + public void test_constructDataBufferReusesProvidedInt16TargetFor3DShape() throws IOException { + final int[] offsets = {1, 4, 6}; + final int[] shapes = {1, 2, 3}; + final short[] values = {1, 2, 3, 4, 5, 6}; + final ProductData targetData = ProductData.createInstance(ProductData.TYPE_INT16, values.length); + final Array rawBuffer = Array.factory(DataType.SHORT, shapes, values); + + final DataBuffer dataBuffer = SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, targetData, ProductData.TYPE_INT16); + + assertSame(targetData, dataBuffer.getData()); + assertArrayEquals(values, (short[]) dataBuffer.getData().getElems()); + assertArrayEquals(offsets, dataBuffer.getOffsets()); + assertArrayEquals(shapes, dataBuffer.getShapes()); + } + + @Test + @STTM("SNAP-4122") + public void test_constructDataBufferRejectsUnknownDataType() { + final int[] offsets = {0, 0}; + final int[] shapes = {1, 2}; + final Array rawBuffer = Array.factory(DataType.FLOAT, shapes, new float[]{1.0f, 2.0f}); + final ProductData targetData = ProductData.createInstance(ProductData.TYPE_FLOAT32, 2); + + try { + SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, targetData, -999); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("Unknown data type: -999", e.getMessage()); + } + } + + @Test + @STTM("SNAP-4122") + public void test_constructDataBufferRejectsUnsupportedShapeWhenTargetMustBeCreated() { + final int[] offsets = {0, 0, 0, 0}; + final int[] shapes = {1, 2, 3, 4}; + final Array rawBuffer = Array.factory(DataType.FLOAT, shapes, new float[24]); + + try { + SeadasCacheUtils.constructDataBuffer(rawBuffer, offsets, shapes, null, ProductData.TYPE_FLOAT32); + fail("Expected IOException"); + } catch (IOException e) { + assertEquals("Illegally shaped variable", e.getMessage()); + } + } +} \ No newline at end of file