From c8a3ac76c7f11bae0c56245cd81d17acfac2128c Mon Sep 17 00:00:00 2001 From: Joerg Kiegeland Date: Sun, 12 May 2024 13:13:09 +1000 Subject: [PATCH] describe DJL blocks as tensorflow --- api/src/main/java/ai/djl/nn/Blocks.java | 222 +++++++++++ api/src/main/java/ai/djl/nn/core/Add.java | 54 +++ .../main/java/ai/djl/nn/norm/BatchNorm.java | 45 +++ .../djl/examples/training/DescribeTest.java | 197 ++++++++++ .../src/test/resources/describe/Mnist.txt | 8 + .../resources/describe/MnistFunctional.py | 11 + .../resources/describe/MnistInitialized.txt | 8 + .../resources/describe/MnistSequential.py | 10 + .../src/test/resources/describe/Resnet50.py | 222 +++++++++++ .../tests/training/EvaluateDatasetTest.java | 2 +- .../listener/EarlyStoppingListenerTest.java | 6 +- .../java/ai/djl/basicmodelzoo/basic/Mlp.java | 8 +- .../cv/classification/ResNetV2.java | 347 ++++++++++++++++++ 13 files changed, 1131 insertions(+), 9 deletions(-) create mode 100644 api/src/main/java/ai/djl/nn/core/Add.java create mode 100644 examples/src/test/java/ai/djl/examples/training/DescribeTest.java create mode 100644 examples/src/test/resources/describe/Mnist.txt create mode 100644 examples/src/test/resources/describe/MnistFunctional.py create mode 100644 examples/src/test/resources/describe/MnistInitialized.txt create mode 100644 examples/src/test/resources/describe/MnistSequential.py create mode 100644 examples/src/test/resources/describe/Resnet50.py create mode 100644 model-zoo/src/main/java/ai/djl/basicmodelzoo/cv/classification/ResNetV2.java diff --git a/api/src/main/java/ai/djl/nn/Blocks.java b/api/src/main/java/ai/djl/nn/Blocks.java index a4ff77c5d19..299e97dedd9 100644 --- a/api/src/main/java/ai/djl/nn/Blocks.java +++ b/api/src/main/java/ai/djl/nn/Blocks.java @@ -17,9 +17,15 @@ import ai.djl.ndarray.NDManager; import ai.djl.ndarray.types.DataType; import ai.djl.ndarray.types.Shape; +import ai.djl.nn.convolutional.Convolution; +import ai.djl.nn.norm.BatchNorm; +import ai.djl.training.Trainer; +import ai.djl.training.loss.Loss; import ai.djl.util.Pair; import ai.djl.util.PairList; +import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -167,4 +173,220 @@ public static String describe(Block block, String blockName, int beginAxis) { } return sb.toString(); } + + /** + * Builds an equivalent tensorflow model from the the DJL model using functional or sequential + * API. + * + * @param trainer The trainer containing the DJL model + * @param functionalApi if true, keras's functional API is used, otherwise the + * sequential API. The model should be initialized when using the functional API. + * @return Python code + */ + public static String describeAsTensorflow(Trainer trainer, boolean functionalApi) { + Block block = trainer.getModel().getBlock(); + String model = + describeAsTensorflow(block, "SequentialBlock", "", functionalApi ? "inputs" : null); + if (functionalApi) { + String inputLayer = + block.isInitialized() + ? String.format( + "inputs = tf.keras.layers.InputLayer(input_shape = %s).output", + block.getInputShapes()[0].slice(1)) + : "# define input tensor here"; + return String.format( + "%s%n%s%nmodel = tf.keras.Model(inputs=inputs, outputs=outputs)%n%nloss = %s", + inputLayer, model, describeAsTensorflow(trainer.getLoss())); + } + return String.format( + "model = %s%n%nloss = %s", model, describeAsTensorflow(trainer.getLoss())); + } + + static String describeAsTensorflow(Loss loss) { + switch (loss.getClass().getSimpleName()) { + case "SoftmaxCrossEntropyLoss": + return "tf.keras.losses.categorical_crossentropy"; + default: + return "tf.keras.losses.mean_squared_error"; + } + } + + /** + * Builds a tensorflow layer equivalent to the passed {@link Block}. + * + * @param block the block to translate + * @param blockName the DJL name of the passed block, or null if the block's class + * name is to be used + * @param pythonName the name to be used for the keras layer name + * @param input if not null, the input tensor to call the layer with required by + * functional API, otherwise sequential API is used + * @return Python expression for sequential API or Python statements for functional API + */ + public static String describeAsTensorflow( + Block block, String blockName, String pythonName, String input) { + if (block instanceof LambdaBlock + && !LambdaBlock.DEFAULT_NAME.equals(((LambdaBlock) block).getName())) { + blockName = ((LambdaBlock) block).getName(); + } + switch (blockName) { + case "ParallelBlock": + { + Object[][] args = {{-1}}; + return format("tf.keras.layers.Concatenate", args, block, pythonName, input); + } + case "batchFlatten": + { + Object[][] args = {}; + return format("tf.keras.layers.Flatten", args, block, pythonName, input); + } + case "SequentialBlock": + { + Object[][] args = {{-1}}; + String op = + pythonName.isEmpty() + ? "tf.keras.models.Sequential" + : "tf.keras.Sequential"; + return format(op, args, block, pythonName, input); + } + case "Add": + { + Object[][] args = {{-1}}; + return format("tf.keras.layers.Add", args, block, pythonName, input); + } + case "Linear": + { + Object[][] args = { + {block.getOutputShapes(new Shape[] {new Shape(0)})[0].get(0)} + }; + return format("tf.keras.layers.Dense", args, block, pythonName, input); + } + case "GELU": + case "Mish": + case "ReLU6": + case "ReLU": + case "SELU": + case "sigmoid": + case "softPlus": + case "softSign": + case "Tanh": + { + Object[][] args = {{"tf.keras.activations." + blockName.toLowerCase()}}; + return format("tf.keras.layers.Activation", args, block, pythonName, input); + } + case "identity": + { + Object[][] args = {}; + return format("tf.keras.layers.Identity", args, block, pythonName, input); + } + case "Conv2d": + { + Convolution conv = (Convolution) block; + String padding = + new Shape(0, 0).equals(conv.getPadding()) ? "'VALID'" : "'SAME'"; + Object[][] args = { + {conv.getFilters(), "filters"}, + {conv.getKernelShape(), "kernel_size"}, + {conv.getStride(), "strides"}, + {null, "padding", padding}, + {conv.getDilation(), "dilation_rate"}, + {null, "data_format", "'channels_first'"}, + {null, "use_bias", conv.isIncludeBias()} + }; + return format("tf.keras.layers.Conv2D", args, block, pythonName, input); + } + + case "BatchNorm": + { + BatchNorm norm = (BatchNorm) block; + Object[][] args = { + {norm.getScale(), "scale"}, + {norm.getCenter(), "center"}, + {norm.getEpsilon(), "epsilon"}, + {norm.getMomentum(), "momentum"}, + {norm.getAxis(), "axis"} + }; + return format( + "tf.keras.layers.BatchNormalization", args, block, pythonName, input); + } + + case "globalAvgPool2d": + { + Object[][] args = {{null, "data_format", "'channels_first'"}}; + return format( + "tf.keras.layers.GlobalAveragePooling2D", + args, + block, + pythonName, + input); + } + default: + { + Object[][] args = {{-1}}; + return format(blockName, args, block, pythonName, input); + } + } + } + + static String format(String op, Object[][] args, Block block, String pythonName, String input) { + String pref = ""; + StringBuilder sb = new StringBuilder(op + "("); + for (Object[] arg : args) { + String s = arg.length >= 3 ? String.valueOf(arg[2]) : null; + if (Integer.valueOf(-1).equals(arg[0])) { + List nameOfLayers = new ArrayList<>(); + List layers = new ArrayList<>(); + for (Pair pair : block.getChildren()) { + String name = pair.getKey().substring(2); + String pythonNameOfLayer = + pythonName + + (pythonName.isEmpty() ? "" : "_") + + name + + pair.getKey().substring(0, 2); + String layer = + describeAsTensorflow(pair.getValue(), name, pythonNameOfLayer, input); + layers.add(layer); + if (input != null) { + nameOfLayers.add( + layer.substring( + layer.lastIndexOf('\n') + 1, layer.lastIndexOf(" = "))); + if (op.endsWith("Sequential")) { + input = nameOfLayers.get(nameOfLayers.size() - 1); + } + } + } + if (input != null) { + pref = layers.stream().collect(Collectors.joining("\n", "", "\n")); + if (!op.endsWith("Sequential")) { + input = nameOfLayers.stream().collect(Collectors.joining(", ", "[", "]")); + } + continue; + } else { + s = + layers.stream() + .map(b -> b.replaceAll("(?m)^", " ")) + .collect(Collectors.joining(",\n", "[\n", "\n]")); + } + } else if (arg[0] != null) { + s = arg[0].toString(); + } else if (s == null) { + continue; // cannot resolve index, so skip + } + s = "true".equals(s) ? "True" : "false".equals(s) ? "False" : s; + if (arg.length >= 2 && arg[1] != null) { + s = String.format("%s=%s", arg[1], s); + } + sb.append(s); + sb.append(", "); + } + String name = pythonName.isEmpty() ? "outputs" : pythonName; + sb.append(String.format("name='%s'", name)); + sb.append(')'); + if (input != null) { + if (op.endsWith("Sequential")) { + return String.format("%s%s = %s", pref, name, input); + } + return String.format("%s%s = %s(%s)", pref, name, sb, input); + } + return sb.toString(); + } } diff --git a/api/src/main/java/ai/djl/nn/core/Add.java b/api/src/main/java/ai/djl/nn/core/Add.java new file mode 100644 index 00000000000..dfc305ae611 --- /dev/null +++ b/api/src/main/java/ai/djl/nn/core/Add.java @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.nn.core; + +import ai.djl.ndarray.NDArray; +import ai.djl.ndarray.NDArrays; +import ai.djl.ndarray.NDList; +import ai.djl.nn.Block; +import ai.djl.nn.ParallelBlock; + +import java.util.Collections; +import java.util.List; + +/** + * {@code Add} is a {@link Block} whose children form a parallel branch in the network and are + * combined by {@link NDArrays#add(NDArray...)} to produce a single output. + * + *

{@code Add} has no direct parameters. + */ +public class Add extends ParallelBlock { + + /** + * Creates a block whose branches are combined to form a single output by {@link + * NDArrays#add(NDArray...)}. + */ + public Add() { + this(Collections.emptyList()); + } + + /** + * Creates a block whose branches are formed by each block in the list of blocks, and are + * combined by {@link NDArrays#add(NDArray...)} to form a single output. + * + * @param blocks the blocks that form each of the parallel branches + */ + public Add(List blocks) { + super( + list -> { + NDArray[] arrays = list.stream().map(NDList::head).toArray(NDArray[]::new); + return new NDList(NDArrays.add(arrays)); + }, + blocks); + } +} diff --git a/api/src/main/java/ai/djl/nn/norm/BatchNorm.java b/api/src/main/java/ai/djl/nn/norm/BatchNorm.java index 6795b9da04d..2e064290c6d 100644 --- a/api/src/main/java/ai/djl/nn/norm/BatchNorm.java +++ b/api/src/main/java/ai/djl/nn/norm/BatchNorm.java @@ -273,6 +273,51 @@ public static NDList batchNorm( input, runningMean, runningVar, gamma, beta, axis, momentum, eps, training); } + /** + * Returns axis the axis in which channel is specified. + * + * @return axis the axis in which channel is specified + */ + public int getAxis() { + return axis; + } + + /** + * Returns the epsilon value to prevent division by 0. + * + * @return the epsilon value to prevent division by 0 + */ + public float getEpsilon() { + return epsilon; + } + + /** + * Returns the momentum for moving average. + * + * @return the momentum for moving average + */ + public float getMomentum() { + return momentum; + } + + /** + * Returns offset of `beta` to add to normalized tensor. + * + * @return offset of `beta` to add to normalized tensor + */ + public boolean getCenter() { + return center; + } + + /** + * Whether multiply result by `gamma`. + * + * @return whether multiply result by `gamma`. + */ + public boolean getScale() { + return scale; + } + /** * Creates a builder to build a {@code BatchNorm}. * diff --git a/examples/src/test/java/ai/djl/examples/training/DescribeTest.java b/examples/src/test/java/ai/djl/examples/training/DescribeTest.java new file mode 100644 index 00000000000..aabf5163703 --- /dev/null +++ b/examples/src/test/java/ai/djl/examples/training/DescribeTest.java @@ -0,0 +1,197 @@ +/* + * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.examples.training; + +import ai.djl.Model; +import ai.djl.basicdataset.cv.classification.Mnist; +import ai.djl.basicmodelzoo.basic.Mlp; +import ai.djl.basicmodelzoo.cv.classification.ResNetV2; +import ai.djl.ndarray.types.Shape; +import ai.djl.nn.Block; +import ai.djl.nn.Blocks; +import ai.djl.nn.Parameter; +import ai.djl.training.DefaultTrainingConfig; +import ai.djl.training.Trainer; +import ai.djl.training.TrainingConfig; +import ai.djl.training.initializer.Initializer; +import ai.djl.training.loss.Loss; +import ai.djl.util.Utils; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +public class DescribeTest { + + @Test + public void testDescribe() throws IOException { + TrainingConfig config = new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss()); + + // Construct neural network + Block block = + new Mlp( + Mnist.IMAGE_HEIGHT * Mnist.IMAGE_WIDTH, + Mnist.NUM_CLASSES, + new int[] {128, 64}); + + try (Model model = Model.newInstance("mlp")) { + model.setBlock(block); + + try (Trainer trainer = model.newTrainer(config)) { + Path path = Paths.get("src/test/resources/describe/Mnist.txt"); + + try (InputStream is = Files.newInputStream(path)) { + List expected = Utils.readLines(is); + List actual = + Arrays.asList( + Blocks.describe( + trainer.getModel().getBlock(), + "SequentialBlock", + 0) + .split("\\R")); + Assert.assertEquals(actual, expected); + } + } + } + } + + @Test + public void testDescribeInitialized() throws IOException { + TrainingConfig config = new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss()); + + // Construct neural network + Block block = + new Mlp( + Mnist.IMAGE_HEIGHT * Mnist.IMAGE_WIDTH, + Mnist.NUM_CLASSES, + new int[] {128, 64}); + + try (Model model = Model.newInstance("mlp")) { + model.setBlock(block); + + try (Trainer trainer = model.newTrainer(config)) { + Shape inputShape = new Shape(1, Mnist.IMAGE_HEIGHT, Mnist.IMAGE_WIDTH); + trainer.initialize(inputShape); + + Path path = Paths.get("src/test/resources/describe/MnistInitialized.txt"); + + try (InputStream is = Files.newInputStream(path)) { + List expected = Utils.readLines(is); + List actual = + Arrays.asList( + Blocks.describe( + trainer.getModel().getBlock(), + "SequentialBlock", + 0) + .split("\\R")); + Assert.assertEquals(actual, expected); + } + } + } + } + + @Test + public void testKerasSequentialApi() throws IOException { + TrainingConfig config = new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss()); + + // Construct neural network + Block block = + new Mlp( + Mnist.IMAGE_HEIGHT * Mnist.IMAGE_WIDTH, + Mnist.NUM_CLASSES, + new int[] {128, 64}); + + try (Model model = Model.newInstance("mlp")) { + model.setBlock(block); + + try (Trainer trainer = model.newTrainer(config)) { + Path path = Paths.get("src/test/resources/describe/MnistSequential.py"); + + try (InputStream is = Files.newInputStream(path)) { + List expected = Utils.readLines(is); + List actual = + Arrays.asList(Blocks.describeAsTensorflow(trainer, false).split("\\R")); + Assert.assertEquals(actual, expected); + } + } + } + } + + @Test + public void testKerasFunctionalApi() throws IOException { + TrainingConfig config = new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss()); + + // Construct neural network + Block block = + new Mlp( + Mnist.IMAGE_HEIGHT * Mnist.IMAGE_WIDTH, + Mnist.NUM_CLASSES, + new int[] {128, 64}); + + try (Model model = Model.newInstance("mlp")) { + model.setBlock(block); + + try (Trainer trainer = model.newTrainer(config)) { + Shape inputShape = new Shape(1, Mnist.IMAGE_HEIGHT, Mnist.IMAGE_WIDTH); + trainer.initialize(inputShape); + + Path path = Paths.get("src/test/resources/describe/MnistFunctional.py"); + + try (InputStream is = Files.newInputStream(path)) { + List expected = Utils.readLines(is); + List actual = + Arrays.asList(Blocks.describeAsTensorflow(trainer, true).split("\\R")); + Assert.assertEquals(actual, expected); + } + } + } + } + + @Test + public void testKerasFunctionalApiForResnet() throws IOException { + TrainingConfig config = + new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss()) + .optInitializer(Initializer.ONES, Parameter.Type.WEIGHT); + + Block resNet50 = + ResNetV2.builder() + .setImageShape(new Shape(3, 32, 32)) + .setNumLayers(50) + .setOutSize(10) + .build(); + try (Model model = Model.newInstance("resnet")) { + model.setBlock(resNet50); + try (Trainer trainer = model.newTrainer(config)) { + int batchSize = 1; + Shape inputShape = new Shape(batchSize, 3, 32, 32); + trainer.initialize(inputShape); + + Path path = Paths.get("src/test/resources/describe/Resnet50.py"); + + try (InputStream is = Files.newInputStream(path)) { + List expected = Utils.readLines(is); + List actual = + Arrays.asList(Blocks.describeAsTensorflow(trainer, true).split("\\R")); + Assert.assertEquals(actual, expected); + } + } + } + } +} diff --git a/examples/src/test/resources/describe/Mnist.txt b/examples/src/test/resources/describe/Mnist.txt new file mode 100644 index 00000000000..81f9db3a801 --- /dev/null +++ b/examples/src/test/resources/describe/Mnist.txt @@ -0,0 +1,8 @@ +SequentialBlock { + batchFlatten + Linear + ReLU + Linear + ReLU + Linear +} \ No newline at end of file diff --git a/examples/src/test/resources/describe/MnistFunctional.py b/examples/src/test/resources/describe/MnistFunctional.py new file mode 100644 index 00000000000..186dc020960 --- /dev/null +++ b/examples/src/test/resources/describe/MnistFunctional.py @@ -0,0 +1,11 @@ +inputs = tf.keras.layers.InputLayer(input_shape = (28, 28)).output +LambdaBlock01 = tf.keras.layers.Flatten(name='LambdaBlock01')(inputs) +Linear02 = tf.keras.layers.Dense(128, name='Linear02')(LambdaBlock01) +LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock03')(Linear02) +Linear04 = tf.keras.layers.Dense(64, name='Linear04')(LambdaBlock03) +LambdaBlock05 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock05')(Linear04) +Linear06 = tf.keras.layers.Dense(10, name='Linear06')(LambdaBlock05) +outputs = Linear06 +model = tf.keras.Model(inputs=inputs, outputs=outputs) + +loss = tf.keras.losses.categorical_crossentropy diff --git a/examples/src/test/resources/describe/MnistInitialized.txt b/examples/src/test/resources/describe/MnistInitialized.txt new file mode 100644 index 00000000000..585ec5136ce --- /dev/null +++ b/examples/src/test/resources/describe/MnistInitialized.txt @@ -0,0 +1,8 @@ +SequentialBlock(1, 28, 28) { + batchFlatten(1, 28, 28) -> (1, 784) + Linear(1, 784) -> (1, 128) + ReLU(1, 64) -> (1, 64) + Linear(1, 128) -> (1, 64) + ReLU(1, 64) -> (1, 64) + Linear(1, 64) -> (1, 10) +} -> (1, 10) \ No newline at end of file diff --git a/examples/src/test/resources/describe/MnistSequential.py b/examples/src/test/resources/describe/MnistSequential.py new file mode 100644 index 00000000000..471e181f704 --- /dev/null +++ b/examples/src/test/resources/describe/MnistSequential.py @@ -0,0 +1,10 @@ +model = tf.keras.models.Sequential([ + tf.keras.layers.Flatten(name='LambdaBlock01'), + tf.keras.layers.Dense(128, name='Linear02'), + tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock03'), + tf.keras.layers.Dense(64, name='Linear04'), + tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock05'), + tf.keras.layers.Dense(10, name='Linear06') +], name='outputs') + +loss = tf.keras.losses.categorical_crossentropy diff --git a/examples/src/test/resources/describe/Resnet50.py b/examples/src/test/resources/describe/Resnet50.py new file mode 100644 index 00000000000..2838c3f2c9a --- /dev/null +++ b/examples/src/test/resources/describe/Resnet50.py @@ -0,0 +1,222 @@ +inputs = tf.keras.layers.InputLayer(input_shape = (3, 32, 32)).output +Conv2d01 = tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Conv2d01')(inputs) +Add02_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add02_SequentialBlock01_Conv2d01')(Conv2d01) +Add02_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add02_SequentialBlock01_BatchNorm02')(Add02_SequentialBlock01_Conv2d01) +Add02_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add02_SequentialBlock01_LambdaBlock03')(Add02_SequentialBlock01_BatchNorm02) +Add02_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add02_SequentialBlock01_Conv2d04')(Add02_SequentialBlock01_LambdaBlock03) +Add02_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add02_SequentialBlock01_BatchNorm05')(Add02_SequentialBlock01_Conv2d04) +Add02_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add02_SequentialBlock01_LambdaBlock06')(Add02_SequentialBlock01_BatchNorm05) +Add02_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add02_SequentialBlock01_Conv2d07')(Add02_SequentialBlock01_LambdaBlock06) +Add02_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add02_SequentialBlock01_BatchNorm08')(Add02_SequentialBlock01_Conv2d07) +Add02_SequentialBlock01 = Add02_SequentialBlock01_BatchNorm08 +Add02_SequentialBlock02_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add02_SequentialBlock02_Conv2d01')(Conv2d01) +Add02_SequentialBlock02_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add02_SequentialBlock02_BatchNorm02')(Add02_SequentialBlock02_Conv2d01) +Add02_SequentialBlock02 = Add02_SequentialBlock02_BatchNorm02 +Add02 = tf.keras.layers.Add(name='Add02')([Add02_SequentialBlock01, Add02_SequentialBlock02]) +LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock03')(Add02) +Add04_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add04_SequentialBlock01_Conv2d01')(LambdaBlock03) +Add04_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add04_SequentialBlock01_BatchNorm02')(Add04_SequentialBlock01_Conv2d01) +Add04_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add04_SequentialBlock01_LambdaBlock03')(Add04_SequentialBlock01_BatchNorm02) +Add04_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add04_SequentialBlock01_Conv2d04')(Add04_SequentialBlock01_LambdaBlock03) +Add04_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add04_SequentialBlock01_BatchNorm05')(Add04_SequentialBlock01_Conv2d04) +Add04_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add04_SequentialBlock01_LambdaBlock06')(Add04_SequentialBlock01_BatchNorm05) +Add04_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add04_SequentialBlock01_Conv2d07')(Add04_SequentialBlock01_LambdaBlock06) +Add04_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add04_SequentialBlock01_BatchNorm08')(Add04_SequentialBlock01_Conv2d07) +Add04_SequentialBlock01 = Add04_SequentialBlock01_BatchNorm08 +Add04_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add04_SequentialBlock02_LambdaBlock01')(LambdaBlock03) +Add04_SequentialBlock02 = Add04_SequentialBlock02_LambdaBlock01 +Add04 = tf.keras.layers.Add(name='Add04')([Add04_SequentialBlock01, Add04_SequentialBlock02]) +LambdaBlock05 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock05')(Add04) +Add06_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=64, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add06_SequentialBlock01_Conv2d01')(LambdaBlock05) +Add06_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add06_SequentialBlock01_BatchNorm02')(Add06_SequentialBlock01_Conv2d01) +Add06_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add06_SequentialBlock01_LambdaBlock03')(Add06_SequentialBlock01_BatchNorm02) +Add06_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add06_SequentialBlock01_Conv2d04')(Add06_SequentialBlock01_LambdaBlock03) +Add06_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add06_SequentialBlock01_BatchNorm05')(Add06_SequentialBlock01_Conv2d04) +Add06_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add06_SequentialBlock01_LambdaBlock06')(Add06_SequentialBlock01_BatchNorm05) +Add06_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add06_SequentialBlock01_Conv2d07')(Add06_SequentialBlock01_LambdaBlock06) +Add06_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add06_SequentialBlock01_BatchNorm08')(Add06_SequentialBlock01_Conv2d07) +Add06_SequentialBlock01 = Add06_SequentialBlock01_BatchNorm08 +Add06_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add06_SequentialBlock02_LambdaBlock01')(LambdaBlock05) +Add06_SequentialBlock02 = Add06_SequentialBlock02_LambdaBlock01 +Add06 = tf.keras.layers.Add(name='Add06')([Add06_SequentialBlock01, Add06_SequentialBlock02]) +LambdaBlock07 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock07')(Add06) +Add08_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=128, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add08_SequentialBlock01_Conv2d01')(LambdaBlock07) +Add08_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add08_SequentialBlock01_BatchNorm02')(Add08_SequentialBlock01_Conv2d01) +Add08_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add08_SequentialBlock01_LambdaBlock03')(Add08_SequentialBlock01_BatchNorm02) +Add08_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add08_SequentialBlock01_Conv2d04')(Add08_SequentialBlock01_LambdaBlock03) +Add08_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add08_SequentialBlock01_BatchNorm05')(Add08_SequentialBlock01_Conv2d04) +Add08_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add08_SequentialBlock01_LambdaBlock06')(Add08_SequentialBlock01_BatchNorm05) +Add08_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add08_SequentialBlock01_Conv2d07')(Add08_SequentialBlock01_LambdaBlock06) +Add08_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add08_SequentialBlock01_BatchNorm08')(Add08_SequentialBlock01_Conv2d07) +Add08_SequentialBlock01 = Add08_SequentialBlock01_BatchNorm08 +Add08_SequentialBlock02_Conv2d01 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add08_SequentialBlock02_Conv2d01')(LambdaBlock07) +Add08_SequentialBlock02_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add08_SequentialBlock02_BatchNorm02')(Add08_SequentialBlock02_Conv2d01) +Add08_SequentialBlock02 = Add08_SequentialBlock02_BatchNorm02 +Add08 = tf.keras.layers.Add(name='Add08')([Add08_SequentialBlock01, Add08_SequentialBlock02]) +LambdaBlock09 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock09')(Add08) +Add10_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=128, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add10_SequentialBlock01_Conv2d01')(LambdaBlock09) +Add10_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add10_SequentialBlock01_BatchNorm02')(Add10_SequentialBlock01_Conv2d01) +Add10_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add10_SequentialBlock01_LambdaBlock03')(Add10_SequentialBlock01_BatchNorm02) +Add10_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add10_SequentialBlock01_Conv2d04')(Add10_SequentialBlock01_LambdaBlock03) +Add10_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add10_SequentialBlock01_BatchNorm05')(Add10_SequentialBlock01_Conv2d04) +Add10_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add10_SequentialBlock01_LambdaBlock06')(Add10_SequentialBlock01_BatchNorm05) +Add10_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add10_SequentialBlock01_Conv2d07')(Add10_SequentialBlock01_LambdaBlock06) +Add10_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add10_SequentialBlock01_BatchNorm08')(Add10_SequentialBlock01_Conv2d07) +Add10_SequentialBlock01 = Add10_SequentialBlock01_BatchNorm08 +Add10_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add10_SequentialBlock02_LambdaBlock01')(LambdaBlock09) +Add10_SequentialBlock02 = Add10_SequentialBlock02_LambdaBlock01 +Add10 = tf.keras.layers.Add(name='Add10')([Add10_SequentialBlock01, Add10_SequentialBlock02]) +LambdaBlock11 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock11')(Add10) +Add12_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=128, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add12_SequentialBlock01_Conv2d01')(LambdaBlock11) +Add12_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add12_SequentialBlock01_BatchNorm02')(Add12_SequentialBlock01_Conv2d01) +Add12_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add12_SequentialBlock01_LambdaBlock03')(Add12_SequentialBlock01_BatchNorm02) +Add12_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add12_SequentialBlock01_Conv2d04')(Add12_SequentialBlock01_LambdaBlock03) +Add12_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add12_SequentialBlock01_BatchNorm05')(Add12_SequentialBlock01_Conv2d04) +Add12_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add12_SequentialBlock01_LambdaBlock06')(Add12_SequentialBlock01_BatchNorm05) +Add12_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add12_SequentialBlock01_Conv2d07')(Add12_SequentialBlock01_LambdaBlock06) +Add12_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add12_SequentialBlock01_BatchNorm08')(Add12_SequentialBlock01_Conv2d07) +Add12_SequentialBlock01 = Add12_SequentialBlock01_BatchNorm08 +Add12_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add12_SequentialBlock02_LambdaBlock01')(LambdaBlock11) +Add12_SequentialBlock02 = Add12_SequentialBlock02_LambdaBlock01 +Add12 = tf.keras.layers.Add(name='Add12')([Add12_SequentialBlock01, Add12_SequentialBlock02]) +LambdaBlock13 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock13')(Add12) +Add14_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=128, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add14_SequentialBlock01_Conv2d01')(LambdaBlock13) +Add14_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add14_SequentialBlock01_BatchNorm02')(Add14_SequentialBlock01_Conv2d01) +Add14_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add14_SequentialBlock01_LambdaBlock03')(Add14_SequentialBlock01_BatchNorm02) +Add14_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add14_SequentialBlock01_Conv2d04')(Add14_SequentialBlock01_LambdaBlock03) +Add14_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add14_SequentialBlock01_BatchNorm05')(Add14_SequentialBlock01_Conv2d04) +Add14_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add14_SequentialBlock01_LambdaBlock06')(Add14_SequentialBlock01_BatchNorm05) +Add14_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add14_SequentialBlock01_Conv2d07')(Add14_SequentialBlock01_LambdaBlock06) +Add14_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add14_SequentialBlock01_BatchNorm08')(Add14_SequentialBlock01_Conv2d07) +Add14_SequentialBlock01 = Add14_SequentialBlock01_BatchNorm08 +Add14_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add14_SequentialBlock02_LambdaBlock01')(LambdaBlock13) +Add14_SequentialBlock02 = Add14_SequentialBlock02_LambdaBlock01 +Add14 = tf.keras.layers.Add(name='Add14')([Add14_SequentialBlock01, Add14_SequentialBlock02]) +LambdaBlock15 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock15')(Add14) +Add16_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add16_SequentialBlock01_Conv2d01')(LambdaBlock15) +Add16_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add16_SequentialBlock01_BatchNorm02')(Add16_SequentialBlock01_Conv2d01) +Add16_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add16_SequentialBlock01_LambdaBlock03')(Add16_SequentialBlock01_BatchNorm02) +Add16_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add16_SequentialBlock01_Conv2d04')(Add16_SequentialBlock01_LambdaBlock03) +Add16_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add16_SequentialBlock01_BatchNorm05')(Add16_SequentialBlock01_Conv2d04) +Add16_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add16_SequentialBlock01_LambdaBlock06')(Add16_SequentialBlock01_BatchNorm05) +Add16_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add16_SequentialBlock01_Conv2d07')(Add16_SequentialBlock01_LambdaBlock06) +Add16_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add16_SequentialBlock01_BatchNorm08')(Add16_SequentialBlock01_Conv2d07) +Add16_SequentialBlock01 = Add16_SequentialBlock01_BatchNorm08 +Add16_SequentialBlock02_Conv2d01 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add16_SequentialBlock02_Conv2d01')(LambdaBlock15) +Add16_SequentialBlock02_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add16_SequentialBlock02_BatchNorm02')(Add16_SequentialBlock02_Conv2d01) +Add16_SequentialBlock02 = Add16_SequentialBlock02_BatchNorm02 +Add16 = tf.keras.layers.Add(name='Add16')([Add16_SequentialBlock01, Add16_SequentialBlock02]) +LambdaBlock17 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock17')(Add16) +Add18_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add18_SequentialBlock01_Conv2d01')(LambdaBlock17) +Add18_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add18_SequentialBlock01_BatchNorm02')(Add18_SequentialBlock01_Conv2d01) +Add18_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add18_SequentialBlock01_LambdaBlock03')(Add18_SequentialBlock01_BatchNorm02) +Add18_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add18_SequentialBlock01_Conv2d04')(Add18_SequentialBlock01_LambdaBlock03) +Add18_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add18_SequentialBlock01_BatchNorm05')(Add18_SequentialBlock01_Conv2d04) +Add18_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add18_SequentialBlock01_LambdaBlock06')(Add18_SequentialBlock01_BatchNorm05) +Add18_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add18_SequentialBlock01_Conv2d07')(Add18_SequentialBlock01_LambdaBlock06) +Add18_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add18_SequentialBlock01_BatchNorm08')(Add18_SequentialBlock01_Conv2d07) +Add18_SequentialBlock01 = Add18_SequentialBlock01_BatchNorm08 +Add18_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add18_SequentialBlock02_LambdaBlock01')(LambdaBlock17) +Add18_SequentialBlock02 = Add18_SequentialBlock02_LambdaBlock01 +Add18 = tf.keras.layers.Add(name='Add18')([Add18_SequentialBlock01, Add18_SequentialBlock02]) +LambdaBlock19 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock19')(Add18) +Add20_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add20_SequentialBlock01_Conv2d01')(LambdaBlock19) +Add20_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add20_SequentialBlock01_BatchNorm02')(Add20_SequentialBlock01_Conv2d01) +Add20_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add20_SequentialBlock01_LambdaBlock03')(Add20_SequentialBlock01_BatchNorm02) +Add20_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add20_SequentialBlock01_Conv2d04')(Add20_SequentialBlock01_LambdaBlock03) +Add20_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add20_SequentialBlock01_BatchNorm05')(Add20_SequentialBlock01_Conv2d04) +Add20_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add20_SequentialBlock01_LambdaBlock06')(Add20_SequentialBlock01_BatchNorm05) +Add20_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add20_SequentialBlock01_Conv2d07')(Add20_SequentialBlock01_LambdaBlock06) +Add20_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add20_SequentialBlock01_BatchNorm08')(Add20_SequentialBlock01_Conv2d07) +Add20_SequentialBlock01 = Add20_SequentialBlock01_BatchNorm08 +Add20_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add20_SequentialBlock02_LambdaBlock01')(LambdaBlock19) +Add20_SequentialBlock02 = Add20_SequentialBlock02_LambdaBlock01 +Add20 = tf.keras.layers.Add(name='Add20')([Add20_SequentialBlock01, Add20_SequentialBlock02]) +LambdaBlock21 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock21')(Add20) +Add22_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add22_SequentialBlock01_Conv2d01')(LambdaBlock21) +Add22_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add22_SequentialBlock01_BatchNorm02')(Add22_SequentialBlock01_Conv2d01) +Add22_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add22_SequentialBlock01_LambdaBlock03')(Add22_SequentialBlock01_BatchNorm02) +Add22_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add22_SequentialBlock01_Conv2d04')(Add22_SequentialBlock01_LambdaBlock03) +Add22_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add22_SequentialBlock01_BatchNorm05')(Add22_SequentialBlock01_Conv2d04) +Add22_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add22_SequentialBlock01_LambdaBlock06')(Add22_SequentialBlock01_BatchNorm05) +Add22_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add22_SequentialBlock01_Conv2d07')(Add22_SequentialBlock01_LambdaBlock06) +Add22_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add22_SequentialBlock01_BatchNorm08')(Add22_SequentialBlock01_Conv2d07) +Add22_SequentialBlock01 = Add22_SequentialBlock01_BatchNorm08 +Add22_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add22_SequentialBlock02_LambdaBlock01')(LambdaBlock21) +Add22_SequentialBlock02 = Add22_SequentialBlock02_LambdaBlock01 +Add22 = tf.keras.layers.Add(name='Add22')([Add22_SequentialBlock01, Add22_SequentialBlock02]) +LambdaBlock23 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock23')(Add22) +Add24_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add24_SequentialBlock01_Conv2d01')(LambdaBlock23) +Add24_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add24_SequentialBlock01_BatchNorm02')(Add24_SequentialBlock01_Conv2d01) +Add24_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add24_SequentialBlock01_LambdaBlock03')(Add24_SequentialBlock01_BatchNorm02) +Add24_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add24_SequentialBlock01_Conv2d04')(Add24_SequentialBlock01_LambdaBlock03) +Add24_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add24_SequentialBlock01_BatchNorm05')(Add24_SequentialBlock01_Conv2d04) +Add24_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add24_SequentialBlock01_LambdaBlock06')(Add24_SequentialBlock01_BatchNorm05) +Add24_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add24_SequentialBlock01_Conv2d07')(Add24_SequentialBlock01_LambdaBlock06) +Add24_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add24_SequentialBlock01_BatchNorm08')(Add24_SequentialBlock01_Conv2d07) +Add24_SequentialBlock01 = Add24_SequentialBlock01_BatchNorm08 +Add24_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add24_SequentialBlock02_LambdaBlock01')(LambdaBlock23) +Add24_SequentialBlock02 = Add24_SequentialBlock02_LambdaBlock01 +Add24 = tf.keras.layers.Add(name='Add24')([Add24_SequentialBlock01, Add24_SequentialBlock02]) +LambdaBlock25 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock25')(Add24) +Add26_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=256, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add26_SequentialBlock01_Conv2d01')(LambdaBlock25) +Add26_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add26_SequentialBlock01_BatchNorm02')(Add26_SequentialBlock01_Conv2d01) +Add26_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add26_SequentialBlock01_LambdaBlock03')(Add26_SequentialBlock01_BatchNorm02) +Add26_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add26_SequentialBlock01_Conv2d04')(Add26_SequentialBlock01_LambdaBlock03) +Add26_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add26_SequentialBlock01_BatchNorm05')(Add26_SequentialBlock01_Conv2d04) +Add26_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add26_SequentialBlock01_LambdaBlock06')(Add26_SequentialBlock01_BatchNorm05) +Add26_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=1024, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add26_SequentialBlock01_Conv2d07')(Add26_SequentialBlock01_LambdaBlock06) +Add26_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add26_SequentialBlock01_BatchNorm08')(Add26_SequentialBlock01_Conv2d07) +Add26_SequentialBlock01 = Add26_SequentialBlock01_BatchNorm08 +Add26_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add26_SequentialBlock02_LambdaBlock01')(LambdaBlock25) +Add26_SequentialBlock02 = Add26_SequentialBlock02_LambdaBlock01 +Add26 = tf.keras.layers.Add(name='Add26')([Add26_SequentialBlock01, Add26_SequentialBlock02]) +LambdaBlock27 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock27')(Add26) +Add28_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add28_SequentialBlock01_Conv2d01')(LambdaBlock27) +Add28_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add28_SequentialBlock01_BatchNorm02')(Add28_SequentialBlock01_Conv2d01) +Add28_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add28_SequentialBlock01_LambdaBlock03')(Add28_SequentialBlock01_BatchNorm02) +Add28_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add28_SequentialBlock01_Conv2d04')(Add28_SequentialBlock01_LambdaBlock03) +Add28_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add28_SequentialBlock01_BatchNorm05')(Add28_SequentialBlock01_Conv2d04) +Add28_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add28_SequentialBlock01_LambdaBlock06')(Add28_SequentialBlock01_BatchNorm05) +Add28_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add28_SequentialBlock01_Conv2d07')(Add28_SequentialBlock01_LambdaBlock06) +Add28_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add28_SequentialBlock01_BatchNorm08')(Add28_SequentialBlock01_Conv2d07) +Add28_SequentialBlock01 = Add28_SequentialBlock01_BatchNorm08 +Add28_SequentialBlock02_Conv2d01 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(1, 1), strides=(2, 2), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add28_SequentialBlock02_Conv2d01')(LambdaBlock27) +Add28_SequentialBlock02_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add28_SequentialBlock02_BatchNorm02')(Add28_SequentialBlock02_Conv2d01) +Add28_SequentialBlock02 = Add28_SequentialBlock02_BatchNorm02 +Add28 = tf.keras.layers.Add(name='Add28')([Add28_SequentialBlock01, Add28_SequentialBlock02]) +LambdaBlock29 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock29')(Add28) +Add30_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add30_SequentialBlock01_Conv2d01')(LambdaBlock29) +Add30_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add30_SequentialBlock01_BatchNorm02')(Add30_SequentialBlock01_Conv2d01) +Add30_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add30_SequentialBlock01_LambdaBlock03')(Add30_SequentialBlock01_BatchNorm02) +Add30_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add30_SequentialBlock01_Conv2d04')(Add30_SequentialBlock01_LambdaBlock03) +Add30_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add30_SequentialBlock01_BatchNorm05')(Add30_SequentialBlock01_Conv2d04) +Add30_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add30_SequentialBlock01_LambdaBlock06')(Add30_SequentialBlock01_BatchNorm05) +Add30_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add30_SequentialBlock01_Conv2d07')(Add30_SequentialBlock01_LambdaBlock06) +Add30_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add30_SequentialBlock01_BatchNorm08')(Add30_SequentialBlock01_Conv2d07) +Add30_SequentialBlock01 = Add30_SequentialBlock01_BatchNorm08 +Add30_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add30_SequentialBlock02_LambdaBlock01')(LambdaBlock29) +Add30_SequentialBlock02 = Add30_SequentialBlock02_LambdaBlock01 +Add30 = tf.keras.layers.Add(name='Add30')([Add30_SequentialBlock01, Add30_SequentialBlock02]) +LambdaBlock31 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock31')(Add30) +Add32_SequentialBlock01_Conv2d01 = tf.keras.layers.Conv2D(filters=512, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add32_SequentialBlock01_Conv2d01')(LambdaBlock31) +Add32_SequentialBlock01_BatchNorm02 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add32_SequentialBlock01_BatchNorm02')(Add32_SequentialBlock01_Conv2d01) +Add32_SequentialBlock01_LambdaBlock03 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add32_SequentialBlock01_LambdaBlock03')(Add32_SequentialBlock01_BatchNorm02) +Add32_SequentialBlock01_Conv2d04 = tf.keras.layers.Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding='SAME', dilation_rate=(1, 1), data_format='channels_first', use_bias=False, name='Add32_SequentialBlock01_Conv2d04')(Add32_SequentialBlock01_LambdaBlock03) +Add32_SequentialBlock01_BatchNorm05 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=2.0E-5, momentum=0.9, axis=1, name='Add32_SequentialBlock01_BatchNorm05')(Add32_SequentialBlock01_Conv2d04) +Add32_SequentialBlock01_LambdaBlock06 = tf.keras.layers.Activation(tf.keras.activations.relu, name='Add32_SequentialBlock01_LambdaBlock06')(Add32_SequentialBlock01_BatchNorm05) +Add32_SequentialBlock01_Conv2d07 = tf.keras.layers.Conv2D(filters=2048, kernel_size=(1, 1), strides=(1, 1), padding='VALID', dilation_rate=(1, 1), data_format='channels_first', use_bias=True, name='Add32_SequentialBlock01_Conv2d07')(Add32_SequentialBlock01_LambdaBlock06) +Add32_SequentialBlock01_BatchNorm08 = tf.keras.layers.BatchNormalization(scale=True, center=True, epsilon=1.0E-5, momentum=0.9, axis=1, name='Add32_SequentialBlock01_BatchNorm08')(Add32_SequentialBlock01_Conv2d07) +Add32_SequentialBlock01 = Add32_SequentialBlock01_BatchNorm08 +Add32_SequentialBlock02_LambdaBlock01 = tf.keras.layers.Identity(name='Add32_SequentialBlock02_LambdaBlock01')(LambdaBlock31) +Add32_SequentialBlock02 = Add32_SequentialBlock02_LambdaBlock01 +Add32 = tf.keras.layers.Add(name='Add32')([Add32_SequentialBlock01, Add32_SequentialBlock02]) +LambdaBlock33 = tf.keras.layers.Activation(tf.keras.activations.relu, name='LambdaBlock33')(Add32) +LambdaBlock34 = tf.keras.layers.GlobalAveragePooling2D(data_format='channels_first', name='LambdaBlock34')(LambdaBlock33) +LambdaBlock35 = tf.keras.layers.Flatten(name='LambdaBlock35')(LambdaBlock34) +Linear36 = tf.keras.layers.Dense(10, name='Linear36')(LambdaBlock35) +LambdaBlock37 = tf.keras.layers.Flatten(name='LambdaBlock37')(Linear36) +outputs = LambdaBlock37 +model = tf.keras.Model(inputs=inputs, outputs=outputs) + +loss = tf.keras.losses.categorical_crossentropy \ No newline at end of file diff --git a/integration/src/main/java/ai/djl/integration/tests/training/EvaluateDatasetTest.java b/integration/src/main/java/ai/djl/integration/tests/training/EvaluateDatasetTest.java index 22adddbfb90..1d42b32ce11 100644 --- a/integration/src/main/java/ai/djl/integration/tests/training/EvaluateDatasetTest.java +++ b/integration/src/main/java/ai/djl/integration/tests/training/EvaluateDatasetTest.java @@ -50,7 +50,7 @@ public void testDatasetEvaluation() throws IOException, TranslateException { testMnistDataset.prepare(); - Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation::relu); + Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation.reluBlock()); try (Model model = Model.newInstance("lin-reg", TestUtils.getEngine())) { model.setBlock(mlpModel); diff --git a/integration/src/main/java/ai/djl/integration/tests/training/listener/EarlyStoppingListenerTest.java b/integration/src/main/java/ai/djl/integration/tests/training/listener/EarlyStoppingListenerTest.java index 9aee2661411..dcdb87209b6 100644 --- a/integration/src/main/java/ai/djl/integration/tests/training/listener/EarlyStoppingListenerTest.java +++ b/integration/src/main/java/ai/djl/integration/tests/training/listener/EarlyStoppingListenerTest.java @@ -78,7 +78,7 @@ public void closeResources() { @Test public void testEarlyStoppingStopsOnEpoch2() throws Exception { - Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation::relu); + Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation.reluBlock()); try (Model model = Model.newInstance("lin-reg", TestUtils.getEngine())) { model.setBlock(mlpModel); @@ -117,7 +117,7 @@ public void testEarlyStoppingStopsOnEpoch2() throws Exception { @Test public void testEarlyStoppingStopsOnEpoch3AsMinEpochsIs3() throws Exception { - Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation::relu); + Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation.reluBlock()); try (Model model = Model.newInstance("lin-reg", TestUtils.getEngine())) { model.setBlock(mlpModel); @@ -156,7 +156,7 @@ public void testEarlyStoppingStopsOnEpoch3AsMinEpochsIs3() throws Exception { @Test public void testEarlyStoppingStopsOnEpoch1AsMaxDurationIs1ms() throws Exception { - Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation::relu); + Mlp mlpModel = new Mlp(784, 1, new int[] {256}, Activation.reluBlock()); try (Model model = Model.newInstance("lin-reg", TestUtils.getEngine())) { model.setBlock(mlpModel); diff --git a/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java b/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java index 2869e42f55d..51d0189be02 100644 --- a/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java +++ b/model-zoo/src/main/java/ai/djl/basicmodelzoo/basic/Mlp.java @@ -12,14 +12,12 @@ */ package ai.djl.basicmodelzoo.basic; -import ai.djl.ndarray.NDList; import ai.djl.nn.Activation; +import ai.djl.nn.Block; import ai.djl.nn.Blocks; import ai.djl.nn.SequentialBlock; import ai.djl.nn.core.Linear; -import java.util.function.Function; - /** * Multilayer Perceptron (MLP) NeuralNetworks. * @@ -45,7 +43,7 @@ public class Mlp extends SequentialBlock { * @param hidden the sizes of all of the hidden layers */ public Mlp(int input, int output, int[] hidden) { - this(input, output, hidden, Activation::relu); + this(input, output, hidden, Activation.reluBlock()); } /** @@ -57,7 +55,7 @@ public Mlp(int input, int output, int[] hidden) { * @param activation the activation function to use */ @SuppressWarnings("this-escape") - public Mlp(int input, int output, int[] hidden, Function activation) { + public Mlp(int input, int output, int[] hidden, Block activation) { add(Blocks.batchFlattenBlock(input)); for (int hiddenSize : hidden) { add(Linear.builder().setUnits(hiddenSize).build()); diff --git a/model-zoo/src/main/java/ai/djl/basicmodelzoo/cv/classification/ResNetV2.java b/model-zoo/src/main/java/ai/djl/basicmodelzoo/cv/classification/ResNetV2.java new file mode 100644 index 00000000000..19719dba0c7 --- /dev/null +++ b/model-zoo/src/main/java/ai/djl/basicmodelzoo/cv/classification/ResNetV2.java @@ -0,0 +1,347 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +package ai.djl.basicmodelzoo.cv.classification; + +import ai.djl.ndarray.types.Shape; +import ai.djl.nn.Activation; +import ai.djl.nn.Block; +import ai.djl.nn.Blocks; +import ai.djl.nn.SequentialBlock; +import ai.djl.nn.convolutional.Conv2d; +import ai.djl.nn.core.Add; +import ai.djl.nn.core.Linear; +import ai.djl.nn.norm.BatchNorm; +import ai.djl.nn.pooling.Pool; + +import java.util.Arrays; +import java.util.List; + +/** + * {@code ResNetV2} is a variant of {@code ResNetV1} but using no lambda blocks so the DJL model is + * suitable as input to reflective algorithms. + */ +public final class ResNetV2 { + + private ResNetV2() {} + + /** + * Builds a {@link Block} that represents a residual unit used in the implementation of the + * Resnet model. + * + * @param numFilters the number of output channels + * @param stride the stride of the convolution in each dimension + * @param dimMatch whether the number of channels between input and output has to remain the + * same + * @param bottleneck whether to use bottleneck architecture + * @param batchNormMomentum the momentum to be used for {@link BatchNorm} + * @return a list of {@link Block} that as sequence represents a residual unit + */ + public static List residualUnit( + int numFilters, + final Shape stride, + final boolean dimMatch, + boolean bottleneck, + float batchNormMomentum) { + SequentialBlock resUnit = new SequentialBlock(); + if (bottleneck) { + resUnit.add( + Conv2d.builder() + .setKernelShape(new Shape(1, 1)) + .setFilters(numFilters / 4) + .optStride(stride) + .optPadding(new Shape(0, 0)) + .optBias(true) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(1e-5f) + .optMomentum(batchNormMomentum) + .build()) + .add(Activation.reluBlock()) + .add( + Conv2d.builder() + .setKernelShape(new Shape(3, 3)) + .setFilters(numFilters / 4) + .optStride(new Shape(1, 1)) + .optPadding(new Shape(1, 1)) + .optBias(false) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(2E-5f) + .optMomentum(batchNormMomentum) + .build()) + .add(Activation.reluBlock()) + .add( + Conv2d.builder() + .setKernelShape(new Shape(1, 1)) + .setFilters(numFilters) + .optStride(new Shape(1, 1)) + .optPadding(new Shape(0, 0)) + .optBias(true) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(1E-5f) + .optMomentum(batchNormMomentum) + .build()); + + } else { + resUnit.add( + Conv2d.builder() + .setKernelShape(new Shape(3, 3)) + .setFilters(numFilters) + .optStride(stride) + .optPadding(new Shape(1, 1)) + .optBias(false) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(1E-5f) + .optMomentum(batchNormMomentum) + .build()) + .add(Activation.reluBlock()) + .add( + Conv2d.builder() + .setKernelShape(new Shape(3, 3)) + .setFilters(numFilters) + .optStride(new Shape(1, 1)) + .optPadding(new Shape(1, 1)) + .optBias(false) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(1E-5f) + .optMomentum(batchNormMomentum) + .build()); + } + SequentialBlock shortcut = new SequentialBlock(); + if (dimMatch) { + shortcut.add(Blocks.identityBlock()); + } else { + shortcut.add( + Conv2d.builder() + .setKernelShape(new Shape(1, 1)) + .setFilters(numFilters) + .optStride(stride) + .optPadding(new Shape(0, 0)) + .optBias(false) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(1E-5f) + .optMomentum(batchNormMomentum) + .build()); + } + + Add add = new Add(Arrays.asList(resUnit, shortcut)); + return Arrays.asList(add, Activation.reluBlock()); + } + + /** + * Creates a new {@link Block} of {@code ResNetV2} with the arguments from the given {@link + * Builder}. + * + * @param builder the {@link Builder} with the necessary arguments + * @return a {@link Block} that represents the required ResNet model + */ + public static SequentialBlock resnet(Builder builder) { + int numStages = builder.units.length; + long height = builder.imageShape.get(1); + SequentialBlock resNet = new SequentialBlock(); + if (height <= 32) { + resNet.add( + Conv2d.builder() + .setKernelShape(new Shape(3, 3)) + .setFilters(builder.filters[0]) + .optStride(new Shape(1, 1)) + .optPadding(new Shape(1, 1)) + .optBias(false) + .build()); + } else { + resNet.add( + Conv2d.builder() + .setKernelShape(new Shape(7, 7)) + .setFilters(builder.filters[0]) + .optStride(new Shape(2, 2)) + .optPadding(new Shape(3, 3)) + .optBias(false) + .build()) + .add( + BatchNorm.builder() + .optEpsilon(2E-5f) + .optMomentum(builder.batchNormMomentum) + .build()) + .add(Activation.reluBlock()) + .add(Pool.maxPool2dBlock(new Shape(3, 3), new Shape(2, 2), new Shape(1, 1))); + } + Shape resStride = new Shape(1, 1); + for (int i = 0; i < numStages; i++) { + resNet.addAll( + residualUnit( + builder.filters[i + 1], + resStride, + false, + builder.bottleneck, + builder.batchNormMomentum)); + for (int j = 0; j < builder.units[i] - 1; j++) { + resNet.addAll( + residualUnit( + builder.filters[i + 1], + new Shape(1, 1), + true, + builder.bottleneck, + builder.batchNormMomentum)); + } + if (i == 0) { + resStride = new Shape(2, 2); + } + } + return resNet.add(Pool.globalAvgPool2dBlock()) + .add(Blocks.batchFlattenBlock()) + .add(Linear.builder().setUnits(builder.outSize).build()) + .add(Blocks.batchFlattenBlock()); + } + + /** + * Creates a builder to build a {@link ResNetV2}. + * + * @return a new builder + */ + public static Builder builder() { + return new Builder(); + } + + /** The Builder to construct a {@link ResNetV2} object. */ + public static final class Builder { + + int numLayers; + int numStages; + long outSize; + float batchNormMomentum = 0.9f; + Shape imageShape; + boolean bottleneck; + int[] units; + int[] filters; + + Builder() {} + + /** + * Sets the number of layers in the network. + * + * @param numLayers the number of layers + * @return this {@code Builder} + */ + public Builder setNumLayers(int numLayers) { + this.numLayers = numLayers; + return this; + } + + /** + * Sets the size of the output. + * + * @param outSize the output size + * @return this {@code Builder} + */ + public Builder setOutSize(long outSize) { + this.outSize = outSize; + return this; + } + + /** + * Sets the momentum of batchNorm layer. + * + * @param batchNormMomentum the momentum + * @return this {@code Builder} + */ + public Builder optBatchNormMomentum(float batchNormMomentum) { + this.batchNormMomentum = batchNormMomentum; + return this; + } + + /** + * Sets the shape of the image. + * + * @param imageShape the shape of the image + * @return this {@code Builder} + */ + public Builder setImageShape(Shape imageShape) { + this.imageShape = imageShape; + return this; + } + + /** + * Builds a {@link ResNetV2} block. + * + * @return the {@link ResNetV2} block + */ + public SequentialBlock build() { + if (imageShape == null) { + throw new IllegalArgumentException("Must set imageShape"); + } + long height = imageShape.get(1); + if (height <= 28) { + numStages = 3; + int perUnit; + if ((numLayers - 2) % 9 == 0 && numLayers >= 164) { + perUnit = (numLayers - 2) / 9; + filters = new int[] {16, 64, 128, 256}; + bottleneck = true; + } else if ((numLayers - 2) % 6 == 0 && numLayers < 164) { + perUnit = (numLayers - 2) / 6; + filters = new int[] {16, 16, 32, 64}; + bottleneck = false; + } else { + throw new IllegalArgumentException( + "no experiments done on num_layers " + + numLayers + + ", you can do it yourself"); + } + units = new int[numStages]; + for (int i = 0; i < numStages; i++) { + units[i] = perUnit; + } + } else { + numStages = 4; + if (numLayers >= 50) { + filters = new int[] {64, 256, 512, 1024, 2048}; + bottleneck = true; + } else { + filters = new int[] {64, 64, 128, 256, 512}; + bottleneck = true; + } + if (numLayers == 18) { + units = new int[] {2, 2, 2, 2}; + } else if (numLayers == 34) { + units = new int[] {3, 4, 6, 3}; + } else if (numLayers == 50) { + units = new int[] {3, 4, 6, 3}; + } else if (numLayers == 101) { + units = new int[] {3, 4, 23, 3}; + } else if (numLayers == 152) { + units = new int[] {3, 8, 36, 3}; + } else if (numLayers == 200) { + units = new int[] {3, 24, 36, 3}; + } else if (numLayers == 269) { + units = new int[] {3, 30, 48, 8}; + } else { + throw new IllegalArgumentException( + "no experiments done on num_layers " + + numLayers + + ", you can do it yourself"); + } + } + return resnet(this); + } + } +}