exp: Add model of TensorFlow Keras API

This change adds a model of TensorFlow's Keras API to OpenDC.

12 files changed, 869 insertions, 0 deletions

diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt
new file mode 100644
index 00000000..9ef5b621
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras
+
+import org.opendc.experiments.tf20.keras.activations.Activation
+import org.opendc.experiments.tf20.keras.layer.conv.Conv2D
+import org.opendc.experiments.tf20.keras.layer.conv.ConvPadding
+import org.opendc.experiments.tf20.keras.layer.core.ActivationLayer
+import org.opendc.experiments.tf20.keras.layer.core.Input
+import org.opendc.experiments.tf20.keras.layer.pool.Pool2D
+import org.opendc.experiments.tf20.keras.layer.regularization.Dropout
+
+/**
+ * Construct an AlexNet model with the given batch size.
+ */
+fun AlexNet(batchSize: Long): TrainableModel {
+    return Sequential(
+        Input(batchSize, 227, 227, 3, name = "Input"),
+        Conv2D(longArrayOf(11, 11, 3, 96), longArrayOf(1, 4, 4, 1), padding = ConvPadding.VALID, name = "conv1"),
+        Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool1"),
+        Conv2D(longArrayOf(5, 5, 96, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2"),
+        Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool2"),
+        Conv2D(longArrayOf(3, 3, 256, 384), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3"),
+        Conv2D(longArrayOf(3, 3, 384, 384), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4"),
+        Conv2D(longArrayOf(3, 3, 384, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5"),
+        Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool5"),
+        Conv2D(longArrayOf(6, 6, 256, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.VALID, name = "fc6"),
+        Dropout(0.5f, name = "dropout6"),
+        Conv2D(longArrayOf(1, 1, 4096, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "fc7"),
+        Dropout(0.5f, name = "dropout7"),
+        Conv2D(longArrayOf(1, 1, 4096, 1000), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "f8"),
+        ActivationLayer(Activation.Softmax, name = "softmax")
+    )
+}
+
+/**
+ * Construct an VGG16 model with the given batch size.
+ */
+fun VGG16(batchSize: Long = 128): TrainableModel {
+    return Sequential(
+        Input(batchSize, 224, 224, 3, name = "Input"),
+        Conv2D(longArrayOf(3, 3, 3, 64), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv1-1"),
+        Conv2D(longArrayOf(3, 3, 64, 64), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv1-2"),
+        Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool1"),
+        Conv2D(longArrayOf(3, 3, 64, 128), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2-1"),
+        Conv2D(longArrayOf(3, 3, 128, 128), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2-2"),
+        Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool2"),
+        Conv2D(longArrayOf(3, 3, 128, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-1"),
+        Conv2D(longArrayOf(3, 3, 256, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-2"),
+        Conv2D(longArrayOf(3, 3, 256, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-3"),
+        Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool3"),
+        Conv2D(longArrayOf(3, 3, 256, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-1"),
+        Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-2"),
+        Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-3"),
+        Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool4"),
+        Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-1"),
+        Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-2"),
+        Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-3"),
+        Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool5"),
+        Conv2D(longArrayOf(7, 7, 512, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.VALID, name = "fc6"),
+        Dropout(0.5f, name = "dropout6"),
+        Conv2D(longArrayOf(1, 1, 4096, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "fc7"),
+        Dropout(0.5f, name = "dropout7"),
+        Conv2D(longArrayOf(1, 1, 4096, 1000), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "f8"),
+        ActivationLayer(Activation.Softmax, name = "softmax")
+    )
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt
new file mode 100644
index 00000000..3cc6b690
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras
+
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.layer.core.Input
+
+/**
+ * Sequential model groups a linear stack of layers into a TensorFlow Model.
+ *
+ * @param [layers] The layers to describe the model design.
+ */
+public class Sequential(vararg layers: Layer) : TrainableModel() {
+
+    /**
+     * The layers to describe the model design. Main part of the internal state of the model.
+     */
+    public val layers: List<Layer> = listOf(*layers)
+
+    /**
+     * First layer that is responsible for the input shape of the Neural Network.
+     */
+    public val inputLayer: Input
+        get() = layers[0] as Input
+
+    /**
+     * Returns input dimensions in order HWC (height, width, channels)
+     */
+    public val inputDimensions: LongArray
+        get() = (layers[0] as Input).packedDims
+
+    override fun close() {}
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt
new file mode 100644
index 00000000..421dec73
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras
+
+/**
+ * Base abstract class for all trainable models.
+ */
+public abstract class TrainableModel : AutoCloseable {
+
+    override fun toString(): String {
+        return "TrainableModel ${super.toString()}"
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt
new file mode 100644
index 00000000..403acfc0
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt
@@ -0,0 +1,198 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.activations
+
+/**
+ * Neural network hyper-parameter, activation function of a node defines the output of that node given an input or
+ * set of inputs.
+ */
+public enum class Activation {
+    /**
+     * Linear unit. Returns unmodified input.
+     *
+     * NOTE: Doing nothing useful. Returns to ancient times of linear perceptron.
+     */
+    Linear,
+
+    /**
+     * Sigmoid activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * sigmoid(x) = 1 / (1 + exp(-x))
+     * ```
+     *
+     * For small values (<-5), `sigmoid` returns a value close to zero, and for large values (>5)
+     * the result of the function gets close to 1.
+     *
+     * NOTE: Sigmoid is equivalent to a 2-element ActivationLayer, where the second element is
+     * assumed to be zero. The sigmoid function always returns a value between 0 and 1.
+     */
+    Sigmoid,
+
+    /**
+     * Hyperbolic tangent activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * tanh(x) = sinh(x)/cosh(x) = ((exp(x) - exp(-x))/(exp(x) + exp(-x)))
+     * ```
+     */
+    Tanh,
+
+    /**
+     * Rectified linear unit (ReLU).
+     *
+     * With default values, this returns the standard ReLU activation:
+     * `max(x, 0)`, the element-wise maximum of 0 and the input tensor.
+     */
+    Relu,
+
+    /**
+     * Computes Rectified Linear 6:
+     * ```
+     * min(max(features, 0), 6)
+     * ```
+     * @see <a href="http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf">
+     *     Convolutional Deep Belief Networks on CIFAR-10. A. Krizhevsky</a>
+     */
+    Relu6,
+
+    /**
+     * Exponential Linear Unit.
+     *
+     * The exponential linear unit (ELU) with `alpha > 0` is:
+     * `x` if `x > 0` and `alpha * (exp(x) - 1)` if `x < 0`
+     *
+     * For this implementations alpha is equal to 1.0.
+     *
+     * The ELU hyperparameter `alpha` controls the value to which an
+     * ELU saturates for negative net inputs. ELUs diminish the
+     * vanishing gradient effect.
+     *
+     * ELUs have negative values which pushes the mean of the activations closer to zero.
+     *
+     * Mean activations that are closer to zero enable faster learning as they
+     * bring the gradient closer to the natural gradient.
+     *
+     * ELUs saturate to a negative value when the argument gets smaller.
+     * Saturation means a small derivative which decreases the variation
+     * and the information that is propagated to the next layer.
+     *
+     * @see <a href="https://arxiv.org/abs/1511.07289">Fast and Accurate Deep Network Learning by Exponential Linear Units
+     * (ELUs) (Clevert et al, 2016)</a>
+     */
+    Elu,
+
+    /**
+     * Scaled Exponential Linear Unit (SELU).
+     *
+     * The Scaled Exponential Linear Unit (SELU) activation function is defined as:
+     * ```
+     *  if x > 0: return scale * x
+     *  if x < 0: return scale * alpha * (exp(x) - 1)
+     * ```
+     * where `alpha` and `scale` are pre-defined constants (`alpha=1.67326324` and `scale=1.05070098`).
+     *
+     * Basically, the SELU activation function multiplies `scale` (> 1) with the
+     * output of the `tf.keras.activations.elu` function to ensure a slope larger
+     * than one for positive inputs.
+     *
+     * @see <a href="https://arxiv.org/abs/1706.02515">Klambauer et al., 2017</a>
+     */
+    Selu,
+
+    /**
+     * ActivationLayer converts a real vector to a vector of categorical probabilities.
+     * The elements of the output vector are in range (0, 1) and sum to 1.
+     *
+     * ActivationLayer is often used as the activation for the last
+     * layer of a classification network because the result could be interpreted as
+     * a probability distribution.
+     */
+    Softmax,
+
+    /**
+     *
+     */
+    LogSoftmax,
+
+    /**
+     * Exponential activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * exp(x)
+     * ```
+     */
+    Exponential,
+
+    /**
+     * Softplus activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * softplus(x) = log(exp(x) + 1)
+     * ```
+     */
+    SoftPlus,
+
+    /***
+     * Softsign activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * softsign(x) = x / (abs(x) + 1)
+     * ```
+     */
+    SoftSign,
+
+    /**
+     * Hard sigmoid activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * if x < -2.5: return 0
+     * if x > 2.5: return 1
+     * if -2.5 <= x <= 2.5: return 0.2 * x + 0.5
+     * ```
+     * A faster approximation of the sigmoid activation.
+     */
+    HardSigmoid,
+
+    /**
+     * Swish activation function.
+     *
+     * Transforms input 'x' according formula:
+     * ```
+     * swish(x) = x * sigmoid(x)
+     * ```
+     *
+     * It is a smooth, non-monotonic function that consistently matches
+     * or outperforms ReLU on deep networks, it is unbounded above and
+     * bounded below.
+     *
+     * @see <a href="https://arxiv.org/abs/1710.05941">Ramachandran et al., 2017</a>
+     */
+    Swish;
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt
new file mode 100644
index 00000000..eafdf63b
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer
+
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+
+/**
+ * Abstract class from which all layers inherit.
+ *
+ * @param name The name of the layer.
+ */
+public abstract class Layer(public val name: String) {
+    /**
+     * Build the layer for the specified [inputShape].
+     *
+     * @param [inputShape] Input shape, result of [getOutputShape] call from previous layer.
+     */
+    public abstract fun build(inputShape: TensorShape)
+
+    /**
+     * Compute output shape of this layer, based on [inputShape] and [Layer] type.
+     */
+    public abstract fun getOutputShape(inputShape: TensorShape): TensorShape
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt
new file mode 100644
index 00000000..c8078bf8
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.conv
+
+import org.opendc.experiments.tf20.keras.activations.Activation
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+import kotlin.math.ceil
+
+/**
+ * 2D convolution layer (e.g. spatial convolution over images).
+ *
+ * This layer creates a convolution kernel that is convolved (actually cross-correlated)
+ * with the layer input to produce a tensor of outputs.
+ * Finally, if `activation` is applied to the outputs as well.
+ */
+public class Conv2D(
+    public val filter: LongArray = LongArray(4), // [H, W, channel_in, channel_out]
+    public val strides: LongArray = LongArray(4), // [1, stride_h, stride_w, 1]
+    public val activation: Activation = Activation.Relu,
+    public val padding: ConvPadding = ConvPadding.VALID,
+    name: String = "",
+) : Layer(name) {
+
+    private var padHeight: Double = 0.0
+    private var padWidth: Double = 0.0
+
+    override fun build(inputShape: TensorShape) {
+        TODO("not implemented")
+    }
+
+    override fun getOutputShape(inputShape: TensorShape): TensorShape {
+        check(filter[2] != inputShape[3]) { "Input channel ${filter[2]} and ${inputShape[3]} shall match" }
+
+        var outHeight = 0L
+        var outWidth = 0L
+
+        if (padding == ConvPadding.VALID) {
+            outHeight = ceil((inputShape[1] - filter[0] + 1).toDouble() / strides[1].toDouble()).toLong()
+            outWidth = ceil((inputShape[2] - filter[1] + 1).toDouble() / strides[2].toDouble()).toLong()
+            padHeight = 0.0
+            padWidth = 0.0
+        } else if (padding == ConvPadding.SAME) {
+            outHeight = ceil(inputShape[1].toFloat() / strides[1].toFloat()).toLong()
+            outWidth = ceil(inputShape[2].toFloat() / strides[2].toFloat()).toLong()
+
+            val padAlongHeight = (outHeight - 1) * strides[1] + filter[0] - inputShape[1]
+            val padAlongWidth = (outWidth - 1) * strides[2] + filter[1] - inputShape[2]
+
+            padHeight = (padAlongHeight / 2).toDouble()
+            padWidth = (padAlongWidth / 2).toDouble()
+        }
+
+        return TensorShape(inputShape[0], outHeight, outWidth, filter[3])
+    }
+
+    override fun toString(): String {
+        return "Conv2D[filter=${filter.contentToString()}, strides=${strides.contentToString()}, activation=$activation, padding=$padding]"
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt
new file mode 100644
index 00000000..03ae6282
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.conv
+
+/**
+ * Enumeration of convolution padding types.
+ */
+public enum class ConvPadding {
+    /**
+     * Pad evenly to the left/right or up/down of the input such that output has the same
+     * height/width dimension as the input.
+     */
+    SAME,
+
+    /**
+     * No padding.
+     */
+    VALID
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt
new file mode 100644
index 00000000..befcfe50
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.core
+
+import org.opendc.experiments.tf20.keras.activations.Activation
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+
+/**
+ * This layer applies an activation function to an output.
+ */
+public class ActivationLayer(
+    public val activation: Activation = Activation.Relu,
+    name: String = "",
+) : Layer(name) {
+
+    override fun build(inputShape: TensorShape) {
+        // Intentionally left empty
+    }
+
+    override fun getOutputShape(inputShape: TensorShape): TensorShape = inputShape
+
+    override fun toString(): String {
+        return "ActivationLayer[activation=$activation]"
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt
new file mode 100644
index 00000000..b56284d6
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.core
+
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+
+/**
+ * This layer is responsible for the input shape of the built model.
+ */
+public class Input(vararg dims: Long, name: String) : Layer(name) {
+    /**
+     * Input data dimensions. Rank = 3 or 4 for most popular supported cases.
+     */
+    public var packedDims: LongArray = dims
+
+    override fun build(inputShape: TensorShape) {
+        TODO("not implemented")
+    }
+
+    override fun getOutputShape(inputShape: TensorShape): TensorShape {
+        return inputShape
+    }
+
+    override fun toString(): String {
+        return "Input[shape=${packedDims.contentToString()}]"
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt
new file mode 100644
index 00000000..b6ec78e0
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.pool
+
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.layer.conv.ConvPadding
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+import kotlin.math.ceil
+
+/**
+ * Max pooling layer for 2D inputs (e.g. images).
+ *
+ * @property [poolSize] The size of the sliding window for each dimension of input tensor (pool batch, pool height, pool width, pool channels).
+ * Usually, pool batch and pool channels are equal to 1.
+ * @property [strides] Strides of the pooling operation for each dimension of input tensor.
+ * @property [padding] The padding method, either 'valid' or 'same' or 'full'.
+ * @property [name] Custom layer name.
+ */
+public class Pool2D(
+    public val poolSize: IntArray = intArrayOf(1, 2, 2, 1),
+    public val strides: IntArray = intArrayOf(1, 2, 2, 1),
+    public val padding: ConvPadding = ConvPadding.VALID,
+    name: String
+) : Layer(name) {
+
+    private var padHeight = 0L
+    private var padWidth = 0L
+
+    override fun build(inputShape: TensorShape) {
+    }
+
+    override fun getOutputShape(inputShape: TensorShape): TensorShape {
+        var outHeight = 0L
+        var outWidth = 0L
+        // return the output tensor shape
+        if (padding == ConvPadding.VALID) {
+            outHeight = ceil((inputShape[1] - poolSize[1] + 1).toDouble() / strides[1].toDouble()).toLong()
+            outWidth = ceil((inputShape[2] - poolSize[2] + 1).toDouble() / strides[2].toDouble()).toLong()
+            padHeight = 0
+            padWidth = 0
+        } else if (padding == ConvPadding.SAME) {
+            outHeight = ceil(inputShape[1].toFloat() / strides[1].toFloat()).toLong()
+            outWidth = ceil(inputShape[2].toFloat() / strides[2].toFloat()).toLong()
+            val padAlongHeight = (outHeight - 1) * strides[1] + poolSize[1] - inputShape[1]
+            val padAlongWidth = (outWidth - 1) * strides[2] + poolSize[2] - inputShape[2]
+
+            padHeight = padAlongHeight / 2
+            padWidth = padAlongWidth / 2
+        }
+
+        return TensorShape(inputShape[0], outHeight, outWidth, inputShape[3])
+    }
+
+    override fun toString(): String {
+        return "MaxPool2D[poolSize=${poolSize.contentToString()}, strides=${strides.contentToString()}, padding=$padding]"
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt
new file mode 100644
index 00000000..1fead435
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.layer.regularization
+
+import org.opendc.experiments.tf20.keras.layer.Layer
+import org.opendc.experiments.tf20.keras.shape.TensorShape
+
+/**
+ * This layer applies dropout to the input.
+ *
+ * Dropout consists in randomly setting a fraction `rate` of input units to 0
+ * at each update during training time, which helps prevent overfitting.
+ * The units that are kept are scaled by `1 / (1 - rate)`, so that their
+ * sum is unchanged at training time and inference time.
+ *
+ * @property keepProbability The dropout rate, between 0 and 1. E.g. `rate=0.1` would drop out 10% of input units.
+ * @property [name] Custom layer name.
+ */
+public class Dropout(
+    public val keepProbability: Float = 0.1f,
+    name: String
+) : Layer(name) {
+    override fun build(inputShape: TensorShape) {}
+
+    override fun getOutputShape(inputShape: TensorShape): TensorShape {
+        return inputShape
+    }
+
+    override fun toString(): String = "Dropout[keepProbability=$keepProbability]"
+}
diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt
new file mode 100644
index 00000000..7affcb63
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.tf20.keras.shape
+
+import kotlin.math.abs
+
+/**
+ * Represents the shape of a tensor.
+ *
+ * @param dims The sizes of the tensor dimensions.
+ */
+public class TensorShape(vararg dims: Long) {
+    /**
+     * The dimensions of the tensor represented as [LongArray].
+     */
+    private val _dims: LongArray = dims
+
+    /**
+     * Return amount of elements in Tensor with the given shape.
+     */
+    public val numElements: Long
+        get() {
+            var prod = 1L
+            for (i in 0 until rank) {
+                prod *= abs(_dims[i])
+            }
+            return prod
+        }
+
+    /**
+     * Returns the rank of this shape.
+     */
+    public val rank: Int
+        get() = _dims.size
+
+    /**
+     * Returns the value of a dimension
+     *
+     * @param i The index at which to retrieve a dimension.
+     * @return The size of dimension i
+     */
+    public operator fun get(i: Int): Long {
+        return _dims[i]
+    }
+
+    /**
+     * Test whether dimension i in this shape is known
+     *
+     * @param i Target dimension to test
+     * @return Whether dimension i is unknown (equal to -1)
+     */
+    private fun isKnown(i: Int): Boolean {
+        return _dims[i] != -1L
+    }
+
+    /**
+     * Get the size of a target dimension.
+     *
+     * @param i Target dimension.
+     * @return The size of dimension i
+     */
+    public fun size(i: Int): Long {
+        return _dims[i]
+    }
+
+    /**
+     * Clone the [TensorShape] and return a new instance.
+     */
+    public fun clone(): TensorShape {
+        return TensorShape(*_dims)
+    }
+
+    /**
+     * Create a string representation of this [TensorShape].
+     */
+    override fun toString(): String {
+        return _dims.contentToString().replace("-1", "None")
+    }
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (javaClass != other?.javaClass) return false
+
+        other as TensorShape
+
+        if (!_dims.contentEquals(other._dims)) return false
+
+        return true
+    }
+
+    override fun hashCode(): Int {
+        return _dims.contentHashCode()
+    }
+}