From 726a0570977c3426b1dd28a922140a95dad96462 Mon Sep 17 00:00:00 2001 From: Wenchen Lai Date: Fri, 7 May 2021 17:36:01 +0200 Subject: exp: Add model of TensorFlow Keras API This change adds a model of TensorFlow's Keras API to OpenDC. --- .../kotlin/org/opendc/experiments/tf20/Models.kt | 87 +++++++++ .../opendc/experiments/tf20/keras/Sequential.kt | 53 ++++++ .../experiments/tf20/keras/TrainableModel.kt | 33 ++++ .../tf20/keras/activations/Activation.kt | 198 +++++++++++++++++++++ .../opendc/experiments/tf20/keras/layer/Layer.kt | 44 +++++ .../experiments/tf20/keras/layer/conv/Conv2D.kt | 80 +++++++++ .../tf20/keras/layer/conv/ConvPadding.kt | 39 ++++ .../tf20/keras/layer/core/ActivationLayer.kt | 46 +++++ .../experiments/tf20/keras/layer/core/Input.kt | 48 +++++ .../experiments/tf20/keras/layer/pool/Pool2D.kt | 77 ++++++++ .../tf20/keras/layer/regularization/Dropout.kt | 50 ++++++ .../experiments/tf20/keras/shape/TensorShape.kt | 114 ++++++++++++ 12 files changed, 869 insertions(+) create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt new file mode 100644 index 00000000..9ef5b621 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/Models.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras + +import org.opendc.experiments.tf20.keras.activations.Activation +import org.opendc.experiments.tf20.keras.layer.conv.Conv2D +import org.opendc.experiments.tf20.keras.layer.conv.ConvPadding +import org.opendc.experiments.tf20.keras.layer.core.ActivationLayer +import org.opendc.experiments.tf20.keras.layer.core.Input +import org.opendc.experiments.tf20.keras.layer.pool.Pool2D +import org.opendc.experiments.tf20.keras.layer.regularization.Dropout + +/** + * Construct an AlexNet model with the given batch size. + */ +fun AlexNet(batchSize: Long): TrainableModel { + return Sequential( + Input(batchSize, 227, 227, 3, name = "Input"), + Conv2D(longArrayOf(11, 11, 3, 96), longArrayOf(1, 4, 4, 1), padding = ConvPadding.VALID, name = "conv1"), + Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool1"), + Conv2D(longArrayOf(5, 5, 96, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2"), + Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool2"), + Conv2D(longArrayOf(3, 3, 256, 384), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3"), + Conv2D(longArrayOf(3, 3, 384, 384), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4"), + Conv2D(longArrayOf(3, 3, 384, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5"), + Pool2D(intArrayOf(1, 3, 3, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool5"), + Conv2D(longArrayOf(6, 6, 256, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.VALID, name = "fc6"), + Dropout(0.5f, name = "dropout6"), + Conv2D(longArrayOf(1, 1, 4096, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "fc7"), + Dropout(0.5f, name = "dropout7"), + Conv2D(longArrayOf(1, 1, 4096, 1000), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "f8"), + ActivationLayer(Activation.Softmax, name = "softmax") + ) +} + +/** + * Construct an VGG16 model with the given batch size. + */ +fun VGG16(batchSize: Long = 128): TrainableModel { + return Sequential( + Input(batchSize, 224, 224, 3, name = "Input"), + Conv2D(longArrayOf(3, 3, 3, 64), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv1-1"), + Conv2D(longArrayOf(3, 3, 64, 64), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv1-2"), + Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool1"), + Conv2D(longArrayOf(3, 3, 64, 128), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2-1"), + Conv2D(longArrayOf(3, 3, 128, 128), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv2-2"), + Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool2"), + Conv2D(longArrayOf(3, 3, 128, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-1"), + Conv2D(longArrayOf(3, 3, 256, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-2"), + Conv2D(longArrayOf(3, 3, 256, 256), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv3-3"), + Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool3"), + Conv2D(longArrayOf(3, 3, 256, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-1"), + Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-2"), + Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv4-3"), + Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool4"), + Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-1"), + Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-2"), + Conv2D(longArrayOf(3, 3, 512, 512), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "conv5-3"), + Pool2D(intArrayOf(1, 2, 2, 1), intArrayOf(1, 2, 2, 1), padding = ConvPadding.VALID, name = "pool5"), + Conv2D(longArrayOf(7, 7, 512, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.VALID, name = "fc6"), + Dropout(0.5f, name = "dropout6"), + Conv2D(longArrayOf(1, 1, 4096, 4096), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "fc7"), + Dropout(0.5f, name = "dropout7"), + Conv2D(longArrayOf(1, 1, 4096, 1000), longArrayOf(1, 1, 1, 1), padding = ConvPadding.SAME, name = "f8"), + ActivationLayer(Activation.Softmax, name = "softmax") + ) +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt new file mode 100644 index 00000000..3cc6b690 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras + +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.layer.core.Input + +/** + * Sequential model groups a linear stack of layers into a TensorFlow Model. + * + * @param [layers] The layers to describe the model design. + */ +public class Sequential(vararg layers: Layer) : TrainableModel() { + + /** + * The layers to describe the model design. Main part of the internal state of the model. + */ + public val layers: List = listOf(*layers) + + /** + * First layer that is responsible for the input shape of the Neural Network. + */ + public val inputLayer: Input + get() = layers[0] as Input + + /** + * Returns input dimensions in order HWC (height, width, channels) + */ + public val inputDimensions: LongArray + get() = (layers[0] as Input).packedDims + + override fun close() {} +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt new file mode 100644 index 00000000..421dec73 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/TrainableModel.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras + +/** + * Base abstract class for all trainable models. + */ +public abstract class TrainableModel : AutoCloseable { + + override fun toString(): String { + return "TrainableModel ${super.toString()}" + } +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt new file mode 100644 index 00000000..403acfc0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/activations/Activation.kt @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.activations + +/** + * Neural network hyper-parameter, activation function of a node defines the output of that node given an input or + * set of inputs. + */ +public enum class Activation { + /** + * Linear unit. Returns unmodified input. + * + * NOTE: Doing nothing useful. Returns to ancient times of linear perceptron. + */ + Linear, + + /** + * Sigmoid activation function. + * + * Transforms input 'x' according formula: + * ``` + * sigmoid(x) = 1 / (1 + exp(-x)) + * ``` + * + * For small values (<-5), `sigmoid` returns a value close to zero, and for large values (>5) + * the result of the function gets close to 1. + * + * NOTE: Sigmoid is equivalent to a 2-element ActivationLayer, where the second element is + * assumed to be zero. The sigmoid function always returns a value between 0 and 1. + */ + Sigmoid, + + /** + * Hyperbolic tangent activation function. + * + * Transforms input 'x' according formula: + * ``` + * tanh(x) = sinh(x)/cosh(x) = ((exp(x) - exp(-x))/(exp(x) + exp(-x))) + * ``` + */ + Tanh, + + /** + * Rectified linear unit (ReLU). + * + * With default values, this returns the standard ReLU activation: + * `max(x, 0)`, the element-wise maximum of 0 and the input tensor. + */ + Relu, + + /** + * Computes Rectified Linear 6: + * ``` + * min(max(features, 0), 6) + * ``` + * @see + * Convolutional Deep Belief Networks on CIFAR-10. A. Krizhevsky + */ + Relu6, + + /** + * Exponential Linear Unit. + * + * The exponential linear unit (ELU) with `alpha > 0` is: + * `x` if `x > 0` and `alpha * (exp(x) - 1)` if `x < 0` + * + * For this implementations alpha is equal to 1.0. + * + * The ELU hyperparameter `alpha` controls the value to which an + * ELU saturates for negative net inputs. ELUs diminish the + * vanishing gradient effect. + * + * ELUs have negative values which pushes the mean of the activations closer to zero. + * + * Mean activations that are closer to zero enable faster learning as they + * bring the gradient closer to the natural gradient. + * + * ELUs saturate to a negative value when the argument gets smaller. + * Saturation means a small derivative which decreases the variation + * and the information that is propagated to the next layer. + * + * @see Fast and Accurate Deep Network Learning by Exponential Linear Units + * (ELUs) (Clevert et al, 2016) + */ + Elu, + + /** + * Scaled Exponential Linear Unit (SELU). + * + * The Scaled Exponential Linear Unit (SELU) activation function is defined as: + * ``` + * if x > 0: return scale * x + * if x < 0: return scale * alpha * (exp(x) - 1) + * ``` + * where `alpha` and `scale` are pre-defined constants (`alpha=1.67326324` and `scale=1.05070098`). + * + * Basically, the SELU activation function multiplies `scale` (> 1) with the + * output of the `tf.keras.activations.elu` function to ensure a slope larger + * than one for positive inputs. + * + * @see Klambauer et al., 2017 + */ + Selu, + + /** + * ActivationLayer converts a real vector to a vector of categorical probabilities. + * The elements of the output vector are in range (0, 1) and sum to 1. + * + * ActivationLayer is often used as the activation for the last + * layer of a classification network because the result could be interpreted as + * a probability distribution. + */ + Softmax, + + /** + * + */ + LogSoftmax, + + /** + * Exponential activation function. + * + * Transforms input 'x' according formula: + * ``` + * exp(x) + * ``` + */ + Exponential, + + /** + * Softplus activation function. + * + * Transforms input 'x' according formula: + * ``` + * softplus(x) = log(exp(x) + 1) + * ``` + */ + SoftPlus, + + /*** + * Softsign activation function. + * + * Transforms input 'x' according formula: + * ``` + * softsign(x) = x / (abs(x) + 1) + * ``` + */ + SoftSign, + + /** + * Hard sigmoid activation function. + * + * Transforms input 'x' according formula: + * ``` + * if x < -2.5: return 0 + * if x > 2.5: return 1 + * if -2.5 <= x <= 2.5: return 0.2 * x + 0.5 + * ``` + * A faster approximation of the sigmoid activation. + */ + HardSigmoid, + + /** + * Swish activation function. + * + * Transforms input 'x' according formula: + * ``` + * swish(x) = x * sigmoid(x) + * ``` + * + * It is a smooth, non-monotonic function that consistently matches + * or outperforms ReLU on deep networks, it is unbounded above and + * bounded below. + * + * @see Ramachandran et al., 2017 + */ + Swish; +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt new file mode 100644 index 00000000..eafdf63b --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/Layer.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer + +import org.opendc.experiments.tf20.keras.shape.TensorShape + +/** + * Abstract class from which all layers inherit. + * + * @param name The name of the layer. + */ +public abstract class Layer(public val name: String) { + /** + * Build the layer for the specified [inputShape]. + * + * @param [inputShape] Input shape, result of [getOutputShape] call from previous layer. + */ + public abstract fun build(inputShape: TensorShape) + + /** + * Compute output shape of this layer, based on [inputShape] and [Layer] type. + */ + public abstract fun getOutputShape(inputShape: TensorShape): TensorShape +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt new file mode 100644 index 00000000..c8078bf8 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/Conv2D.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.conv + +import org.opendc.experiments.tf20.keras.activations.Activation +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.shape.TensorShape +import kotlin.math.ceil + +/** + * 2D convolution layer (e.g. spatial convolution over images). + * + * This layer creates a convolution kernel that is convolved (actually cross-correlated) + * with the layer input to produce a tensor of outputs. + * Finally, if `activation` is applied to the outputs as well. + */ +public class Conv2D( + public val filter: LongArray = LongArray(4), // [H, W, channel_in, channel_out] + public val strides: LongArray = LongArray(4), // [1, stride_h, stride_w, 1] + public val activation: Activation = Activation.Relu, + public val padding: ConvPadding = ConvPadding.VALID, + name: String = "", +) : Layer(name) { + + private var padHeight: Double = 0.0 + private var padWidth: Double = 0.0 + + override fun build(inputShape: TensorShape) { + TODO("not implemented") + } + + override fun getOutputShape(inputShape: TensorShape): TensorShape { + check(filter[2] != inputShape[3]) { "Input channel ${filter[2]} and ${inputShape[3]} shall match" } + + var outHeight = 0L + var outWidth = 0L + + if (padding == ConvPadding.VALID) { + outHeight = ceil((inputShape[1] - filter[0] + 1).toDouble() / strides[1].toDouble()).toLong() + outWidth = ceil((inputShape[2] - filter[1] + 1).toDouble() / strides[2].toDouble()).toLong() + padHeight = 0.0 + padWidth = 0.0 + } else if (padding == ConvPadding.SAME) { + outHeight = ceil(inputShape[1].toFloat() / strides[1].toFloat()).toLong() + outWidth = ceil(inputShape[2].toFloat() / strides[2].toFloat()).toLong() + + val padAlongHeight = (outHeight - 1) * strides[1] + filter[0] - inputShape[1] + val padAlongWidth = (outWidth - 1) * strides[2] + filter[1] - inputShape[2] + + padHeight = (padAlongHeight / 2).toDouble() + padWidth = (padAlongWidth / 2).toDouble() + } + + return TensorShape(inputShape[0], outHeight, outWidth, filter[3]) + } + + override fun toString(): String { + return "Conv2D[filter=${filter.contentToString()}, strides=${strides.contentToString()}, activation=$activation, padding=$padding]" + } +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt new file mode 100644 index 00000000..03ae6282 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/conv/ConvPadding.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.conv + +/** + * Enumeration of convolution padding types. + */ +public enum class ConvPadding { + /** + * Pad evenly to the left/right or up/down of the input such that output has the same + * height/width dimension as the input. + */ + SAME, + + /** + * No padding. + */ + VALID +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt new file mode 100644 index 00000000..befcfe50 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/ActivationLayer.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.core + +import org.opendc.experiments.tf20.keras.activations.Activation +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.shape.TensorShape + +/** + * This layer applies an activation function to an output. + */ +public class ActivationLayer( + public val activation: Activation = Activation.Relu, + name: String = "", +) : Layer(name) { + + override fun build(inputShape: TensorShape) { + // Intentionally left empty + } + + override fun getOutputShape(inputShape: TensorShape): TensorShape = inputShape + + override fun toString(): String { + return "ActivationLayer[activation=$activation]" + } +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt new file mode 100644 index 00000000..b56284d6 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/core/Input.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.core + +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.shape.TensorShape + +/** + * This layer is responsible for the input shape of the built model. + */ +public class Input(vararg dims: Long, name: String) : Layer(name) { + /** + * Input data dimensions. Rank = 3 or 4 for most popular supported cases. + */ + public var packedDims: LongArray = dims + + override fun build(inputShape: TensorShape) { + TODO("not implemented") + } + + override fun getOutputShape(inputShape: TensorShape): TensorShape { + return inputShape + } + + override fun toString(): String { + return "Input[shape=${packedDims.contentToString()}]" + } +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt new file mode 100644 index 00000000..b6ec78e0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/pool/Pool2D.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.pool + +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.layer.conv.ConvPadding +import org.opendc.experiments.tf20.keras.shape.TensorShape +import kotlin.math.ceil + +/** + * Max pooling layer for 2D inputs (e.g. images). + * + * @property [poolSize] The size of the sliding window for each dimension of input tensor (pool batch, pool height, pool width, pool channels). + * Usually, pool batch and pool channels are equal to 1. + * @property [strides] Strides of the pooling operation for each dimension of input tensor. + * @property [padding] The padding method, either 'valid' or 'same' or 'full'. + * @property [name] Custom layer name. + */ +public class Pool2D( + public val poolSize: IntArray = intArrayOf(1, 2, 2, 1), + public val strides: IntArray = intArrayOf(1, 2, 2, 1), + public val padding: ConvPadding = ConvPadding.VALID, + name: String +) : Layer(name) { + + private var padHeight = 0L + private var padWidth = 0L + + override fun build(inputShape: TensorShape) { + } + + override fun getOutputShape(inputShape: TensorShape): TensorShape { + var outHeight = 0L + var outWidth = 0L + // return the output tensor shape + if (padding == ConvPadding.VALID) { + outHeight = ceil((inputShape[1] - poolSize[1] + 1).toDouble() / strides[1].toDouble()).toLong() + outWidth = ceil((inputShape[2] - poolSize[2] + 1).toDouble() / strides[2].toDouble()).toLong() + padHeight = 0 + padWidth = 0 + } else if (padding == ConvPadding.SAME) { + outHeight = ceil(inputShape[1].toFloat() / strides[1].toFloat()).toLong() + outWidth = ceil(inputShape[2].toFloat() / strides[2].toFloat()).toLong() + val padAlongHeight = (outHeight - 1) * strides[1] + poolSize[1] - inputShape[1] + val padAlongWidth = (outWidth - 1) * strides[2] + poolSize[2] - inputShape[2] + + padHeight = padAlongHeight / 2 + padWidth = padAlongWidth / 2 + } + + return TensorShape(inputShape[0], outHeight, outWidth, inputShape[3]) + } + + override fun toString(): String { + return "MaxPool2D[poolSize=${poolSize.contentToString()}, strides=${strides.contentToString()}, padding=$padding]" + } +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt new file mode 100644 index 00000000..1fead435 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/layer/regularization/Dropout.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.layer.regularization + +import org.opendc.experiments.tf20.keras.layer.Layer +import org.opendc.experiments.tf20.keras.shape.TensorShape + +/** + * This layer applies dropout to the input. + * + * Dropout consists in randomly setting a fraction `rate` of input units to 0 + * at each update during training time, which helps prevent overfitting. + * The units that are kept are scaled by `1 / (1 - rate)`, so that their + * sum is unchanged at training time and inference time. + * + * @property keepProbability The dropout rate, between 0 and 1. E.g. `rate=0.1` would drop out 10% of input units. + * @property [name] Custom layer name. + */ +public class Dropout( + public val keepProbability: Float = 0.1f, + name: String +) : Layer(name) { + override fun build(inputShape: TensorShape) {} + + override fun getOutputShape(inputShape: TensorShape): TensorShape { + return inputShape + } + + override fun toString(): String = "Dropout[keepProbability=$keepProbability]" +} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt new file mode 100644 index 00000000..7affcb63 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/shape/TensorShape.kt @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.keras.shape + +import kotlin.math.abs + +/** + * Represents the shape of a tensor. + * + * @param dims The sizes of the tensor dimensions. + */ +public class TensorShape(vararg dims: Long) { + /** + * The dimensions of the tensor represented as [LongArray]. + */ + private val _dims: LongArray = dims + + /** + * Return amount of elements in Tensor with the given shape. + */ + public val numElements: Long + get() { + var prod = 1L + for (i in 0 until rank) { + prod *= abs(_dims[i]) + } + return prod + } + + /** + * Returns the rank of this shape. + */ + public val rank: Int + get() = _dims.size + + /** + * Returns the value of a dimension + * + * @param i The index at which to retrieve a dimension. + * @return The size of dimension i + */ + public operator fun get(i: Int): Long { + return _dims[i] + } + + /** + * Test whether dimension i in this shape is known + * + * @param i Target dimension to test + * @return Whether dimension i is unknown (equal to -1) + */ + private fun isKnown(i: Int): Boolean { + return _dims[i] != -1L + } + + /** + * Get the size of a target dimension. + * + * @param i Target dimension. + * @return The size of dimension i + */ + public fun size(i: Int): Long { + return _dims[i] + } + + /** + * Clone the [TensorShape] and return a new instance. + */ + public fun clone(): TensorShape { + return TensorShape(*_dims) + } + + /** + * Create a string representation of this [TensorShape]. + */ + override fun toString(): String { + return _dims.contentToString().replace("-1", "None") + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as TensorShape + + if (!_dims.contentEquals(other._dims)) return false + + return true + } + + override fun hashCode(): Int { + return _dims.contentHashCode() + } +} -- cgit v1.2.3