diff options
| author | Dante Niewenhuis <d.niewenhuis@hotmail.com> | 2024-03-05 16:50:35 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-03-05 16:50:35 +0100 |
| commit | 960b3d8a13c67ac4b7f479d5764b0b618fc9ea09 (patch) | |
| tree | 4f103bcf6635341827d9cfa10c10cfde9543f04f /opendc-compute/opendc-compute-topology | |
| parent | 5864cbcbfe2eb8c36ca05c3a39c7e5916aeecaec (diff) | |
Cpu fix (#208)
* Updated the topology format to JSON. Updated TopologyReader.kt to handle JSON filed. Added documentation for the new format.
* applied spotless kotlin
* small update
* Updated for spotless apply
* Updated for spotless apply
Diffstat (limited to 'opendc-compute/opendc-compute-topology')
| -rw-r--r-- | opendc-compute/opendc-compute-topology/build.gradle.kts | 5 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpecReader.kt | 123 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt | 94 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt (renamed from opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpec.kt) | 46 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt (renamed from opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/HostSpec.kt) | 3 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/JSONSpecs.kt | 113 | ||||
| -rw-r--r-- | opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/TopologySchema.json | 159 |
7 files changed, 368 insertions, 175 deletions
diff --git a/opendc-compute/opendc-compute-topology/build.gradle.kts b/opendc-compute/opendc-compute-topology/build.gradle.kts index 0dedf8a9..f236cbbd 100644 --- a/opendc-compute/opendc-compute-topology/build.gradle.kts +++ b/opendc-compute/opendc-compute-topology/build.gradle.kts @@ -25,6 +25,7 @@ description = "OpenDC Compute Topology implementation" // Build configuration plugins { `kotlin-library-conventions` + kotlin("plugin.serialization") version "1.9.22" } dependencies { @@ -32,6 +33,8 @@ dependencies { implementation(projects.opendcCommon) implementation(project(mapOf("path" to ":opendc-simulator:opendc-simulator-compute"))) - implementation(libs.jackson.dataformat.csv) + implementation(libs.jackson.module.kotlin) + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0") + implementation(project(mapOf("path" to ":opendc-trace:opendc-trace-api"))) testImplementation(projects.opendcSimulator.opendcSimulatorCore) } diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpecReader.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpecReader.kt deleted file mode 100644 index 13314f7d..00000000 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpecReader.kt +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.topology - -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.MappingIterator -import com.fasterxml.jackson.databind.ObjectReader -import com.fasterxml.jackson.dataformat.csv.CsvMapper -import com.fasterxml.jackson.dataformat.csv.CsvSchema -import java.io.File -import java.io.InputStream - -/** - * A helper class for reading a cluster specification file. - */ -public class ClusterSpecReader { - /** - * The [CsvMapper] to map the environment file to an object. - */ - private val mapper = CsvMapper() - - /** - * The [ObjectReader] to convert the lines into objects. - */ - private val reader: ObjectReader = mapper.readerFor(Entry::class.java).with(schema) - - /** - * Read the specified [file]. - */ - public fun read(file: File): List<ClusterSpec> { - return reader.readValues<Entry>(file).use { read(it) } - } - - /** - * Read the specified [input]. - */ - public fun read(input: InputStream): List<ClusterSpec> { - return reader.readValues<Entry>(input).use { read(it) } - } - - /** - * Convert the specified [MappingIterator] into a list of [ClusterSpec]s. - */ - private fun read(it: MappingIterator<Entry>): List<ClusterSpec> { - val result = mutableListOf<ClusterSpec>() - - for (entry in it) { - val def = - ClusterSpec( - entry.id, - entry.name, - entry.cpuCount, - entry.cpuSpeed * 1000, - entry.memCapacity * 1000, - entry.hostCount, - entry.memCapacityPerHost * 1000, - entry.cpuCountPerHost, - ) - result.add(def) - } - - return result - } - - private open class Entry( - @JsonProperty("ClusterID") - val id: String, - @JsonProperty("ClusterName") - val name: String, - @JsonProperty("Cores") - val cpuCount: Int, - @JsonProperty("Speed") - val cpuSpeed: Double, - @JsonProperty("Memory") - val memCapacity: Double, - @JsonProperty("numberOfHosts") - val hostCount: Int, - @JsonProperty("memoryCapacityPerHost") - val memCapacityPerHost: Double, - @JsonProperty("coreCountPerHost") - val cpuCountPerHost: Int, - ) - - public companion object { - /** - * The [CsvSchema] that is used to parse the trace. - */ - private val schema = - CsvSchema.builder() - .addColumn("ClusterID", CsvSchema.ColumnType.STRING) - .addColumn("ClusterName", CsvSchema.ColumnType.STRING) - .addColumn("Cores", CsvSchema.ColumnType.NUMBER) - .addColumn("Speed", CsvSchema.ColumnType.NUMBER) - .addColumn("Memory", CsvSchema.ColumnType.NUMBER) - .addColumn("numberOfHosts", CsvSchema.ColumnType.NUMBER) - .addColumn("memoryCapacityPerHost", CsvSchema.ColumnType.NUMBER) - .addColumn("coreCountPerHost", CsvSchema.ColumnType.NUMBER) - .setAllowComments(true) - .setColumnSeparator(';') - .setUseHeader(true) - .build() - } -} diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt index aadf52a6..47ba8058 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyFactories.kt @@ -24,34 +24,38 @@ package org.opendc.compute.topology +import org.opendc.compute.topology.specs.CPUJSONSpec +import org.opendc.compute.topology.specs.ClusterJSONSpec +import org.opendc.compute.topology.specs.HostJSONSpec +import org.opendc.compute.topology.specs.HostSpec +import org.opendc.compute.topology.specs.TopologyJSONSpec import org.opendc.simulator.compute.SimPsuFactories import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.CpuPowerModel -import org.opendc.simulator.compute.power.CpuPowerModels +import org.opendc.simulator.compute.power.getPowerModel import java.io.File import java.io.InputStream import java.util.SplittableRandom import java.util.UUID import java.util.random.RandomGenerator -import kotlin.math.roundToLong /** - * A [ClusterSpecReader] that is used to read the cluster definition file. + * A [TopologyReader] that is used to read the cluster definition file. */ -private val reader = ClusterSpecReader() +private val reader = TopologyReader() /** * Construct a topology from the specified [file]. */ public fun clusterTopology( file: File, - powerModel: CpuPowerModel = CpuPowerModels.linear(350.0, 200.0), random: RandomGenerator = SplittableRandom(0), ): List<HostSpec> { - return clusterTopology(reader.read(file), powerModel, random) + val topology = reader.read(file) + + return topology.toHostSpecs(random) } /** @@ -59,48 +63,78 @@ public fun clusterTopology( */ public fun clusterTopology( input: InputStream, - powerModel: CpuPowerModel = CpuPowerModels.linear(350.0, 200.0), random: RandomGenerator = SplittableRandom(0), ): List<HostSpec> { - return clusterTopology(reader.read(input), powerModel, random) + val topology = reader.read(input) + + return topology.toHostSpecs(random) } /** - * Construct a topology from the given list of [clusters]. + * Helper method to convert a [TopologyJSONSpec] into a list of [HostSpec]s. */ -public fun clusterTopology( - clusters: List<ClusterSpec>, - powerModel: CpuPowerModel, - random: RandomGenerator = SplittableRandom(0), -): List<HostSpec> { - return clusters.flatMap { it.toHostSpecs(random, powerModel) } +private fun TopologyJSONSpec.toHostSpecs(random: RandomGenerator): List<HostSpec> { + return clusters.flatMap { cluster -> List(cluster.count) { cluster.toHostSpecs(random) }.flatten() } +} + +/** + * Helper method to convert a [ClusterJSONSpec] into a list of [HostSpec]s. + */ +private var clusterId = 0 + +private fun ClusterJSONSpec.toHostSpecs(random: RandomGenerator): List<HostSpec> { + val hostSpecs = + hosts.flatMap { host -> + ( + List(host.count) { + host.toHostSpecs(clusterId, random) + } + ) + } + clusterId++ + return hostSpecs } /** - * Helper method to convert a [ClusterSpec] into a list of [HostSpec]s. + * Helper method to convert a [HostJSONSpec] into a [HostSpec]s. */ -private fun ClusterSpec.toHostSpecs( +private var hostId = 0 + +private fun HostJSONSpec.toHostSpecs( + clusterId: Int, random: RandomGenerator, - powerModel: CpuPowerModel, -): List<HostSpec> { - val cpuSpeed = cpuSpeed - val memoryPerHost = memCapacityPerHost.roundToLong() +): HostSpec { + val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", cpus.sumOf { it.coreCount }) + + val units = cpus.flatMap { cpu -> List(cpu.count) { cpu.toProcessingUnit(unknownProcessingNode) }.flatten() } - val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", cpuCountPerHost) - val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) + val unknownMemoryUnit = MemoryUnit(memory.vendor, memory.modelName, memory.memorySpeed, memory.memorySize) val machineModel = MachineModel( - List(cpuCountPerHost) { coreId -> ProcessingUnit(unknownProcessingNode, coreId, cpuSpeed) }, + units, listOf(unknownMemoryUnit), ) - return List(hostCount) { + val powerModel = getPowerModel(powerModel.modelType, powerModel.power, powerModel.maxPower, powerModel.idlePower) + val hostSpec = HostSpec( - UUID(random.nextLong(), it.toLong()), - "node-$name-$it", - mapOf("cluster" to id), + UUID(random.nextLong(), (hostId).toLong()), + "$name-${(hostId)}", + mapOf("cluster" to clusterId), machineModel, SimPsuFactories.simple(powerModel), ) - } + hostId++ + + return hostSpec +} + +/** + * Helper method to convert a [CPUJSONSpec] into a list of [ProcessingUnit]s. + */ +private var globalCoreId = 0 + +private fun CPUJSONSpec.toProcessingUnit(unknownProcessingNode: ProcessingNode): List<ProcessingUnit> { + val units = List(coreCount) { ProcessingUnit(unknownProcessingNode, globalCoreId++, coreSpeed) } + return units } diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpec.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt index 7a8a121c..70e08e3b 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/ClusterSpec.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/TopologyReader.kt @@ -22,25 +22,31 @@ package org.opendc.compute.topology +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import org.opendc.compute.topology.specs.TopologyJSONSpec +import java.io.File +import java.io.InputStream + /** - * Definition of a compute cluster modeled in the simulation. - * - * @param id A unique identifier representing the compute cluster. - * @param name The name of the cluster. - * @param cpuCount The total number of CPUs in the cluster. - * @param cpuSpeed The speed of a CPU in the cluster in MHz. - * @param memCapacity The total memory capacity of the cluster (in MiB). - * @param hostCount The number of hosts in the cluster. - * @param memCapacityPerHost The memory capacity per host in the cluster (MiB). - * @param cpuCountPerHost The number of CPUs per host in the cluster. + * A helper class for reading a topology specification file. */ -public data class ClusterSpec( - val id: String, - val name: String, - val cpuCount: Int, - val cpuSpeed: Double, - val memCapacity: Double, - val hostCount: Int, - val memCapacityPerHost: Double, - val cpuCountPerHost: Int, -) +public class TopologyReader { + @OptIn(ExperimentalSerializationApi::class) + public fun read(file: File): TopologyJSONSpec { + val input = file.inputStream() + val obj = Json.decodeFromStream<TopologyJSONSpec>(input) + + return obj + } + + /** + * Read the specified [input]. + */ + @OptIn(ExperimentalSerializationApi::class) + public fun read(input: InputStream): TopologyJSONSpec { + val obj = Json.decodeFromStream<TopologyJSONSpec>(input) + return obj + } +} diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/HostSpec.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt index ffaa093e..23fbdcb5 100644 --- a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/HostSpec.kt +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/HostSpec.kt @@ -20,7 +20,7 @@ * SOFTWARE. */ -package org.opendc.compute.topology +package org.opendc.compute.topology.specs import org.opendc.simulator.compute.SimPsuFactories import org.opendc.simulator.compute.SimPsuFactory @@ -38,6 +38,7 @@ import java.util.UUID * @param psuFactory The [SimPsuFactory] to construct the PSU that models the power consumption of the machine. * @param multiplexerFactory The [FlowMultiplexerFactory] that is used to multiplex the virtual machines over the host. */ + public data class HostSpec( val uid: UUID, val name: String, diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/JSONSpecs.kt b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/JSONSpecs.kt new file mode 100644 index 00000000..fbdb4f5f --- /dev/null +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/JSONSpecs.kt @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2024 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.topology.specs + +import kotlinx.serialization.Serializable + +/** + * Definition of a Topology modeled in the simulation. + * + * @param clusters List of the clusters in this topology + */ +@Serializable +public data class TopologyJSONSpec( + val clusters: List<ClusterJSONSpec>, + val schemaVersion: Int = 1, +) + +/** + * Definition of a compute cluster modeled in the simulation. + * + * @param name The name of the cluster. + * @param hosts List of the different hosts (nodes) available in this cluster + * @param location Location of the cluster. This can impact the carbon intensity + */ +@Serializable +public data class ClusterJSONSpec( + val name: String = "Cluster", + val count: Int = 1, + val hosts: List<HostJSONSpec>, + val location: String = "NL", +) + +/** + * Definition of a compute host modeled in the simulation. + * + * @param name The name of the host. + * @param cpus List of the different CPUs available in this cluster + * @param memCapacity The amount of RAM memory available in Byte + * @param powerModel The power model used to determine the power draw of a host + */ +@Serializable +public data class HostJSONSpec( + val name: String = "Host", + val cpus: List<CPUJSONSpec>, + val memory: MemoryJSONSpec, + val powerModel: PowerModelJSONSpec = PowerModelJSONSpec("linear", 350.0, 200.0, 400.0), + val count: Int = 1, +) + +/** + * Definition of a compute CPU modeled in the simulation. + * + * @param vendor The vendor of the storage device. + * @param modelName The model name of the device. + * @param arch The micro-architecture of the processor node. + * @param coreCount The number of cores in the CPU + * @param coreSpeed The speed of the cores in Mhz + */ +@Serializable +public data class CPUJSONSpec( + val vendor: String = "unknown", + val modelName: String = "unknown", + val arch: String = "unknown", + val coreCount: Int, + val coreSpeed: Double, + val count: Int = 1, +) + +/** + * Definition of a compute Memory modeled in the simulation. + * + * @param vendor The vendor of the storage device. + * @param modelName The model name of the device. + * @param arch The micro-architecture of the processor node. + * @param memorySpeed The speed of the cores in ? + * @param memorySize The size of the memory Unit in MiB + */ +@Serializable +public data class MemoryJSONSpec( + val vendor: String = "unknown", + val modelName: String = "unknown", + val arch: String = "unknown", + val memorySpeed: Double = -1.0, + val memorySize: Long, +) + +@Serializable +public data class PowerModelJSONSpec( + val modelType: String, + val power: Double = 400.0, + val maxPower: Double, + val idlePower: Double, +) diff --git a/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/TopologySchema.json b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/TopologySchema.json new file mode 100644 index 00000000..93aa001f --- /dev/null +++ b/opendc-compute/opendc-compute-topology/src/main/kotlin/org/opendc/compute/topology/specs/TopologySchema.json @@ -0,0 +1,159 @@ +{ + "$schema": "OpenDC/Topology", + "$defs": { + "cpu": { + "description": "definition of a cpu", + "type": "object", + "properties": { + "vendor": { + "type": "string", + "default": "unknown" + }, + "modelName": { + "type": "string", + "default": "unknown" + }, + "arch": { + "type": "string", + "default": "unknown" + }, + "coreCount": { + "type": "integer" + }, + "coreSpeed": { + "description": "The core speed of the cpu in Mhz", + "type": "number" + }, + "count": { + "description": "The amount CPUs of this type present in the cluster", + "type": "integer" + } + }, + "required": [ + "coreCount", + "coreSpeed" + ] + }, + "memory": { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "default": "unknown" + }, + "modelName": { + "type": "string", + "default": "unknown" + }, + "arch": { + "type": "string", + "default": "unknown" + }, + "memorySize": { + "description": "The amount of the memory in B", + "type": "integer" + }, + "memorySpeed": { + "description": "The speed of the memory in Mhz. Note: currently, this does nothing", + "type": "number", + "default": -1 + } + }, + "required": [ + "memorySize" + ] + }, + "powerModel": { + "type": "object", + "properties": { + "modelType": { + "description": "The type of model used to determine power draw", + "type": "string" + }, + "power": { + "description": "The constant power draw when using the 'constant' power model type in Watt", + "type": "number", + "default": 400 + }, + "maxPower": { + "description": "The power draw of a host when idle in Watt", + "type": "number" + }, + "idlePower": { + "description": "The power draw of a host when using max capacity in Watt", + "type": "number" + } + }, + "required": [ + "modelType", + "maxPower", + "idlePower" + ] + }, + "host": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "Host" + }, + "count": { + "description": "The amount hosts of this type present in the cluster", + "type": "integer", + "default": 1 + }, + "cpus": { + "type": "array", + "items": { + "$ref": "#/$defs/cpu" + }, + "minItems": 1 + }, + "memory": { + "$ref": "#/$defs/memory" + } + }, + "required": [ + "cpus", + "memory" + ] + }, + "cluster": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "Cluster" + }, + "count": { + "description": "The amount clusters of this type present in the Data center", + "type": "integer", + "default": 1 + }, + "hosts": { + "type": "array", + "items": { + "$ref": "#/$defs/host" + }, + "minItems": 1 + } + }, + "required": [ + "hosts" + ] + } + }, + "properties": { + "clusters": { + "description": "Clusters present in the data center", + "type": "array", + "items": { + "$ref": "#/$defs/cluster" + }, + "minItems": 1 + } + }, + "required": [ + "clusters" + ] +} |
