diff options
Diffstat (limited to 'simulator/opendc-format')
10 files changed, 81 insertions, 68 deletions
diff --git a/simulator/opendc-format/build.gradle.kts b/simulator/opendc-format/build.gradle.kts index 38fcb329..3ab704bd 100644 --- a/simulator/opendc-format/build.gradle.kts +++ b/simulator/opendc-format/build.gradle.kts @@ -31,7 +31,8 @@ dependencies { api(project(":opendc-core")) api(project(":opendc-compute")) api(project(":opendc-workflows")) - api("com.fasterxml.jackson.module:jackson-module-kotlin:2.9.8") { + implementation(project(":opendc-simulator:opendc-simulator-compute")) + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.9.8") { exclude("org.jetbrains.kotlin", module = "kotlin-reflect") } implementation(kotlin("reflect")) diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt index 027548a8..3fde1a6c 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt @@ -26,10 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import kotlinx.coroutines.CoroutineScope -import org.opendc.compute.core.MemoryUnit -import org.opendc.compute.core.ProcessingNode -import org.opendc.compute.core.ProcessingUnit -import org.opendc.compute.metal.driver.SimpleBareMetalDriver +import org.opendc.compute.metal.driver.SimBareMetalDriver import org.opendc.compute.metal.service.ProvisioningService import org.opendc.compute.metal.service.SimpleProvisioningService import org.opendc.core.Environment @@ -37,6 +34,10 @@ import org.opendc.core.Platform import org.opendc.core.Zone import org.opendc.core.services.ServiceRegistry import org.opendc.format.environment.EnvironmentReader +import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit import java.io.InputStream import java.time.Clock import java.util.* @@ -74,14 +75,13 @@ public class Sc18EnvironmentReader(input: InputStream, mapper: ObjectMapper = ja else -> throw IllegalArgumentException("The cpu id $id is not recognized") } } - SimpleBareMetalDriver( + SimBareMetalDriver( coroutineScope, clock, UUID.randomUUID(), "node-${counter++}", emptyMap(), - cores, - listOf(MemoryUnit("", "", 2300.0, 16000)) + SimMachineModel(cores, listOf(MemoryUnit("", "", 2300.0, 16000))) ) } } diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt index 634d5de6..63539d61 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt @@ -23,11 +23,8 @@ package org.opendc.format.environment.sc20 import kotlinx.coroutines.CoroutineScope -import org.opendc.compute.core.MemoryUnit -import org.opendc.compute.core.ProcessingNode -import org.opendc.compute.core.ProcessingUnit import org.opendc.compute.metal.NODE_CLUSTER -import org.opendc.compute.metal.driver.SimpleBareMetalDriver +import org.opendc.compute.metal.driver.SimBareMetalDriver import org.opendc.compute.metal.power.LinearLoadPowerModel import org.opendc.compute.metal.service.ProvisioningService import org.opendc.compute.metal.service.SimpleProvisioningService @@ -36,6 +33,10 @@ import org.opendc.core.Platform import org.opendc.core.Zone import org.opendc.core.services.ServiceRegistry import org.opendc.format.environment.EnvironmentReader +import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit import java.io.File import java.io.FileInputStream import java.io.InputStream @@ -68,7 +69,7 @@ public class Sc20ClusterEnvironmentReader( var memoryPerHost: Long var coresPerHost: Int - val nodes = mutableListOf<SimpleBareMetalDriver>() + val nodes = mutableListOf<SimBareMetalDriver>() val random = Random(0) input.bufferedReader().use { reader -> @@ -102,19 +103,21 @@ public class Sc20ClusterEnvironmentReader( repeat(numberOfHosts) { nodes.add( - SimpleBareMetalDriver( + SimBareMetalDriver( coroutineScope, clock, UUID(random.nextLong(), random.nextLong()), "node-$clusterId-$it", mapOf(NODE_CLUSTER to clusterId), - List(coresPerHost) { coreId -> - ProcessingUnit(unknownProcessingNode, coreId, speed) - }, + SimMachineModel( + List(coresPerHost) { coreId -> + ProcessingUnit(unknownProcessingNode, coreId, speed) + }, + listOf(unknownMemoryUnit) + ), // For now we assume a simple linear load model with an idle draw of ~200W and a maximum // power draw of 350W. // Source: https://stackoverflow.com/questions/6128960 - listOf(unknownMemoryUnit), LinearLoadPowerModel(200.0, 350.0) ) ) diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt index 9fd38d13..36136888 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt @@ -26,10 +26,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import kotlinx.coroutines.CoroutineScope -import org.opendc.compute.core.MemoryUnit -import org.opendc.compute.core.ProcessingNode -import org.opendc.compute.core.ProcessingUnit -import org.opendc.compute.metal.driver.SimpleBareMetalDriver +import org.opendc.compute.metal.driver.SimBareMetalDriver import org.opendc.compute.metal.power.LinearLoadPowerModel import org.opendc.compute.metal.service.ProvisioningService import org.opendc.compute.metal.service.SimpleProvisioningService @@ -38,6 +35,10 @@ import org.opendc.core.Platform import org.opendc.core.Zone import org.opendc.core.services.ServiceRegistry import org.opendc.format.environment.EnvironmentReader +import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit import java.io.InputStream import java.time.Clock import java.util.* @@ -80,17 +81,16 @@ public class Sc20EnvironmentReader(input: InputStream, mapper: ObjectMapper = ja else -> throw IllegalArgumentException("The cpu id $id is not recognized") } } - SimpleBareMetalDriver( + SimBareMetalDriver( coroutineScope, clock, UUID.randomUUID(), "node-${counter++}", emptyMap(), - cores, + SimMachineModel(cores, memories), // For now we assume a simple linear load model with an idle draw of ~200W and a maximum // power draw of 350W. // Source: https://stackoverflow.com/questions/6128960 - memories, LinearLoadPowerModel(200.0, 350.0) ) } diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt index 89d4246d..f98a5a6d 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt @@ -22,14 +22,14 @@ package org.opendc.format.trace.bitbrains -import org.opendc.compute.core.image.FlopsHistoryFragment -import org.opendc.compute.core.image.VmImage +import org.opendc.compute.core.image.SimWorkloadImage import org.opendc.compute.core.workload.IMAGE_PERF_INTERFERENCE_MODEL import org.opendc.compute.core.workload.PerformanceInterferenceModel import org.opendc.compute.core.workload.VmWorkload import org.opendc.core.User import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimTraceWorkload import java.io.BufferedReader import java.io.File import java.io.FileReader @@ -66,7 +66,7 @@ public class BitbrainsTraceReader( .filterNot { it.isDirectory } .forEach { vmFile -> println(vmFile) - val flopsHistory = mutableListOf<FlopsHistoryFragment>() + val flopsHistory = mutableListOf<SimTraceWorkload.Fragment>() var vmId = -1L var cores = -1 var requiredMemory = -1L @@ -99,11 +99,11 @@ public class BitbrainsTraceReader( val flops: Long = (cpuUsage * 5 * 60 * cores).toLong() if (flopsHistory.isEmpty()) { - flopsHistory.add(FlopsHistoryFragment(timestamp, flops, traceInterval, cpuUsage, cores)) + flopsHistory.add(SimTraceWorkload.Fragment(timestamp, flops, traceInterval, cpuUsage, cores)) } else { if (flopsHistory.last().flops != flops) { flopsHistory.add( - FlopsHistoryFragment( + SimTraceWorkload.Fragment( timestamp, flops, traceInterval, @@ -114,8 +114,8 @@ public class BitbrainsTraceReader( } else { val oldFragment = flopsHistory.removeAt(flopsHistory.size - 1) flopsHistory.add( - FlopsHistoryFragment( - oldFragment.tick, + SimTraceWorkload.Fragment( + oldFragment.time, oldFragment.flops + flops, oldFragment.duration + traceInterval, cpuUsage, @@ -139,17 +139,19 @@ public class BitbrainsTraceReader( uuid, "VM Workload $vmId", UnnamedUser, - VmImage( + SimWorkloadImage( uuid, vmId.toString(), - mapOf(IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems), - flopsHistory.asSequence(), - cores, - requiredMemory + mapOf( + IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems, + "cores" to cores, + "required-memory" to requiredMemory + ), + SimTraceWorkload(flopsHistory.asSequence()) ) ) entries[vmId] = TraceEntryImpl( - flopsHistory.firstOrNull()?.tick ?: -1, + flopsHistory.firstOrNull()?.time ?: -1, vmWorkload ) } diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt index 9f8fb558..53bc3956 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt @@ -22,10 +22,11 @@ package org.opendc.format.trace.gwf -import org.opendc.compute.core.image.FlopsApplicationImage +import org.opendc.compute.core.image.SimWorkloadImage import org.opendc.core.User import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimFlopsWorkload import org.opendc.workflows.workload.Job import org.opendc.workflows.workload.Task import org.opendc.workflows.workload.WORKFLOW_TASK_DEADLINE @@ -137,7 +138,7 @@ public class GwfTraceReader(reader: BufferedReader) : TraceReader<Job> { val task = Task( UUID(0L, taskId), "<unnamed>", - FlopsApplicationImage(UUID.randomUUID(), "<unnamed>", emptyMap(), flops, cores), + SimWorkloadImage(UUID.randomUUID(), "<unnamed>", emptyMap(), SimFlopsWorkload(flops, cores)), HashSet(), mapOf(WORKFLOW_TASK_DEADLINE to runtime) ) diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt index bfcc30ce..560ad846 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt @@ -22,14 +22,14 @@ package org.opendc.format.trace.sc20 -import org.opendc.compute.core.image.FlopsHistoryFragment -import org.opendc.compute.core.image.VmImage +import org.opendc.compute.core.image.SimWorkloadImage import org.opendc.compute.core.workload.IMAGE_PERF_INTERFERENCE_MODEL import org.opendc.compute.core.workload.PerformanceInterferenceModel import org.opendc.compute.core.workload.VmWorkload import org.opendc.core.User import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimTraceWorkload import java.io.BufferedReader import java.io.File import java.io.FileReader @@ -109,7 +109,7 @@ public class Sc20TraceReader( } val flopsFragments = sequence { - var last: FlopsHistoryFragment? = null + var last: SimTraceWorkload.Fragment? = null BufferedReader(FileReader(vmFile)).use { reader -> reader.lineSequence() @@ -130,8 +130,8 @@ public class Sc20TraceReader( last = if (last != null && last!!.flops == 0L && flops == 0L) { val oldFragment = last!! - FlopsHistoryFragment( - oldFragment.tick, + SimTraceWorkload.Fragment( + oldFragment.time, oldFragment.flops + flops, oldFragment.duration + traceInterval, cpuUsage, @@ -139,7 +139,7 @@ public class Sc20TraceReader( ) } else { val fragment = - FlopsHistoryFragment(timestamp, flops, traceInterval, cpuUsage, cores) + SimTraceWorkload.Fragment(timestamp, flops, traceInterval, cpuUsage, cores) if (last != null) { yield(last!!) } @@ -165,13 +165,15 @@ public class Sc20TraceReader( uuid, "VM Workload $vmId", UnnamedUser, - VmImage( + SimWorkloadImage( uuid, vmId, - mapOf(IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems), - flopsFragments.asSequence(), - maxCores, - requiredMemory + mapOf( + IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems, + "cores" to cores, + "required-memory" to requiredMemory + ), + SimTraceWorkload(flopsFragments.asSequence()) ) ) entries[uuid] = TraceEntryImpl( diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt index 13bd7ac3..5f312d01 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt @@ -22,12 +22,12 @@ package org.opendc.format.trace.swf -import org.opendc.compute.core.image.FlopsHistoryFragment -import org.opendc.compute.core.image.VmImage +import org.opendc.compute.core.image.SimWorkloadImage import org.opendc.compute.core.workload.VmWorkload import org.opendc.core.User import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimTraceWorkload import java.io.BufferedReader import java.io.File import java.io.FileReader @@ -103,7 +103,7 @@ public class SwfTraceReader( memory /= 1000 // convert KB to MB } - val flopsHistory = mutableListOf<FlopsHistoryFragment>() + val flopsHistory = mutableListOf<SimTraceWorkload.Fragment>() // Insert waiting time slices @@ -112,7 +112,7 @@ public class SwfTraceReader( if (waitTime >= sliceDuration) { for (tick in submitTime until (submitTime + waitTime - sliceDuration) step sliceDuration) { flopsHistory.add( - FlopsHistoryFragment( + SimTraceWorkload.Fragment( tick * 1000L, 0L, sliceDuration * 1000L, @@ -137,7 +137,7 @@ public class SwfTraceReader( step sliceDuration ) { flopsHistory.add( - FlopsHistoryFragment( + SimTraceWorkload.Fragment( tick * 1000L, flopsFullSlice / sliceDuration, sliceDuration * 1000L, @@ -149,7 +149,7 @@ public class SwfTraceReader( if (runtimePartialSliceRemainder > 0) { flopsHistory.add( - FlopsHistoryFragment( + SimTraceWorkload.Fragment( submitTime + (slicedWaitTime + runTime - runtimePartialSliceRemainder), flopsPartialSlice, sliceDuration, @@ -164,13 +164,14 @@ public class SwfTraceReader( uuid, "SWF Workload $jobNumber", UnnamedUser, - VmImage( + SimWorkloadImage( uuid, jobNumber.toString(), - emptyMap(), - flopsHistory.asSequence(), - cores, - memory + mapOf( + "cores" to cores, + "required-memory" to memory + ), + SimTraceWorkload(flopsHistory.asSequence()) ) ) diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt index de6647d0..09f2764b 100644 --- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt +++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt @@ -25,10 +25,11 @@ package org.opendc.format.trace.wtf import org.apache.avro.generic.GenericRecord import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetReader -import org.opendc.compute.core.image.FlopsApplicationImage +import org.opendc.compute.core.image.SimWorkloadImage import org.opendc.core.User import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimFlopsWorkload import org.opendc.workflows.workload.Job import org.opendc.workflows.workload.Task import org.opendc.workflows.workload.WORKFLOW_TASK_DEADLINE @@ -81,7 +82,7 @@ public class WtfTraceReader(path: String) : TraceReader<Job> { val task = Task( UUID(0L, taskId), "<unnamed>", - FlopsApplicationImage(UUID.randomUUID(), "<unnamed>", emptyMap(), flops, cores), + SimWorkloadImage(UUID.randomUUID(), "<unnamed>", emptyMap(), SimFlopsWorkload(flops, cores)), HashSet(), mapOf(WORKFLOW_TASK_DEADLINE to runtime) ) diff --git a/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt b/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt index 40132ad3..78066c61 100644 --- a/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt +++ b/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt @@ -24,6 +24,8 @@ package org.opendc.format.trace.swf import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +import org.opendc.compute.core.image.SimWorkloadImage +import org.opendc.simulator.compute.workload.SimTraceWorkload import java.io.File class SwfTraceReaderTest { @@ -33,13 +35,13 @@ class SwfTraceReaderTest { var entry = reader.next() assertEquals(0, entry.submissionTime) // 1961 slices for waiting, 3 full and 1 partial running slices - assertEquals(1965, entry.workload.image.flopsHistory.toList().size) + assertEquals(1965, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().size) entry = reader.next() assertEquals(164472, entry.submissionTime) // 1188 slices for waiting, 0 full and 1 partial running slices - assertEquals(1189, entry.workload.image.flopsHistory.toList().size) - assertEquals(5_100_000L, entry.workload.image.flopsHistory.toList().last().flops) - assertEquals(0.25, entry.workload.image.flopsHistory.toList().last().usage) + assertEquals(1189, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().size) + assertEquals(5_100_000L, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().last().flops) + assertEquals(0.25, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().last().usage) } } |