From ef3868ec729f7ce3f5976d4f9a0c8b95098d9857 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 4 May 2021 14:40:46 +0200 Subject: build: Update to Kotlin 1.5.0 --- opendc-experiments/opendc-experiments-serverless20/build.gradle.kts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts index bdb0d098..88479765 100644 --- a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts @@ -38,7 +38,8 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.config) - implementation(libs.parquet) { + implementation(libs.parquet) + implementation(libs.hadoop.client) { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } -- cgit v1.2.3 From cc9310efad6177909ff2f7415384d7c393383106 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 5 May 2021 23:14:56 +0200 Subject: chore: Address deprecations due to Kotlin 1.5 This change addresses the deprecations that were caused by the migration to Kotlin 1.5. --- .../experiments/capelin/telemetry/parquet/ParquetEventWriter.kt | 1 + .../org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt | 2 +- .../kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt | 2 ++ .../kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt | 4 ++-- .../src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt | 4 ++-- .../kotlin/org/opendc/experiments/tf20/network/NetworkController.kt | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt index 38930ee5..4fa6ae66 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt @@ -52,6 +52,7 @@ public open class ParquetEventWriter( /** * The writer to write the Parquet file. */ + @Suppress("DEPRECATION") private val writer = AvroParquetWriter.builder(Path(path.absolutePath)) .withSchema(schema) .withCompressionCodec(CompressionCodecName.SNAPPY) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt index ffbf46d4..bd27cf02 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt @@ -105,7 +105,7 @@ public class Sc20RawParquetTraceReader(private val path: File) { val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) val vmFragments = fragments.getValue(id).asSequence() - val totalLoad = vmFragments.sumByDouble { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs + val totalLoad = vmFragments.sumOf { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs val workload = SimTraceWorkload(vmFragments) entries.add( TraceEntry( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt index 7713c06f..1f9e289c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt @@ -109,6 +109,7 @@ public class TraceConverterCli : CliktCommand(name = "trace-converter") { traceParquet.delete() } + @Suppress("DEPRECATION") val metaWriter = AvroParquetWriter.builder(Path(metaParquet.toURI())) .withSchema(metaSchema) .withCompressionCodec(CompressionCodecName.SNAPPY) @@ -116,6 +117,7 @@ public class TraceConverterCli : CliktCommand(name = "trace-converter") { .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) .build() + @Suppress("DEPRECATION") val writer = AvroParquetWriter.builder(Path(traceParquet.toURI())) .withSchema(schema) .withCompressionCodec(CompressionCodecName.SNAPPY) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt index 5c8727ea..6de3f265 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt @@ -69,7 +69,7 @@ public fun sampleRegularWorkload( val totalLoad = if (workload is CompositeWorkload) { workload.totalLoad } else { - shuffled.sumByDouble { it.meta.getValue("total-load") as Double } + shuffled.sumOf { it.meta.getValue("total-load") as Double } } var currentLoad = 0.0 @@ -129,7 +129,7 @@ public fun sampleHpcWorkload( val totalLoad = if (workload is CompositeWorkload) { workload.totalLoad } else { - trace.sumByDouble { it.meta.getValue("total-load") as Double } + trace.sumOf { it.meta.getValue("total-load") as Double } } logger.debug { "Total trace load: $totalLoad" } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt index 411ddb59..83995fa1 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt @@ -49,10 +49,10 @@ public class Sequential(vararg layers: Layer) : TrainableModel(*layers) { } override fun forward(): Double { - return layers.sumByDouble { it.forward() } + return layers.sumOf { it.forward() } } override fun backward(): Double { - return layers.sumByDouble { it.backward() } + return layers.sumOf { it.backward() } } } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt index 75b11423..9771cc20 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt @@ -82,7 +82,7 @@ public class NetworkController(context: CoroutineContext, clock: Clock) : AutoCl val target = channels[to] ?: return // Drop if destination not found - scheduler.startSingleTimer(message, delayTime) { target.offer(message) } + scheduler.startSingleTimer(message, delayTime) { target.trySend(message) } } /** -- cgit v1.2.3 From 9e5e830e15b74f040708e98c09ea41cd96d13871 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 27 May 2021 16:34:06 +0200 Subject: simulator: Centralize resource logic in SimResourceInterpreter This change introduces the SimResourceInterpreter which centralizes the logic for scheduling and interpreting the communication between resource consumer and provider. This approach offers better performance due to avoiding invalidating the state of the resource context when not necessary. Benchmarks show in the best case a 5x performance improvement and at worst a 2x improvement. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 2 ++ .../org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 8 ++++---- .../main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 7 ++----- 3 files changed, 8 insertions(+), 9 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 7c7f0dad..0dade513 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -48,4 +48,6 @@ dependencies { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } + + testImplementation(libs.log4j.slf4j) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 2d5cc68c..893e8bcd 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -113,8 +113,8 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(207389912923, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(207122087280, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(207380244590, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(207112418947, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, { assertEquals(267825640, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) @@ -150,8 +150,8 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(96350072517, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(96330335057, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(96344616902, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(96324879442, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, { assertEquals(19737460, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index f4c18ff1..f345a4a5 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -35,10 +35,7 @@ import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.SimResourceCommand -import org.opendc.simulator.resources.SimResourceConsumer -import org.opendc.simulator.resources.SimResourceContext -import org.opendc.simulator.resources.SimResourceEvent +import org.opendc.simulator.resources.* import java.time.Clock import java.util.* import kotlin.coroutines.Continuation @@ -67,7 +64,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - scope.coroutineContext, clock, SimMachineModel(listOf(pu), listOf(memory)), + SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), PerformanceScalingGovernor(), SimpleScalingDriver(powerModel) ) -- cgit v1.2.3 From cc87c9ad0b8e4ed3fa4fbad4ab94c5e53948ef3c Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 2 Jun 2021 13:03:10 +0200 Subject: simulator: Add uniform interface for resource metrics This change adds a new interface to the resources library for accessing metrics of resources such as work, demand and overcommitted work. With this change, we do not need an implementation specific listener interface in SimResourceSwitchMaxMin anymore. Another benefit of this approach is that updates will be scheduled more efficiently and progress will only be reported once the system has reached a steady-state for that timestamp. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 893e8bcd..4b21b4f7 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -114,7 +114,7 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, { assertEquals(207380244590, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(207112418947, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(207112418950, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, { assertEquals(267825640, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) -- cgit v1.2.3 From 1ee02377aca356a99506ac247b376d7cf070048d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 2 Jun 2021 23:06:36 +0200 Subject: simulator: Start consumers directly from workload This change updates the SimWorkload interfaces to allow implementations to start consumers for the machine resource providers directly. --- .../main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index f345a4a5..db131e82 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -116,9 +116,11 @@ public class SimTFDevice( */ private var activeWork: Work? = null - override fun onStart(ctx: SimMachineContext) {} - - override fun getConsumer(ctx: SimMachineContext, cpu: ProcessingUnit): SimResourceConsumer = this + override fun onStart(ctx: SimMachineContext) { + for (cpu in ctx.cpus) { + cpu.startConsumer(this) + } + } override fun onNext(ctx: SimResourceContext): SimResourceCommand { val activeWork = activeWork -- cgit v1.2.3 From 84468f4e3a331d7ea073ac7033b3d9937168ed9f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 2 Jun 2021 23:23:55 +0200 Subject: simulator: Split CPUFreq subsystem in compute simulator This change splits the functionality present in the CPUFreq subsystem of the compute simulation. Currently, the DVFS functionality is embedded in SimBareMetalMachine. However, this functionality should not exist within the firmware layer of a machine. Instead, the operating system should perform this logic (in OpenDC this should be the hypervisor). Furthermore, this change moves the scaling driver into the power package. The power driver is a machine/firmware specific implementation that computes the power consumption of a machine. --- .../opendc/experiments/energy21/EnergyExperiment.kt | 18 +++++++++--------- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 5 ++--- 2 files changed, 11 insertions(+), 12 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 7460a1e7..37e10580 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -173,23 +173,23 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { */ public enum class PowerModelType { CUBIC { - override val driver: ScalingDriver = SimpleScalingDriver(CubicPowerModel(206.0, 56.4)) + override val driver: PowerDriver = SimplePowerDriver(CubicPowerModel(206.0, 56.4)) }, LINEAR { - override val driver: ScalingDriver = SimpleScalingDriver(LinearPowerModel(206.0, 56.4)) + override val driver: PowerDriver = SimplePowerDriver(LinearPowerModel(206.0, 56.4)) }, SQRT { - override val driver: ScalingDriver = SimpleScalingDriver(SqrtPowerModel(206.0, 56.4)) + override val driver: PowerDriver = SimplePowerDriver(SqrtPowerModel(206.0, 56.4)) }, SQUARE { - override val driver: ScalingDriver = SimpleScalingDriver(SquarePowerModel(206.0, 56.4)) + override val driver: PowerDriver = SimplePowerDriver(SquarePowerModel(206.0, 56.4)) }, INTERPOLATION { - override val driver: ScalingDriver = SimpleScalingDriver( + override val driver: PowerDriver = SimplePowerDriver( InterpolationPowerModel( listOf(56.4, 100.0, 107.0, 117.0, 127.0, 138.0, 149.0, 162.0, 177.0, 191.0, 206.0) ) @@ -197,17 +197,17 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { }, MSE { - override val driver: ScalingDriver = SimpleScalingDriver(MsePowerModel(206.0, 56.4, 1.4)) + override val driver: PowerDriver = SimplePowerDriver(MsePowerModel(206.0, 56.4, 1.4)) }, ASYMPTOTIC { - override val driver: ScalingDriver = SimpleScalingDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, false)) + override val driver: PowerDriver = SimplePowerDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, false)) }, ASYMPTOTIC_DVFS { - override val driver: ScalingDriver = SimpleScalingDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, true)) + override val driver: PowerDriver = SimplePowerDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, true)) }; - public abstract val driver: ScalingDriver + public abstract val driver: PowerDriver } } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index db131e82..001547ef 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -29,11 +29,10 @@ import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine import org.opendc.simulator.compute.SimMachineContext import org.opendc.simulator.compute.SimMachineModel -import org.opendc.simulator.compute.cpufreq.PerformanceScalingGovernor -import org.opendc.simulator.compute.cpufreq.SimpleScalingDriver import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel +import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.resources.* import java.time.Clock @@ -65,7 +64,7 @@ public class SimTFDevice( */ private val machine = SimBareMetalMachine( SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), - PerformanceScalingGovernor(), SimpleScalingDriver(powerModel) + SimplePowerDriver(powerModel) ) /** -- cgit v1.2.3 From 38ef2cabbecf694f66fa3bd5e69b9431c56a3f8d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 3 Jun 2021 15:18:22 +0200 Subject: exp: Remove Jupyter Notebook from Energy experiments This change removes the Jupyter Notebook that was shipped with the initial version of the Energy21 experiments. At the moment, it is not up to date anymore and we will probably move the plotting facility to the web interface. --- .../opendc-experiments-energy21/plots.ipynb | 270 --------------------- 1 file changed, 270 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-energy21/plots.ipynb (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-energy21/plots.ipynb b/opendc-experiments/opendc-experiments-energy21/plots.ipynb deleted file mode 100644 index 7b18bd2b..00000000 --- a/opendc-experiments/opendc-experiments-energy21/plots.ipynb +++ /dev/null @@ -1,270 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as pyplot\n", - "import seaborn as sns\n", - "\n", - "sns.set()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.read_parquet(\"output/host-metrics\")\n", - "df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x = df.groupby(['power_model', 'host_id', pd.Grouper(freq='1D', key='timestamp')]).mean()\n", - "x" - ] - }, - { - "cell_type": "code", - "execution_count": 125, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 125, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEJCAYAAAByupuRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACQUElEQVR4nOydd5xdZZ3/388pt987vSWTTAqQRg8EAiE0CU0UsQGu2FZ0FQurrrv21VV01cW2/lbXsoqiIqIUEaS3QCCUQEhvk2R6v/2e8jy/P86dOzOZSTIpk0yS83698oK599xzvueW5/s83+f7/XyFUkrh4+Pj43NMoh1uA3x8fHx8Dh++E/Dx8fE5hvGdgI+Pj88xjO8EfHx8fI5hfCfg4+PjcwzjOwEfHx+fYxjfCfj4+PgcwxiH24B9pa8vg5T7XtpQVRWjpyc9ARYdGJPVrj0xGW2ejDbB5LRrMto0Hiaj3ZPRpl3RNEFFRXS3zx9xTkBKtV9OYPC1k5HJateemIw2T0abYHLaNRltGg+T0e7JaNO+4IeDfHx8fI5hfCfg4+PjcwzjOwEfHx+fYxjfCfj4+Pgcw/hOwMfHx+cYxncCPj4+PscwvhPwmRQo10E51uE2w8fnmOOIqxPwOTqR/W3IdC8iEEJEy9HitQghDrdZPj5HPf5KwOewo6SLzA4gwnEQOqq/HaRzuM3y8Tkm8FcCPocdZWVBSYTQQNdQQoCSh9ssH59jAn8l4HPYUZk+hG4O/Q0gfSfg43Mo8J2Az2FlMBSEGRz2oADpHj6jfHyOIXwn4HNYUYUMoLxQ0CBCofxwkI/PIWFCncDdd9/NlVdeyZVXXsm3vvUtANauXctb3/pWLr30Uj7/+c/jOP4G4LGMyvQjNHP0E/5KwMfnkDBhTiCXy/H1r3+d2267jbvvvpuVK1eyfPlyPvOZz/DFL36RBx98EKUUd9xxx0SZ4DPJUdJB5vpHhoIAITSU6zsBH59DwYQ5Add1kVKSy+VwHAfHcTAMg3w+z6mnngrANddcwwMPPDBRJvhMduwCwMhQEAAayKO3cMzN9KH8lY7PJGHCUkRjsRif+MQnuPzyywmFQixatAjTNKmpqSkdU1NTQ0dHx0SZ4DPJkUUnMApNg6M0TKhcG9nbglZjQih2uM3x8Zk4J7Bu3Tr+9Kc/8dhjjxGPx/n0pz/NM888M+q4fa0Krara/x9OTU18v187kUxWu/bEwbC5oPqQIoGT7Kbv6TtJnLaM8IwTkbaB0HVC+3iNyfo+DrfLTvVSCEkCYZdA9eGzd7K+V3tjMto9GW3aFybMCTz99NMsXryYqqoqwAv9/PznP6e7u7t0TFdXF7W1tft03p6e9H61c6upidPVldrn1000E22XGizCOogcLJud9i4QgsILf0f2tdP76K8xZp6BMf9ihKZjBMZ/jSPl83Xat4Frona2YsjEuCdBMpdCOXm0WPUBy2lM1vdqb0xGuyejTbuiaWKPk+cJ2xOYO3cuy5cvJ5vNopTi0UcfZdGiRQSDQV588UUA/vKXv7B06dKJMuGYRxUyyP7JGW5TSqKcAjKbRLZvwDhhCcZx5+BsXYn14l9Qu8hGKLtwxAvMKacAVg4RCCNcG/bhflSmB9mzHdmzHeUenaEyn8PDhK0ElixZwpo1a7jmmmswTZOTTjqJG2+8kUsuuYQvfOELZDIZ5s+fzw033DBRJhzzKNdGFdKH24yxcW0E4Gx9HoSGOfssRCiOsjK4HZtBOiilSrNemekFI4geqzygyyqlkKkutGjFiCrlQ4HMplBCIPCqoqWVRS9mRikrCwhEIDzaZukicylEpAKVT+F2bkavnXXI7fc5OplQ7aAbb7yRG2+8ccRjc+fO5c4775zIy/oUUXYBZedHDKaTBsdG2nmc5pfRG09EhLy4qggnwMqgFJ5+kNCLx1sUBSUOjEIG2bsDCjm06umH7H1RSqEyPQgzBIAwApDph2gFSro4Xc1o0Qr0MZwAVg5UsaAuGEMVUshkJ3rF1ENiu8/RjV8xfDRjZT01zkmYjijtPHLn6+BYmLPPLj0uglFvwLNzI+xWdgGs3WQTjROlJG5fCyIYQ+b6kameAzrfPmHnwS4Mzd6NALKQ8Wb56R6wMt7nNQYyl0Ro+tADgSgq1YOycuO+vJvsws30Hcgd+Byl+E7gKEY5BYQS4NqH25RRqHwGZ/sraJXT0CqmlB4Xwaj3fCE7QklUuRbSzh/QNWU2ibLzCCPohZ76W4qyFROPzKdh2KrD26yXqFwS1d+OiJQjxxjUlVLIbD8UVxCl1+omsr8Npfa+OlL5NLJ3O+QGDsat+Bxl+E7gKMXbeLVRujZqk3UyIAfaUNl+9Oknj3hcBL0sBmkNrQSUdD2H4Nqo/VQXVdJF9rciAhHvOkIDM4zTvc3bsJ1o8kkwRldGy4EO0HSEZiCUi9rVYdt5hHRGrgQAEQgj86m97vkox8LpafZWP/n0uJyGz7GF7wSOVlwHgUIgUM7kWgkoKb24PIyKaw+uBLAyMDhgSReB8OL3cv/uRWb7QTojNlOFEUCg4XZtGz34HkSUlMhCFnQTt2MT9oanvSfMkLdaG1z9IEZlDMlCZrc7IcIMI/vbdn9dJZF9OxHgrX6UBPfIzrDyOfj4TuAIZ7czO9f2Bg9N9+LRkwnXQg20ezPgxMg6keHhoJK0QjFTSKFgf9Mjs/2IXWbi4M2okQ5ud/PESTk4BUTxc7JeexD79YdRhQxCM9DCZSMO3dUZqUwfwggxFsIIgJXf/UqmkPWyigLF91QJb2/Fx2cYvhM4glFWDpnsHPs51/GSaTSjpNEzWVCOhRzoRCurR2i7JKgFwoAAK1calL28eAFq9CA5rutJB1nIgB4Y83kRiHrvZV/LhIRLpJVDCYHs3YlKdQHgdm4ebYemo/JDexTKtb26AmNsu6GYapofe19DZnoRw+5ZaJq31+LjMwzfCRzJSHe3s3xl5704sqYf8Ibq7lBWDre/fZ9fJ60sMtkxYkN4EC8NMuLlzRcHfOU6CFEcJMfh0Eb1IrDyoPYsUaKF4shM38RkDOXTCN3E2fYi6CYEwrgdm0Yfp5vFegEPVcii8O7H2fkabudmZKZvxP0JMwjZ0Vk/ynVGN+sxAqj85K5u9Tn0+D2Gj2CUkrufGdt5lNC8QbW4oSq08fl8N9WNFqvaaw69LGSQ2T708vp9s7t7OzgWWvmQE/D6DCtEMIoIejPzwdCPymdwdq5Gn34qYi8OTSmJ27kVvbLRGyDxMnMGN1ZlpheVS6FXN416rQjFUX07UYEQ4iCJuymlkPkUIHBbXkdvPAmkjduxabSkh6ajrGyprkNl+xBGAHfHa1gv/nnIzlgVofPe69VW6AFkIY3mOgh96Ocs86mh2oLS+Q1UIeUrmPqMwF8JHMkouVspBZlLkn/oh7g7X9unDVVl51HJzvGlleYGwC7sU/aRkhK3eysAWnFTWDmWt4UdjCBzA96+wGCNA+Bufxn7tQeR/e2ovTmBQgaV6UWme4cey/WDEURZOQpP/4rCU7/EevVvo+QXvFVIFKfn4O0PKMdCKBe3dQ24NsaM09Hrjgcri+xrLR5TwN70LEhZ3Ly1S1XCGAGc5pcR0QqCS96DecoVqFyS/LO3e5vKQoBSI1YQACrZWSpMAy8kNujUlTXJ9oh8Diu+EziCUdIF1xkVx1ZKInt3gpXF7dq6TxuqMpf0cuf3MrAr6XoZL5q2Txo4MtPrZbToJiJe7YU27BxG9XS06hlolY0II+jZUHREMunF0VW6C+UU9hi3V+keRDCKTHejXMdzkrYFmo71yn2oXBJ92kk4m1dQePIXyF0KqIRuIly5T/e0J5SVRylwt72ISNSiVUxFr50NgNuxEQB79UPYrz2I274epbwMocGVkcr2I7u3YTSdhl4zE3PWIoKL3o7qb6fw/J0o6SJ0E5XpH3bNnLcZXdxLcDs2kbv3Ftzend7fvhPwGYbvBI5kBvPn1S6zVtdBJT3hOJXsKG6o7t0JeLo6PV7YYJe0UpkdGFmhaufxtiXF7vsC7Hp+1/YKnJLdaOUNXgexfBpRVueFgYRAj1UhwolidpBns8z0Fu+lc4/Fb8qxUNkBMMNQLLJSVg4lwN3+Cm7L65jzLiR4xlsJnPVOZKaX/GM/wW1bP/I87N8G9Fi4uRQq7Tk+Y8ZChBCIYBStohHZsQm3exvO1pXeffZsRwiQxfsQuoHT/Aog0KefUjqnXn8C5qlXIjs24mx4Gsyg57yl9NJRM71QDAMpO4/18j0gXdyWNQg94K0wfHyK+E7gSEY6nhPYtYDKtXEHvKwhmez0EmvGUxBl5xCujTDMUcerwRn84KULGVSy23NE48w4kclOUC5yoB2tYqoXKgmE0OK7pImGE+BaXlaNlKhsv/f6gQ6UULtdpchcsrgPUgwtJTtRmX5UPou16n606hkYJywBwJgyj9CFNyIi5RSe+x3W6w+XNlyF4KAplrqZpDfjFwKj8cTS43r9cci+FqwX/4KIlKNVNOJ2N3vZXIW058CMAO72V9DqZo9KJTVnnoFWPQOn5fVS9bHsa8FpXYNMdUOxKM5e/RAql0JEKzw7dNMTsvOLxnyK+E7gSEa6ngPYJX6tBlcCQnirglzay5AZg+GDgcwOeOEdzRh1vLRyqHyqFJOX/W0Ulv8Gp/nlcUkvKCuHSnWj8l6sXyufgnLyiFjVqA1rMTjgFbwMoZITSHV69QJjrGoG1UFLAm2aAa6NzCdxW14D1yGw8OoRG6VatJLQ+R9AbzoNZ8PTuNtXFZ8wwB6/Ls9u79mxkE4et209WvWMoUI4QKs73jsm20/gtKvQ6majBjo8p5dPg5Sorm2oXBKj6bQxz6/XH49KdiKzAwgjhMoNIMwwWrgMITTczi04217EOO5sjJlnolJdnoSEYyFT3Z7KrFKoQga3cwvubtKNfY5ufCdwJDPYjH0XJyDzXgiiNNCku0dtqCorh9u1FbdtfTEjRXqbqWbYCwcNGwTVoAidpiPTvSjX9vLclfRm505hr3IOMtPnnXfAW01oFVMQCrTijHU4Ilru/U8+jSykoZDxVgd2AfIZlD16li4LWXCskRXBZhglXdydq9FqZ6FFykdfSzcJnPYmtIqp2Gse9VYAunHAm6fKsbz3N9WPSnejT5k39JxdQJQ1IKIVGDMWotfORq+aDihkf6u3QtJ0nOaXIRBGr58z5jX04ucrOzd51c/BWCkLSimJ9cp9iFgV5vyL0OuO847t2IgejiEH2nFa1+G2b8Dp2IiyC8XP0q8oPtbwncCRjHRBiFF58bJrK6Awpp0MCGRqaENVSU9J023f6A10QuC2b/Q0bKRbqi0YsQHr2AgUBCLIdA9uNoXs8TYZVbLTe24vcgQy5+Wsu707PTG0SDlKiJF57EVEcbBWdg414O1t6A3eQKjSPeCMnqW7mX6EZnj7AkU9HWEEIDuAyg4U34uxEUJgnrgMlU/hbH7Ou/+iBPf+MOgAkC6Ftk1F++d6z0kHZWUQrkXoDR/FPPVKALTKRhDCk9PQNJTr4LauxZh28ojUzxF2x2sQ4cSYNQdu23pUphdz/sXeZne8BhEuw+3YhNANtFDcW5kIDS1c7jW6ERxaZVWfSYHvBI5glHRBaKPCI7J7OwBa1XREvKq4oeoN1LJnuyddHIp7P3wjCMEYMtlZmkULIUZswKqiBIUQGiiF3d+J7Gvxnsv0Ih1njzNI5VgI10bl07g7XkVvmINwbbRQbMzWl1pxJaAKWeSAV4w2OBuWmZ4xZ+luNglGgMKKP5B76L9LRVHujldBN9GnzN3je6lXN6E3zPV0fQpZz7Hth/CeUgq3e5vnUAMR8s2veUqp4YT3vJVDRCuKs32jdP/CCKKVNSB7mtFCCZzml0BJjFmLdnstIQRa3XG4nVtGpbQ6m59DRMpKzlMIgV53HG7XltL3RQhtZGOaQBSV6jw0gno+kwbfCRyhKNch/9T/eTPrYYOVUgq3vxWCUUQojpao82b5CNyeHchcEi00sret0HQvjjysoYkSlJyA07aBwjO/QRXSiGAEu6/dcyyDuj/Zvj1q2yungALsNY8BYM670Mu+2WWzs2TP4EqgkEUWN7i1snpEpByVHJ0mqqSLsvLIZCeyczPYOS8d1HVwWl5Hb5g7pm7QrpgL3gCug73ucU93aX8yhAoZb6APRJCZXuye1lIoaHDFpidqGasOT6uajuxtQdkFnK0r0eqOR4tV7fFyeu1x4BS8lOAisr8N2d2MMWvRCPVRrf54cCyszm1jnksIbz9o8D33OTbwncARiqdD34rqa4Hh6ZzSRQ20o5VP8WaKZXWobH8xXz5fmpHu/QJDG7DujldRyU6cbS8hNAOZSwIKY9aZ3iUzvXvMEFL5DCrVjbtjFcbss9Ei5Qil0MbqokVR1E0PoKyst2rRTQhG0RK13opFypEDtJ1HoXDWPwVmCGPu+bht67Feuhvs/B5DQcPR4tXojSfi7FztbdDuJq1W5dOjpSmKyGRnyeG4resAhvYDrJzX1jIYRenmqCI7rWo6SAf79YegkMGcfdbI60oHVUgj88nSP618CgitVHMAYG9eAbqJ0XT6iNfrNTNB08nvWLf7NyEQQaV7D1mfBZ/Dj+8EjlBkMe7tFVUNhWJUIeNtCpc3AKAl6rzHrUxJq393KKUovHwPhZfv8cTGimGBwdCPs/VFb8OxoxmEhtF4shc/T/cgrcxuY+gql8Re/xSYYcwTlnhyF5o25n4AeJk9oqgfpLJ93srAtRHxGlS6GymdESsPaedxB3pw29ZhzFqEOfd8L+Vy52sQiKDVzhppTyHjDaK5pLeJPsyhaBVTPQ1/Oz96M10p3IE2nI6NXsvHXe+zmEElzJB3bMvrmFVT0aIVRUOd0sxei5SPEvbTq6cX3+eViHg1WrGoDCjZKeLVGLWzMWpno1fPAF1Hq5xW2hdQhTTuztcwpp86ql+xMIJoVdPJ71i7WycmhIBACLd3x24rwZUco0BRupOyb4XP3vGdwJFKvugE8ukRM1bZ0wyoki6PKPOcgCwWj5WO628jv/w3ONtfKT3mbFmBu+0l3B2vodA8JU/X8WQkglFUbgDZvpFC+5ai5INCRCuRyS6EVGOGT5R0cDs2Ibu2Ys5d6g1MjrXb/YBBRDDqhVYy/V4M3S54zkApyKdQw0XT8mky658D3fAa1guNwMI3g2Z4G6uljBnltWoMxdCrpqPXzPCcpZKo3ADKsdCKIS6Z6RuRJuv1QNiJHOhEaOaYBVcy7RXagbd6kn0tRI4/w3u9nUcU92HAE6zbNY4vgjFE0UkYs84aknmQDmgaRv3x6GX1JX0lLZxAC0bQamaiBtopPPs78o//DKSLsesqwimgnALGjNNxBrqw1z4+8vlh3yFhBFGOXQwjjqT0eQ6X5VAS2dPsbyofofgCckcog8t1lU+O+AG73c0AaBUNXqaM8Gbcg1k2Sro465/CXv8kAFbHJlSmH612FvZrf0eEYp5jyfSidN37bz6FOf8inC3PY29ajuzZiXH8Od7mZrwG1bvDq7IdJlVQwi7gdmwA3cCYWRwQXRsRGlkgtisiFPeymnID6HXHIYRCi1cD3gAtolVoxY1xt6+FwtZXMWadWcrF1+I1hC65aag/gZKoXAotUYNWXj/CAalYFSqXxO3eNuQE0j0j0mRlqsuTZg4lvIK3TB9aWd3QQO1YyHQvIhRDZvu94rSq6UTnLqZ/IO85mMphDXQCIa+Ib1AsrpABTaDXzMKxshjDKoRVIYdWOWW07DZAqAy9dhbutpXITC9aWR36nKWl92rwM/cqvAVG40kY/c1k1z+JXjkNrWYG9uuP4Gx5nuDi60uppCIY8+ougjG0yNDejUx2eUqufS2e0F4w6u3FpHvR4v5wciTif2pHKENOIDViY9jt2gaBCCKU8EITgtLmsBzooPDin1ED7ejTTiJw4qVYrz+Eve5x2PAUIlJGcNE7yD/2E2R/GyKS8M6HFybRm07HKToPvbqpeO5qnNY1IB3vNXXREQOsLGSQXdvQqmcOZR+x+/2AQUQojmrfCMr1NoQBYlVe+CnZCTWzvJCMYSLbNniZNLvMfofXBah8Cq28Hi1RO0odVQgBYU+Rc3D/QaW6ixvQ0pOgKA6IQggQBrhZb2At3ofM9JV6CFsr/wwoAgvf4oXVXMfTShpWLCY0Ay0Y9TKvXAd0A+w85omXYMw5r+RMlWuDbqBFKsZ8n7RgGBmtIHz5p3f7XqpCBq18Kirbh3IsyhdfTa5jB4WVd3lOP9UFRgB77aNotbO97DAhIBDF7dkOogktnPDCXckuT73UdXC6t6GVNaD6O7zv22RrXuQzLiYsHPTHP/6RN7/5zaV/Cxcu5Ktf/SrLly/nqquuYtmyZdx6660TdfmjntLGnWN5M/7BBiwD7WiJGu+HjFcwJeLVyP5W8o/9BJVPETjrnQTPeCsiFCNw+tUYc88HM0Rw0TsQZfUQjHptCZXA7dwCeNk5xoyFgAAhEBWNKKGhVzR618152kK7CrLJ7mavl3BxhqlcG4zAiMbpYxIuK2kiiUgCYYbRIwlErMrbfNV0TybCLuD2bEdPVKFFK8d+rxwLYYZK78tYCKGhJWpQVhYtUYtKdSEAXMcL/Ug5ss+vEKVmLsqxPJuCUZzNK5A9zQROvqK0F6DsXPHau/zcwmWetLMQ3qZtMAqoEZv3ysp6Oku7kwE3Q4Da/X6MnUcEI2jxSm/VZufRjADBs94BykVZOYLnvIvASZci+1qRnUM1B0I3EWYI2bUVN9ntZaIZAS+11AggEMjuZghGvAI7P7X0iGTCVgJvf/vbefvb3w7Axo0b+ehHP8oHP/hBrrvuOm677TYaGhr40Ic+xBNPPMH5558/UWYctYzI3sinRlT06lPmeoOtGUTEKtESdbjSRZ8yn8CpV46ckQpBYN6FmHMvKA2QekWj1wVL4BUvheLesWYQfdqJGG6+qM8TQxZ1+WWyE6NyGrK/DS2c8JQtpcRpWeOdc7B62c6hVU7fa68CLToUghDBYiw9EEFL1OJ2bPSyazJ9COki+3YSme1JKyilPPG1YLiUpaPsHHrtrD3uQQBokTJvNROvwW1+GakUumujUiNlmQGvP3CmBxLVyFSPtwhwbez1T6HVzi4JvimlQKkxs7K0YBQViKBXz/AG1Vi1JyJXsruACITRIrvP6BKajjDD3n7MLqE4pSQ4heKqTUMLx5FCQ0kXLVZF6A0f9XoPmyFPanrdk9jrnkCrPa70+QjdRIXiXnKAJtBCQ7aIQATM0ND7Wswo211xm8/k5JBsDH/lK1/h5ptvZseOHTQ1NTFt2jQMw+Cqq67igQceOBQmHH0McwIqn/Y2N62sJwIXqwQn78XNg1H0KXMJXfxRAovePsIBDGf4oKxVNqLSPaViLa2s3ps1uhaBhW+h6tIPoFwLinUIGEGvbkDTEXibzm6yE7djI7JrKyJaiRar9MIemjmuNFUxvIYgFPEcQCDsrSjsAqp7G0K6ngKoYxGa4jkZrAxazCvGUo5VnAlH95oZBd6Ap0XKvWI110Zlk6hccswWj0I3wS54q5FUJwSinhqolcWcN+RQlZVDi5aP2SJSBMLodccPbRaH417xn5LFATyPXjltr85rUHBvFIUMoqy2dH6h6WjxKmQhV7xe2QitJeOEJd7md9fWkecXGlqkbIQDGP7c0B/sV4Gdz+Flwp3A8uXLyefzXH755XR2dlJTU1N6rra2lo6O0RkIPntHWdlSDFoVVwJusbpWi1aBVGihKBhBLx4dq9zr7HsQrbIY4ulvQWV60crqvQ1mzfTCIkJDAFogjGYG0eLVQ72OA1Fkph810On1HOjdiV5/fMlmrax2XB3OBp2ACMURWgDNCIARRFQ1gRH0Vhia8IrDgEDD7OLgqdDKGzDqZnvhCTuPXt4w/nuPVSIG4++ZPk9C2y5QWHkX9qZnPeG1YuhF4SmbohkgXeyNy9FqZqFXTiudTzrWHgu+hs+ahaZ7x1q5osR2/ag0zzFtDkZRcnRPCUXxuzD82GhFKcy2K0bTaYhQHHvto7ttquM0v0x++W9GxP+VY2Fveg7pWOPuW+EzeZjwddvvf/973ve+9wGMGbcc749zkKqq/W/7V1MT3+/XTiT7Y1erslCxStxUD0EKVFWEyfYNkAPK6urQ43HCU7wYuKVNxUn1oodGi7WNhYwdR9szAtG+DpQiVj+FSF0teiSO3eM1Yy8rixBuqAYlaa+qI799LeXl4eLn6a028jvWkZcOZbNPJFgWRIZjhJumjYyt74aC20gLYJZVUV4eIlRfiWYGyTv19E2fR37nesqXXkNPshWzcgp6KEqZTGLUzyBQ5Q18blUMN9NPoGr87S+VipHNzaJjBQRlknDQIbt9Lfkdr+LueBX7tQcJTptL1Rveh4obyFwKvaqazJpnyBUyVJ65jGCF9z57fRbC1EytG/f3XCZ0cs39aKEyQo0zx/VeKTdE1mrDiAx9vm42hVHfVHovhoiTbxsgoWXRQ6NXhZkzL6f/qTvg9fspP+/tI2b6dl87nav+6inTrrqbije8B6Sk5+HfYbespyz2FsrKT8WIT9zvbDL+hiejTfvChDoBy7J44YUX+OY3vwlAXV0d3d3dpec7Ozuprd1zquCu9PSkkXLsTbA9UVMTp6tr8jXT2F+7rHQSzAiYGXIDfXR3J7FbvPTQVF5DCwXJdA/WEug4vSm08NBApFzbK7gSXmx31/RDkaij0LoBgJyIYeU1NKHj9mcoN4Mk84JMj1clbAcrUVaOnnWr0OtPGLJx82rQDbKhejLtXWjlU0qv2RvS8jKJ3ECC/oE8Rl8BISzcvIlbPgO15RV6NqzG6t6JMWsRSrr0J/MY0TBixPuZgH18f13pSW5kuzuwpwYobHgBrf4EAidfjrP5OQqbV9CzeQN61TQgjOpJkV/1GFpVE9lgPdm+rDeTLqSpW3Aa3cXPYfzXj6HpVaTH+V4BOBkJ2QEvhq8kqpDFiIR2eS88qqqn0f7aiwjTHR2/r56POe9Ccmsfw5Y65smXeyms0iX/xO9AD2Acfy75dU/Q+fS9qNwAbovXlCfT04Xd3o2en5hhZTL+hiejTbuiaWKPk+cJDQetX7+eGTNmECnOUE455RS2bt1Kc3Mzruty3333sXTp0ok04ahFWdlS2qHKZ1CO44VkzJBXRTo87j4sJ13ZeU/RU0m0yqlecxc7jyqkR6zU9GJICCMI4QRaIOxtXoYSnlhbaChmbzadjkjUUnjudzjbXvSqZftaijr6M0EzvPDRHjY4d0WEYkWpiDqEGSrNpLVgGFE9HYwg9usPg3TRa2Yh8xm0RP1B2ZQUoVgxC6kL2b0VlU9jNJ2GFq3AnH+R1/d36wul451tK71airlD32VVSCMqpnohuX1Er5o2rjDQCJvDZV61ePGz1GLVY+5DAF4Ir2o6ykqPuTo35izFOG4xzpbnKTx7O86OVz2Z7f42Aqe+EXPuBegzTsfZ+AzuztWeUmko7nV18zOEjjgmdCWwY8cO6uuHluLBYJBvfvObfOxjH6NQKHD++edz2WWXTaQJRy9WDhGtBBXzpJOl5VXuRsq9Dbphg0gpJz03gAgnvBlsIDI0sIbinpZ8pg8phJeRUtEIW1d6BVEIzxkAIlGNyO5ECw47f6yS4NnXYa36K9bL9yLWPeFtqGo65swrvH63ofhuB6WxEEIjfMlNSNsaOSCaQYRuojfM8RRChYZWPR2F2icns8drG0G0WBXO9le8zd5gtLTCEUYQY9opOM0voU66DDQNe92TaDUz0Wo8eQpVSKOFy/cq/nYw0WKVxX7PBYQbQovvRXguUgbxGq9/s2agkF6mkBEsSWsLM4iz9UWsoi6RPu1kjKnzAQiccgWWY6HFqzFOWOJlbGX7/VqBI5AJdQJXXHEFV1xxxYjHFi9ezD333DORlz0mUFbOGxA13Wsn6DiodI9XDGUER8WStfIpUCbHzA4SRgC9ahqqrBaZTSEH2hDlnvMWiTqEGSzNsEUgimFUgTak+yMCYdB0gmdfh73mUWSyw5stTpmHCIQ9obOKKft8jyIQReTTpVaJMOjQIuj1J+DueLW4iS3QgtF9cjJ7xAhCvBqki+zYiHH8uSPeT2PWGThbX8DZ/rI36FlZAgsuKYVNFKBXTtnn/a4DQRgB9MS+hVa1snrQgyBA03Rkz3aUZnhZXkJgzr0AY85SL2OoZwfGzIVD19MMgme+bejvaCVu52YvI6tYBe1zZOAn9B6BKNf2BNWMoDdAFuWLVbYfUX+8lze+C+MJLwgjiJ4Igq7j9mzHOGGJJ9kwLL1SaBqB2lmIYXFuYZje3oKmEzjxkpG2Kglou01N3SNm0Mux30VoTsRrEBUDiFAcfco8lJ3HSDTBQYpECE1Dr5jCYJ6LMeN0L71VFJ1Qog6tajrO5udRVga98cSSk1NWxlNwHa7TP0kRmo6eGCYv4RarvoeFEoXQPJ2lqul7PJcWq8Td/oqXqiq9CmmfIwNfQO4IRA3KNpuhUiql29sMSnqVtmO0bNwXtHAZQg9gzj3fSy0NjdxUGjXL0012O++z8l6e/DiyXEahm0P/hl8/GEMYAYKXfBxj9tkIQA8f3AwNrXpm8b9NaLEqlJXxWlsW1TeNmWegcgMgJea8i4BBoTd9SDX0CEOLVSECIU8baA+MpUAqBqujM0k/TfQIw3cCRyDK8grFBqUQAGSXlxkkwuUI88DCIkLT0MobUHauKD2x54YsQjdRxSKnUbZKZ78HRaEb3mpgVyegaWjxaoSb9ypli/UKBxMtWo4x70ICJ17qZfpoBqJiCiqXQimFPmU+Ilzm9UeIeXIVqpD1mt/sj8ObBAhN8xIC7PxupaaVa6OKEtwjXlusKpe5Pl9S+gjDDwcdiQxfCcSLTqDYWUqLlB2UUIQWTiA1E6XcUXIEYx4fCHuy07rhzQSVLElZMI5QlCwOOtqwvHShG1617xjxZS1SgTvQgVKF/dpv2BvCCGLOOM0TRiuk0OK1njKnnfcURMMJQss+DkV7PaE3c7dCb0cKIhDxnF1/KyoQGfFd8tJeM2jVM1DpblQh6/V9sPNF3SNQ2QGUM1pS3Gfy4q8EjkAGdYNEIIw+uBLobwWhQzgGxoE7gcHVgBaK71W2APCUN+2c11pR07zBJFaBXjV9r69XSrEz3cbGvi305vtwB6tVzdBuNzuFGfRmn9JBG4ckxD5jDFvdSOk5VyHQKqaUQiaDG6gwDqG3Iwg9UYNe61Vcq0LGm/1L16tirmxEj5ajV01HCeHJZiiFUX+CJx+SHQDHzxA6kvBXAkcgyiquBIyQJ68sdG+jOFrhKT+OpTu/H+jRctQ4Y+2DM+X9uXbKSpMsJIkYYdrTnXTrvcwum4G+l1WESNQg1N7DVfuDEJq3urGyXjptSWNHR6+Yitu+CWUEvIygcQi9HWmIUAyj/gSv9sTOI+08WqK6lPYqjABGzQxPW6piKmg6Ilru7ZPsZU/BZ3LhO4EjEFnsKiZCMTTd9DThcwOIUAJtjMygA2G88W1vBrzvs2BXurRlOggbYQzNIBYwSNkZbGl7TmAPaMM6dU0IoTgq1V3sojaECEYRsXJULu2FQZw8eu3s8a2YjiCEESgVDY71SYhA2JPAHvw7UoEa6PAlpY8wjq5v7bHCYDiomLUjijNQESkfV/x9MtGb78ORLubwFYRS2OPcXDxYq54xzx0IQyCKFhq9GtLK6kE6qELG24wPTUBI6ghDi1d7LUit/G77G/hMPnwncASi8hnQjFJGjCh20BLRckRgL81aJhGWa9GZ7UYXghc7VtGR7fJUQIVGfhLElYUZ9lpRjiFFIYwgoqwO5dpo5XX7dN6jdYDUErVeQkAuNWa/aZ/JiR8OOgJRVnZE6qQodtQSkZG69a50saVNyJicjmGgkMTF5Z7ND7Az3QpAWSDBovrTiZonT9h1806egB4YkYk0FkI30If16t0VLV7tbYDvrUvaMFzpsj3VQn20hrBxZK3a9oYobwBAZvuRKU8w0K8cnvz4K4EjEFXIeNXCRScwmKeuRcpG5NQnrTStmfbDYuPekErSnevliR3L2Zlu5dKmi7is6WICusmTLc+SHdbkHaAv30/azuzmbOMnWUixqX8rzcmdFMZqxLIPCM0YV4Oc4XRkOklZKVrSbaW0WICeXC+pwu7VRruy3Wzq30pvvg97ks6y9bKiTpidQya7kamew2uQz7jwncCRiJUZsRIwZi7CmHmGp3czLEbel+8nY2XHHV8/lGSdHM+0Ps+6vo2cP/UcTq05kVNqFnB67SkU3AK9hWGpokB/foC+/MABXTNZSLE91ULUiGC7Fpv6t9CX7z/AOxk/A4UkvYU+ygIJCo5FT64X8D6nnek2uvNjD5o5J09nthsUtGU62NC3hc5M94j3Z29YroUzwd8DkagGTffafobjqL4W3OyBfWY+E48fDjoCUYXcCJE4rawWY84StGHKoLZrk3dyaJoXXzcDk2vjcn3vRl7uepWTq+dzVv2QMFlD1KsL6Mp2Y0mbsKbjSpecmwc37/VK3o+K3KydZUeqhYgRQtd0dE3HlC4t6TYKboHaSM1ew0MHguVatGbaact0sjq3jrPqF3oDO4L2bCdlgTgZO4vl2gSGreakkrSm2wnoZumfUoquXA8DdpKp0QYiu8kIU0qRLKToLfSRsbMkggmmx6eOeezBQOhBRKTccwJCQwWjyN4daKHohG7g7yvKtb0KcD9UBfgrgSOSwT0BJTRs6XjOQA/CsMEgY2fpyfeDEiStfWtqMm47lNqn2eggtmvzxM7lBDSTCxuXjPgxVoeq0IVOZ7YbW3phDy9so1BKkduPDWNXuuxItRLSgwwUkmzq31pyJnEzRk+uj+3JlgkLs0gl2Zluoyvbzb1bHuSp1udY07uBgG7SlmknZkTQhIYQgrQ98rPqy/eTd/MUXItCMfVSCEE8EEUo2JbcjrWbsFZvvo/tqRZc6RI3YyQLSTL2+BvV7CtC0xDRCmTaW9EI3QAlkZNsNeB2N4+ySUnpiQQeg0we9+wzbpSVRTNC5KTFQDbN1FgDmhkakR66eWArf9jwZ86uX8gZ9aeh1PhbHFquRUDfu1REf2GA/sIAMxLT92lWtal/K5sHtrG44UxCRgilFBk7ixIKU5jUhqvpyvV4g14gztre9fx23Z284/irGbBSxAJ7ViRNWWkiRri0YujN9+NIl43JLfy9+TFc5RI1I5xSfSKn1Z5EPBAj42TZ2L+FylAFlaHycd3/eOnMdtOT7eWvWx8iakaImVEe3vEETfFGyoNlNCd38GLnKi5oPJfefD+VIU96Iu8UaM90sbZ3A4/tfBqAqBmhPlLLgqq5HF8+C0PotKTbaUo0jljJZO0sbZkO4oEomtDI2jmCepD2TAdNat8kp/cFLV6D07kFt2eH1xzHjCAHOtGiFeOuo1B2wRMlnIDqa2XnIZ/2KqAjiZJNMtkBrr1XtdSjEX8lcIShpAQ7D2YQW7mkLE/QjEgZWtEJ2K7Nc20volC81rMWW9oU3PEV8AwUkmwe2Lbb2eUgjnR4rXsNffl+sk5uj8cOJ2NnebD5MUzN5My6U3GlS9JOUxmuYHq8kbAZpjJUQVeuuzRrfaH9ZQquxeqetSQLyREbqrvSk+tl60AzO1ItONKh4Fq0pTt4pvU5/rbtYRpjDVw9+wrqIrUsb3ue/3n1//h782M4rkPECNNX6C/e/8FZFfTnkrRnOnl4x+Pk3QLXHPdGrpp1KUop7t/2ME+3ruD3G/7Mxv4tPNf+IgWnQN4p4EqXlnQrO1MtPLbzaWaXzeCCqecyM9FEZ66be7Y8wI9W/YxN/VvJOll6c32la9quzfZUC2E9RG++jzs33sMPV/0vG/o2kXPy9OeTB+XexsKYdyEinKCw/Dbc7mZvNeBaqPz4VqNKOjidm5Hd2yZkZi6LzY6UXSjJryi7gEp2IrNJTx/pGMNfCRxpFCUjRCBMXrlYro0tHQLFDCHwMlDW922iIlhOX6GfHalWpsYa9poqmrLS7Ei1olCkrAxV4aHZcKqQRmiCmOnNwl/tXsMfNvyFU6tPpCpcSdTcs3y17dp0ZDvZOrCdjf2bObt+IYZmknXyTI9PpSzoZdnEAzFmlE3n9d51tGc6mRprYNPAVgBe71nHwrpTyDt5ImNcrz8/QFu6nUQgTs7Nsy25HUOYPNW6nDW9GzirfiFLpy5GExpzKo6jL9/Pc+0vsqr7dVZ1v87SKYtZVH86GSdLykpTFT4wMbisnaW3r4unW55lZ7qNq2ZdRl3E03q6aNp5PNj8KM2pHSyomoshdF7tXsMpNQtIWSls6bIz3c4DzY9QH6nlTbMuL+0VKKVoTu3k6ZbneLD5Ud4z/1qvxgJFwbXI2FmkkjzR+iwvda4ioJtUhSp4ePsTvHveO2hJtlGuaggOW+040kEg9mu/ZTh6WS2BRW/HWnkXheW/IXjOP6CV1yMHOrzucntZMcpUN0gXZWVxOzej18zwMuHGiXIsT+oiGCvpPZWeUwqZ6kGYYYR0kf3tiLoYcqAdNNPrg2Dl4Bgr/PNXAkcYg7pBwgyzpm8DOSc/atb+WMszuMrlLcddScQIs75v06h9Acu16csP0JbuoCvbQ1++n+1Jb+N0cEY8iFSS1kw7Wwe205npJl3IcN+WvwOwpm8Dvbn+PRZ3DQrEtaU7eaJlOaZmckbdaWSdHE2JxpIDGOS48hmAlwmzrncjOSfPmXWnYUmbDb2bSI2xx5HKew7stZ61PLDtUVzpIqXk6dZnWdO7gXMaFnFB47kjQiYVoXIun3ExHzrpPRxXPpPHW57hDxv+giMdevN9+13UpZSiJ9fHloHtPNm8grV9G1k6dTHzK09AKsmAlWJ+5RzOaVjElTMv4coZl7BkytnoQuPljtfoyfexuX8r92/9OwE9wDXHvXHEZrEQghmJaVw9+woMzeT+bQ8T0AN053rJ2TkMofPYjqd5sfMVTq05kQ+d9B7eccLVCKHxt22P4ErJpr4t7Ey1kLRS7Ei1sqFvs7eqsMe/qhsLoZuIcILQee9FhOJYL/7ZU1q1sqVK992+b3YBNdDhyXIEYyAlTtsG3FS3twLe02ulxE1147StR2X7kT3NXqcza9j92DmEa3nqtGYQrCwq1Y3K9kMgjNANZLb/gO7/SMR3AkcYg0vYAV3jrk1/ZXXP2hGbpRk7y8udr3Fc2UxqwlUsqJrLloFt9OR6yDsF+vIDbOlvZmPfZlrT7SStFN25Hloz7VjS4rGdT3vhl2JYAuCVztf4vzW/oyvbRVe+m7+sfZCObCenVC/Aci02D2ylZ1g4YleShRSru9fy+w1/pjvXyxUz3oASiqpg+Zjx/WnxqRhCpyvXzctdr6EJjbPrz6AhWsdrPWvpyw+MCAlJJVnfvYUHmx/h6dYVrO5Zy89W38ZjO5/h2baVzKs8gSVTziod70gHRzqlQT4RiHP1rCu4rOkiWjNt3LHhbvJOjry7f5vQrel2WtNtrO1dzzPbV3JazUmcXX8GUknSdprqcCV5N8+SKWdxYtU8hBDEAlFOrT2JNb3r2dC3mbu3/A2pJG8//s3EAzGUUjjS8Zxb8d5jgSjLmi6gLdPBS52riJoRDM3gb82PsKZ3PUunLmZZ04WEjTCJQJxLmy6kNdPOM9tXEjUiZOwcO1It5OwcUSOCLnS2DDTTk+vdY8htj+gmAoUIxQmc+kZUth9n8wowQridWzz57+J7P6RS6oV9ZL83Ix+M04tAGBGIIPtbcdrW46T7x7ykcizcrq3IvlaveC8YQwuXgevgtm8opanK7MCIFGr0AG5/Kxghb8VghpDZ/r06nKMNPxx0hDHoBFo1L2bdkm4j42SpwVN3fKZ1BQW3wFn1C7Fci+PKZ/FCx8us691ESA/hSklPvpeWdCs70q0ENJPyUBm2dHi9Z13pxz81Vk+t5YUM7t3yIEkrxZ823cd5UxbzSver1EdquaBxCS3pNlb3rGVuxfHURKpGbai60uWRHU/yyI4nqYvU8KZZl5EIxLGkTW20Zsx7DBthaiLVdGS7yTk5psem4iqH+ZVzeGTHk2xNNlMVrqQm4t3ztuR2fvH67QwUUiybfgGzymbwyI4nebX7daZE67l8xhsQQmC5FgU5tOmddwqeIxBgCJ2TqucTMkL8ZfP9NCd3Uh0ZWdU7OBA7yiWkB0eFNmzXZke6lZ5cH8tbV7ChfzMLak/gDdPOByBlZaiP1lITqUYpGCgMEDUj3sa4k+OU6gW80rWaB5sfJR6Ice0Jb6EyVIErXTJ2hqAZAgVSymLOv2BafCpzK47nqZbnWNH+Umnv54Kp53JWw1DqrSMdTiifzYlV83h827Ns6m7momnnURaI05zaQU+uj5Oq5xEzI7RlOkhaaRqidYR2CcW40qW/kCRshEakpuadPJrQMHUT5b1Z6LWz0OtPwF7/JMb0UyEUQw50IFNdAAilSscKM4iyc7htG3C3v0pg4dVFVVrd6+ng2hTaN+M6IU+yu5gqi5XF6W5GoEYV7gkzhNINVNdW3KppyHTPiNarIhAGW5RUaMWgdLidK/VHOBbwncARxmA4qEV5s9SObBe9uT6a4o0IIVje+gJ1kRoaonVknRz1kRrqI7Ws7llLe7aTHakWbGkjENRHa8k4Nju7W3GUy8nVCzi15kRuX/8nXuxcRV2klm3J7XTmulk2/QKaUzt5qvVZAN488wosaXFSzQIe2/EUXfkejAGDKdF6EsEhwbU1Pet4fOczNMWn8bbjr0ITGmk7zfT4NIw95I43xqawsuMVFIpF9adhagFOrl7As20v8Gr3GhpjU4kFIhjC4I4Nd5Oxs1w35600xjzpgmuOeyPtmU4qQmXoQiNppQgbIZri04iaXj3F4KCedwukrTQ9+T5mJqZTFkjwWs9aZpfPpC7iyUZ0ZbvpzfcjUSglqYvUUBMZkpTIO3makzvZPLCVx3Y8TcEtcP7Uc1g2dwl9fRnSToa6SDXVYc9x1UWqSVkp8k6egrSpClWS1nTOrl/Ipv6tvGX2lSSCcWzpkHfyTIs3UhYaGuSk8hxBZ7aHJVPOIh6IIZUkqAepj9ZyfPksoFgv4hYwNRNHOVw07TxOqG3igY1P8Ou1fxjxnq/sfJk3zryUWWVN5J0Cm/q3UhuuIhKIENBM8k6Btkw7jnRRKMqDCcqD5fTk+0gWkpSHypkWn4IwQp52kBHAPHEZ7iM/xl77GIHTrkKEE97mqxAIoZXakirXRva3Y6+633s/n/g5wbOvRa9uArwwkx4Jozq6cHJJhG6gHBshpbdi2E3jI6EZqFAc2bsD0Eap4u4q+SGEjswOoPtOwGeyMthfuE3mEAgUiu2pnSyonkt3roeuXDcXT1tK1s1RG6khakaYXzWHR3c8hS40Tqqax4yy6UyPTyWoezMgpRRSydKm4Jm1p/JM2/O0ZNp4quVZygIJZpfN5LjyWUyJ1hMOm5SFE1QEy1kSXsQzLSt4vWcdTfFGtqd2Um4l0IVBxs7yu/V/JqQHuWrWpQCk7Az1kboRjmIsZpY18ULHywBMjU2hKlxJxAxzWu3JPNO6glVdrxEygmzp38qOVAtvnruMxliDN2i6eQSCylA5CkXaztIQracyVD5i9i6EwNRNTN0kHogRM2PsSLdwSs0Cnmx5lvZMJ/XRWrpyvaztWUciEKc8VE6ZGacj24mhGVSEyhkoJNk2sIMV7StZ1f06dZEarp15DTXhKmxpk3XyTI01UBEqL11b13SmxhtoSbXRFG8kEYyTc8qwXZuz68/w0jqdLFIpZpRNH7XxrgmNgB6gIVpLzs1x7pRFpc9zkIydxdAMGuNTiAdiFFyL7ckdnFg3l+mhGbzc+SoKxfR4IwHN5J6tD/LHjXdzZt1pLKo/nZgZoSffR3e+t+Qw1/dtor+Q5My6U0lbWQasFK7r0p7tBCFoiNaildXi9rWgcnlEtAJj1pk4m59Hq25CbzxpTHlylenDWnkXoryB4MKrKay4g8IzvyZwxlsxps4vfV7eXoELKERw9GpsLISmQygB49njMUNe57jy+qNOGnx3jMsJnHfeeVxyySUsW7aMRYsWoY0zf/fRRx/lRz/6EdlsliVLlvCFL3yB5cuXc8stt1AoFLj88su5+eabD+gGjnaUdFCFLCrTD2bA69wEtDsppsYa6Mr1sDPdiuVaPN/+EgLBjMR0IkaEqnBFMZ5+JjMTTVSHhzKIpJIUXAulJLrQR8zKz6w7jRc7V/HgtkdJ2WmWTb+AoBkkqAWYV3kCU2qqaO/pozpchQDmVh7P6u61GMJgVlkTPbk+Bqwk63o3krRSXDvnLZiaQc4plAa8vTEr4c0Ap0TriRhh4oEYAd3kgsZz6c718Gz7Skw9wHNtK2mMNXDm1FNo7erF0HSmxafiSIeklcKVktnlUwmPQ0QvHowxS5+BLR2ebVvJ6z1riQWiPNj8KG2ZjtJxiUCca467kpZ0K2k7y9aBZh7d8SRduR7OrDuN86eeg67pZOwsYeLMLm8aMzMrZkY5vmJWabM6bISYEqtnZ7oVgaAiWD5miG04uqYzLTaFzf3bSp+jUoq0kyERiDMlWl9y7mEjxMyyJpKiF0vanFW/cMQgesPcd/Dwjid5oeNlXuxcxZyK42iKTyNkBCm4Fstbn2fASqILjdd61nBazUkoNZSGfN6Us71Mr4gnrS3Tvaj+NowTliB7d2KtvAut+RXMOUu8vYB0r/dfp4Ds2gp6gODZ16KFywid/wEKz96OtfJPiHCi1NcAxt/jYjhCaAwuO5RSu3UeQtNR0kGmerxw1DFQVTwuJ/DHP/6RRx99lP/93//ls5/9LEuXLuXSSy9lyZIlu33Njh07+PKXv8wf//hHqqqqeM973sMTTzzBl7/8ZW677TYaGhr40Ic+xBNPPMH5559/0G7oaEEpiUx1Iwc6AIXQTMgnkb07cIVGlzXA/HgDAc1kR6qFrJXjpc5XmRqrJ2gEmBqrLw0uddFqUnaKlJ1G4E2IDM0gbIQwNYO8UyDr5FB4MVqlFGfUncbTrc9RFkgwo6yJhkgdETNMa7qD/nyS+khtqQfAxdOXknNybOjfxGs9a0r3IBBc2LiEukgttnKYVd40rsEYYGq8gdpwNfMrTyAWiJayY2oi1Vw4bQlJK8WTLcvRhc6lTReTtjLUhKuoCleUBr3Boqt9IWQEmVXWxNzK43m9Zx0d2S4GrCRXzriE8mAZfYV+nmx5ltvX38VVMy9lTc8Gnml9HlM3eNtxb2J2+QykkiStFOXBMk6onk1fz+6rdHeVqhhcLYSM0Ljfq5ARYmq8ga5sDzk7DQpqwlVUR6pGnT+gBzihahZOdhvduR4CmolUEldJNE3jsqaLOKv+dF7qfI3XetawtndD6bU14WreecLV1ISrearlOV7qfBVNCOZVzvEyufo2cfaUMykLJhCagZ6oxdVNVHczgaXvx932Ivbrj1B4+tdDBpkhTwIlGCNwyhXehi5er+Pg2deRf/x/sVb8nuAFN0LFntOQx4OzdSX25hWElr4PERj7fCIU99q12jmvnegkkryYCMZ1d/X19Vx//fW86U1v4uGHH+b73/8+d955J2vXrt3tax566CGuuOIK6us9ZcFbb72V5uZmmpqamDZtGgBXXXUVDzzwwGFxAoObSrKQA00b6os7jqbq+3wt6eB2bQOKSou7aZ5eOt6xkL07kfk0IhQbuSxViu5wGFdJqoIV1Iar2ZJsZmXny/Tm+zi58VwqQxUjZo+GZjAtNhVb2gT0gBcCGeOLPTgY5OwcjrTZPLCV02pOoiwQL8XRp8TqCMahkBp6XWOsgTdMv4CYGaE13U7OyVEeKqc8WIZULq5ymZloGrXJuCc0oXHjye+lLd1OVWhoBWNqBrPKZnDp9It4oPkRTqqeT9AIUB+vxcwf+CAB3gz9rPrTea17DRknyztOuJqacBWukkyJ1dOUmMadG+/hT5vuBWBGYjpXzriEqBkha+dwlVsKPxn7MWsdHjYaL+XBsuL77X2GY32+g+iaTn20lkQgRk++n5AeIGgESRfS9Ob7iJkx3jB9KRc0nkPW9rKkHOlSH61FExqWa7Gs6QLOaTgTXdOJmhFe6lzFQ9ufYNvAdqZG6zGLTluPVni/s1QP5qxFGFPm4fa1okXKvXaoY/zePG0fHRGMElx8Hfknfo713O9wL30/sP/9s1U+hbX67+BY2GsfJ3DKFd7jVg573eMYs89Ci1Z6exXhMmR2AGXn0aub9qlW4UhjXE7ghz/8IcuXL2fnzp0sWrSIm266aY+rAIDm5mZM0+QDH/gAXV1dXHjhhRx//PHU1AxlhNTW1tLR0bGHsxx8lHSRuRQq2YGy8wihA6qYFSPQyva/V+7Y13Nwu5q9Kl/NwOncjDBDiGAUglFkYmiQUEohswPIvp0ItFK2g3IsVKYXio29O8LeF7ImUkVjbCqP7HiSx3Y+jSY0ZiWaqAiWjbIjuptZz3A0oXkZHsE4x1fM5i3aG1Eo6qO1JaelCY3ycJyu9JAXCBkhomaEtJ2hLJSgTCUQQpB3CxhCZ0Zi+j45gEFiZoRoIDJKIC1qRjihcjYhM4QmBHEzRn2shp78gUtNgxd7nlNxHFfMuISpsQaCRoCAHiRmRujK9SCExnVzruGxHc9QG6lmYe0pFFyLlJ2hPJigOly1X/d7MBj8DMdDxIyMKLpLBOKUhcpoTbeTstJoQiNc/GzB03BypE3EjJB1cgghMIROys4wLdaILnTW9m7glJoFVOjlQzaVNaCsnCeBHopjNMwZ0x4lXS/7zTARVg5lBNASdQTOuAZrxR9o/8M30GpmYsxciD5l/qiJlFISlekDp4Aoaxj1vPX6I+A66PUn4Gx9AWPmGYh4NYWVf0J2bEJm+ggtvn7I7lAcZeVwOjZh1Mya2Famh5FxjXR//etfSafTXHvttZx33nmcfPLJe42Vua7LypUrue2224hEInzkIx8hHB79Ju5rzK2qav+q+aRdoIx+nEwPSIkoj6KZI8MFSkqvf282T2jKbLTg/s8slZIo28Lq7kBGQA8P6bUox0a5NsrpIb+zm5gZwiivw031IZ0BtJpKEILUK4+Q37keu6fF69hUpLOxAYHkuIZpnFw3j8p15fTm+plVMZ3ZDY1Mqx479XJfqCFOdXWMrJWjNja6sUpNzci4flnlCeTsnJf/DehCw9AMArqJMUZnrvFQKSPUOokxawlqiFOeCdOT7WN2ZROa0EbZdGDEWRo6g55cH+WhBDMrpmNoOo47je5sH+3pTq4+8RJCRoC0laUmkKCxbGxFz4Nr18FhdzbVEGe6qiFvF0gW0gwUkkgpkUpREyijLlpNJBDGkS4D+STJfIpEKE7EDLOi+wU29W6hEEhTUzNtxHll1QIKrZtRTgE9PPLaynWQhSwgMBuPx0hUoewCVtcOZCGLPv90nMbpZDe9SHbTy1jP/5HI8WdQvvgtCMMk37qR9KuPY3Xt8LSBAD1RTfT4M4kcdxp6tByrawfZ7a8QO+l8YiddQMed/4la+xBGdSOyYxOB2hlY7RuIFDoI1s8cZl0EaRdQ+RaC5bPQo6MnWJPx890XxvXrfOCBB2hpaeGpp57iZz/7GWvXrmX+/Pn84Ac/2O1rqqurWbx4MZWV3lL+4osv5oEHHkDXh2a+nZ2d1Nbum5hVT08aKfe9kjOheunZsbMYXjHBcoCxtEkMlFOg//VXMGpmjuodq6QDdgGUlyqoXAfyaWQhA0p6WilC81YZKBDespb8WHFhg4qKCL1d/aiudV4lYyCCKuSxXvgTbsvraFXTMY4/x+tpKwRIl9bcespUAXImvT1ZZidm0Jt7haboNPRCiK6u1BjX2j8EQbpyI89XUxPfzTUEg7tvDlDAITPme7xv5Njd/ZiUq2r6enN7sGn/MZ0IRiFH3KgYEdfXCVEj6mnPdtFr9VIfraVMlpHpd8gw3vfq8DFem3RCVBIa+lgdyAwMv0eDGBXIDKSxWVh5Gq93bmD55lfQckHKQ+UjKp1VsB6Z3onsay9uwLreHpVuosVr0CIJhB2A4nutAvW4vRshO4DQI1ScfinOjHOx1z1Bdt0T5LtavdV0x0Zv87jxRLTyKQA421eRfPFvJF/8G1rFFJRtQTCK27SYZE5gzD2fwqsPUGjdiDFjIfpJlyIe+gG9z91HcOn7R68yXIVa9ypatAKtrKEUxhp8L5WSXr3DAcpuTASaJvY4eR73FK2srIzy8nJisRiu65LN7lmS9sILL+Szn/0syWSSaDTKU089xWWXXcZPf/pTmpubaWxs5L777uOtb33r+O/mQFAKYQTGlfYljCAIDbdzMyJeC0KBlMVMBsvbWRWAEiAUQg94XwohQEq89LXouFPMhBEofamUklgv3YPb8jrmgkswTzh31PFta1+mIlxJtKjjs2TK2XRmuzm+/Pi9avgcbUxkD4CQEaQxPmXM5wJ6gOnxqfvd3+Bo5JTaBSS2xFnTs56p8QZiuR4qQuXURWrQNd3rxFY9HVLdICVaIOR9783wmBEBITREvBrZ24IIm6XHAvMuRCtrwHrxLsj0YC64BGP2IsQwh2PMOB2Z7sFtWYPbtg6V7iaw8C2lugBj5pk421chjCDmKZcjNANj7gXYr9yH274eo2HuSFt0E0JlqHwaJ7seLVGDVmzrqvJp3L4WhBFCr2maqLd3whiXE7j++uvZuHEjixcv5pJLLuFzn/sc8fiel0CnnHIK//iP/8j111+Pbduce+65XHfddcyaNYuPfexjFAoFzj//fC677LKDciMHG6GbqGCsFIsHAbqBFtrL0k8fPSipQhbZ34pWVofYw+uVU8Ba9Tfc7a9gzD1/hANQSoF0yds5+pwMs8LHleLOx1XM4m0nvIm4GZ/QQdFnNL4DGCJkhFjccCYPNj/Kr9f8gcbYFE6qns+8yuOZHm/E1D1JCD2x+9W/VJKMnUUXGkE96MXli70khmNMmYte+XFvA3k3sXotVoU25zzMOeehXBuhmzjSpSXXTdQIU7n0A15FshC4SiKmn4zYuBz79UfQa2ePcCpQDF0Hop7AXaoLd6CDnF2J0+VVIqtcP8qqPeL2DoQah0rWI488wpIlSwgGD/8O+X6Hg2QPve1dpZmAzCWR3c2obB8ql0JJB628Ab2yEZGoOyjLOpntx9n0HM62F70KSkCEy7yy93g1WqyaRMMUMjKMTPdgvXQ3KtOHMWcp5rwLQTooK4cQoPBWKNusAW7dfCeXNV3ElbOWlQb9gmthCP2QDEpHcojjUDMZ7ZpIm7J2llc6V7M9tZPVPevoK/Qzq6yJC6Yu4YTK2ZiagaEZuEpiuzaucjE0A1MzvZajLc/Rme2mKdFIXaSWaCCC2d+FKRX1dTUUUu6YE52Ca1OQNpa0yUubsBYgaoQIamZplWFJh+3ZThzl4ipJ3AxTF6wgbefosgbQhGBaMolacQd6wxwCi96xx3FAKUV5wmQg5YU8VSGLCMfRq6bt9jWHg4MSDpo9ezbf/va3yWazXgaLlDQ3N/P73//+oBl6KJDZAdydz3jLw6J+CQBmyAv/NL+MjTdQG3OWYEw/zYvxF1FWDtnfisolvTQ2xwblevsD0oFC1gsZ5QaQmT4vI0gI9MaTMBpPQqa7vdTPZAeqfQMoSfcw+0S0guB570WvnuHZa2XRKhrRwnGUpqNpOh0tKwCoj9SN+DEED2ITFB+f/SVshKmP1VIWTHBKzQJe7V7D8rYXuG3dHUyNNVAbqaE8kMCWNra0cZVEKUnOzbOhb3NJDHFl5yskAnHOql/InGgjzkArmaRDNmOTMMIENBOBQKIYsDNYxf7JoqgDlXKyqIJCExohPUBIM0k6WRQQKa6gs06BTbZXmBc2ArjKZVs8xvQTL8Fd/RDWqr8SOPWq3ReWCYFmBCjtLQbCqEwfKlFb0iMaCyVdcApe1p+VBSuHtPJo8Wr0srqD9lmMl3E5gU996lOceOKJvPzyy1x55ZU89thjLFiwYKJtO2jIXJKeJ39Jofl1AC/NbPqp6LUzkdEqpK7jSBcjn0br3Ymz5XnsV/6Ks+5JRKQclPRS3DK9e7iKgGDE2wsIxTEqGhHRCvQp89CiFVjShtoZmLPPRhPCS4fL9BEVWZIdbaAkxoyFoJtY0iFvZ+m3U/RktpPpz5F1c2gI1vZuxNQMpk1gr1gfn/1FCEFTfBo5J0/KznBKzYlMi09lVddqWjMdbEtuH/t1CGaXzeDUmhOpDlXSnNrJq91reGj74+hNF7FADxM3w0hdkHbzuM7QnmRQM4ntogEULNYTSKVwlMuAY6MLjcCw1O/ILmm8utAQaGyrn8qU3JkEN7+AHYwSmH/xuO9daRoy3Y1eMfbvU1k5nK5tQ5EBzQDdQCBKjx1qxuUEMpkM//7v/87Xv/51li5dyg033MD73ve+ibbtoCF7dmD3taPNXkS26WQGDA1LObjKgVx7qZwcBcHKWhK1byfS24bW/LL3wQgNLVyGmH4qqrwBGUmgdBMMA4QOAhQaCMFgoMouxjEL0qY/3YJVbNohhMIUBrrQQBeUx8vIaGE0IbCsPvKuhUTSnNzBw/1rsOXo7JqGSN1eWyz6+BwudE0nFogSC0Spi1STLKSoClV6nd5kgbxdIKAHCOgGGnqxUl2gC4EmNKJmhOOMANPjjTzQ/CgPND+KXn82Z+RNNMclJDTQQ0O/2z2gCUFAjD9F2dR0dBGmY8ZJVOT6ia9/igIQmnsB5nhCrYEIKtWNqxTYBZR0ENEKtHAZys4je7Z7iSS7KJ4qJYtJJYeecb075eXlADQ1NbFx40ZOPvlk5BGkuW00LkBe9VHWtWxG01wCCC9WiBgtByxduu0kKhZBLTgXUzNK3zVHuUVhhTzCLYCL9/eo5aLygvgIhICgbhLXvc0ipVRRidL7Z0uXnLJQ0lu6hvUAqwe28Pfe16gNV3NqzYmEjJAX7lHg4hIPxP3wj88RgSY0ykNlJIJxCm6BvGN5wnhSeuEUoREsVbGbBPUAmtCQSpK1c1wmLuL+rQ9zf/sKVqWqKdOjNAYqODHcgDiAOp492yyImCHyJ16ErhSR9U/R4eawZi0kYUSIGEFCWhBtNxlNygyjcinQdC9dfKADt7/dyyoMRkeEmIe90gstHwbG5QSampr4+te/zlve8hY+//nPk81msaw996CdbGTdPAFNJ7gXPRZT00se3yt8UqXZfQBjvwSlLOnQZ6cI6gFCWsBbBRRPE9ANgpq3dO0q9PPKwBZWJbcwIzaVS2ZeTCIQwxCeI9I0HV0YBIvKl0cLUilc16vYFgJ0bbRz9jmy8aqPw4SNMBWMLrga6/hYIEpVuIorZy7jpc5V9Ng9NKe6eD25jZZIF8vqz0Tbh0QIV0lW9m9gZ66bcysXUF/UllJKMeBkSBjREQO70HSyJ70BXUHlppXk0v30zjqVjnAcQ+hUBxKEXYOsW8BybQyhEzPHkLUez29ViBEFoYeScWUH5XI5nnzySS699FJuv/12nnnmGd7//vezcOHCvb30oLO/2UH9ybVsaN3C5nw3XdYAGSfvfXjSwVZep6OwHiSiB0mYEaoDCSoDCUzhfclcJclLm7xrlY6Xw1LXFApXSRzl4igXSzpY0qbPTpPapRF7UDMJaAYBzSRsBNCURs4teBkKCE6ON3F24yU8+nwPmhselQ1hO5JMziZbcDB1jUjIIGjqWI4kbznkCy45y6FguZimTixkEAmZ3vdMFVcvxcWKlAqplPdfqZDKmwkFAzqhgO6tVhyJIxVCeL5L1zWyORvLkRi6RiigEzA0hOb9gLy3RKGUd35HDp3fkyRWWLaL5UhsW2K7I7/8uiYoiwWoiAVxpSKds8lbDgFDJxz07tXQBbquIaVnnxJQsNzS56EJgaYJXKlwXInjeNexHa/6VRMCXfdCELquoWve8bomMHQN09AIGBqa8DYflYTh3zpN864hAKk8R+at7rx7dov/NE1g2S5SKixHlv7f0DUMQ0Mp7zgpFUHTuz/T0LzXu0M2265Ew7NRiKFKe10TmIaGMSw1WSmF63r37RZtUkqV7isUMHClHDWTHTyvpgk0GPPzHPz+6EKg6d7zUnrn14vvn6ZppcXx4DWG27w7Bj87XRPEIgHiYZO5s2LIUC/xQMwrrOzN8EzrCp5pe57Z4VrOrjqRjkIfXYWB4u/ZRqGIGxESRoS4ESFmhFAonuxZTY+VxBQ6jnI5tWw2cSPCa8mt9NlpGgLlLKs7k5pdZVekS2TTC4S3vQpKkp+2gOQJZ5FDEosFSae8BBCpJFND1VQE9l3VQBX3A4z64/f5tXtjb9lB43IC73nPe/jVr351UA3bX/bHCfTkevnFa79kW9rTKYprEUIiSEgLYAoDAy9XOC8t8soiLbNk5N57rQpvO6f0ly40dDRP0hcdQ+jEtQgJPUZEBLGlTQEHSzk4FJ2Ppig4FgLBNLOeWaKcfjWVR1ZmyeUlVYnRKxdd8wbeoKnjSEnBcnFcVRq4TEMjYOqYuobteo6hYEkQQ2HU4T9Sb1le1GsX3oBmOwrbcT19GN0bGAYdR8DUvc5RmjdY2Y6L7chRMVpv9SJKA/LgdTRNYOoauq5h6kODMHgDgVV0cumcg6ELQgGDgKlhOxKreC05bJA1dI1Q0MBxJEUzSwOyJryBSde943S9OHBL7z53dYCDg/Kg0/DeKzEq4jc48HmPe89rYnCA9u5RIAgGDVzHRRMCozhYCwGuK3Gl8r43mneNwftzXFVySoY28v2RUiG9N6r0tyOHraQYuv7gv8HBV7qeM9d1DctykUoN3w7z/lt8Hzw9rZGRTiGGvu+D79Pge1NyhsVJxTATS1IilF47OoI6/DFXKrJ5h1zB+/wXnxmmqT5OPB4inS4ggE3J13k59ULp9QEMQsL7PQsgo/Lk1MhoRUSEONOcTVWwgdW5tWx0WlBAjVZGvVHJBmsnNg4LQrOZH5w95MCKrzesLHVtr1LVtYFCvJb0GVcQr64k2ZfC7G0hXzmFjHKYEqokYUS972FxciiVxJYOWTdPzrVwkQgEhtAoM2PEhIkpNIyGEzjYHJQU0VQqRTabJRI5MqtRB6wkliOZb8xgdriJikDU+7koNfYSTAhsJCk3423wUJwdC5OAMDGEPuZ+wgiKxV1C2V61sa4DOiARUgIShEYoFiOXd3FdRXdfnnWpMC9tzGCa8KYljZx/ygwiwV2XvKIYMvF+mIMD2dBANNquPfn63R0/+Aox7BilFDU1cbq707s9ftT5d3ON8TA4a9/1WsAI+2prEwc9912qotfbk+m7PD/8UCHEIa0T2JNO/nCG2zTW92JPU6zhZ9/dccPPOfz04+npMojluHT15fh/d6/mmRVZtEUaCytjOKY3Y15QdRK1Rph8tpNqs5xwdCqE4wjHQTh5UF4/5rydJGslsZDU6wn0YDkyVsUit4ETBzajXJuyYBVOrI651gAvdi/ntfwmet0BlsZOI6ANC+WE43TOOpdM2VQaNz9J2fI7EQ1NVO7YgObamDNOQcw5h7Z8H+30l5za4C9DFCeKhqYTwPBW4krSXuhDSZcGM86Bq37tO+NaCbzrXe9i06ZNzJkzZ4Qj+J//+Z8JNW4s9jcctGL186x4vY/tXSapPHv8Rh68cHRp3o3adZZcnG15KBwXZPGg+hqT886MMbNsBnOml+/3ADpRHGsFUAfCZLRrMtq0Oza3DPDz+1+nvSdPLKITDGhUlhmce3o5AUOiZ3pwI+Wg76GQ1bXQ8kkvZBOugMHwqnTQChlkKOZl+QFapov1qfWsyG0grke4KH4G5cboWbSZ7mLa+kcwpUOubjaalSPYu5Ouc65F7k1VYBiiGJbMOwWiSmP67NEyMQfKQVkJvO1tbztoBh0OXt/ay0/uSwMmDeWSxikK9ADKCAx9IYoMjs1COeB6PUyVbqKKO/3serQa8coiwjt2HGN30DSwCgUMbKqrI9TWRhFmnrAsp7ZibE0VH59jhYaqCG85bybPbNiKa5ukMjabtufoSzpccX4Vofg4iqv0ADI6WgkXzUCGR8b/ZaSSudY0yo0Ej6Vf4Z7+p1gQnsnJkeMwh6Wa2rEaNp76dgxNkLMVppVmXs8OjDXPsX3GPgzkCsoTQUxTIMdIBz8UjMsJvOUtb5loOyaUaXUxrjivikAgTW1FHDSj5PkPN4l4mGRqaP/BkhZS6YSNKImonwbqc2wTChpEQ0EWLSgnHDPJZ1yaW3M88mwf9z7azTmnlzPObrdDc7ZiCG/U9Kr4WMSopl5zeHP5ubyY3chruc1sKuyk0ayl0ohTa1ZSZZSh6zqRaBCVKUCgjL76+VS2rWag8UQKkcpdz04w00uiezPhVAcIDakZ2IEoyVAVuao6zPLEuEN6B5M9OoG5c+fusWR6zZo1Yz432UhEApwyv5rmDjXmstHrxuTgbe7qh02EreDmEUJQrtVSm4iOyPjw8TkW0YSgKhEi0xvHkSkK0qKuXvCGcxM8ujzFXx/v3vtJ9oMTZyU4e0aS8+InMyfUxKrsRrZb7Wws7ADgzOh8FoRnjnhNz9STKe/cQN3WZ0lVzUQBhl0gkB8gmO0lmBtACUEu5gno6U6BcLqLCncD7IC248+D4y9gXCGEg8gencCzzz6LUorvf//7TJ06lXe+853ous5dd91Fa2vrobLxoCKVLM62pRf7VwpN6IT0kNd8XRaKXcZgMNtCCYVQolgYRqkQrNSwl8GPTZSO9R5QQ1kPxdfvmo2h25JsMYU0oAepDdSTzysq4odfrM/HZzKQiAYI9oVpqphCn8h4v9+aZt52WW1JvG1vk2fvdzgsy2swVXqXrW6lYFtLntWbMmzvCHPucQWmNZRxSdkilFJkZYHnM6/zQmYNOZlnaeSUUro4RpCuaQup2/YskVRn8YwCOxjDCiXoq5tHqmom7nCJC6UIpbuZ8fp9iHx6SKb+ELJHJ1BR4RVTrF69mn//938vPX7DDTdwzTXXTKxlE0BB5jEkxIwEYSOEIQx04akaDsdVLo60sZWDlC665sk8DO7mKyVBeCmAmvBSQjW8Wbsneys9ydxiFpFElpZ5g8dJJK50SMTD9MmMJ0LnGuQLirrKCOHg0d3c2sdnvHjp0AaW41XUBrQA5YEKkiRpjB582ebG+hCzGsM88UIff1sVILRGMqsBokGAAPWhUwmVr2F1bgsbd+7EljYSxazgVBbVziNZPQuhvL4iUg94+4m7QwgKkXLv/6WzbylUB4lxjTS5XI4tW7Ywa9YsANavX49tHx6xo/0lHogzJdxIWShWCvfkCw45VwFj3YuX0imAYkLnKFTxcaf01yCDYZxdy8AHXzF0nF3QcAqGV6wWNJhZH/MdgI/PMIQQVJUFSeVssjmbgKlRZpaTtAeQSo4I3yqlsGRhzMndvjClLsg7Lq9jR1uWzZv7Wb9T4sqhKfrxU+dz1vEJUloazdVxlMOG/HZarE5OjhxHSHgreelKbOXVBmXdHCmZw1UujYFaZgQbSOjRISch3cNSNTyud+mTn/wk73znO5kzZw5KKTZt2sR3vvOdibbtoFIWitMl7VJ5V77ggBDMmhIfUwPkUFFdHSvl3AdN3SvK8vHxGUFlIsTUaIgdLX30pSxcR1IZqKK70IWpeQkUrnRAKOJGGTk3S9bJYIjBfHxP90sIDQ2tWPjm/dacYpUxQEgfqtDXdcGMxigzGkz0gRakHgA0Xt6ieH6DwrYbuXpJmELeK0qbE2piefpVXsisHfMeQiJITA+jULyUXc9L2fVMC9SxNH6qFyiWLnuu0pgYxuUEli1bxsKFC3nxxRcRQrBw4cJS7+D77ruPN77xjRNq5MGgoTpKV3eavlQBwxBoCGZNSRA0D2+WUCRk+jN/H5+9oAlBIhqgtiJCWTTIhh19xAJxL3SrHBSKoJkgZsQxNG9lnXNzpJwkOp5EiyY0r6GMdHBxUcqTEEkEE4T0EDk3R0+hm5AeRh+ePagHkLFatEwXKMnCWQGCpsZTryt+9Jcc4SCEA6BpEeAs4kYOtOJeodJA6ghpINBwgJAJS5oKpMM7WZXbyIMDK5hhGJM7HARQVVXFsmXLRj3+85///IhwAqahM70uTlk0QNdAnmk1scPuAHx8fPadYECnpjxM90CeykjVmMcIIYgYESLG+FUOgnoIU5h0FjrQ0AkOyyRUgSiuGUJYObR8HydOtUhEgnQMaCTTLjlL4RYTRjQ3MjoSPIyOftjaEaS2bDanzI3zqvMKP52S4Jqsg5LyUO8Lj98J7I5xFBxPKspiQcpifuaNj8+RTFVZmJ5kAVdK9HEXCuydqBljqh6gJ99FxkkT1EJDewtCRwVjuIEw+kAL0ytt5s2Ik8kU9ukarqtY16J4aZNi5cpazjt7Ic/yApsLNgvG3H2cWA7YCfgVrT4+Poca09CoqwyzsytNKHBwV/QCnbpQAzk3S1ehEyUV5nANIaHjxuswBlrAdb0+AK4FaGDsfYKp64IF0wVNNYo/PiN58dUEzANHSNRh6NPiB6N9fHyOSCrjISxb4rrFaMRY89Fd1FxGP7dL+bACR0pSGYt4NEqDNpWW7I5SKngJPYgbqwOnD1wXFSpH5JNeXH+cWUmxsGDZaRr3vqATAlylcKV7yAdl3wn4+PgckWiaYEr1wW+zqpSivTdLZ1+ORDRAXaie9nwbYT0yMh01EIVIGW7G9uqGjCBasg0COuOr+FJMrXA5+wSNV5SiIEHKQ99dbEL3BG644QZ6enowDO8yX/3qV9m+fTv/7//9P2zb5r3vfS/vete7DtQEHx8fn4OGEIL6Sm9DubM/RyISpSpYQ2+he/AAglrQWxnoJohipZAZQYUSCCsNxp6K2BQ4VnHVoDOnQeO1FDh4K4FDzbicwHe/+10+9alPjfncVVddNebjSim2bNnC448/XnICHR0d3Hzzzdx1110EAgGuvfZazjrrLI477rj9NN/Hx8fn4DPoCJSC7v4cZdEy4mYcR9oU3ALdhS6C2hg6ZJFKdCsLTg70wGihSqcA0kEF4siIp2Bq9rSgq6KywH7I5B8o49pWf/zxx3f73Ac+8IExH9+yZQtCCD74wQ/ypje9id/85jcsX76cs88+m/LyciKRCJdeeikPPPDAfhnu4+PjM5EIIWioilBVFiKVsdHQCOohEoEyGsJTKMgCttxFbUDouIkGVKjcK/6ycwg7C3YO7CwYAWRZIzJe64lZ6kG0eCW6BEdM4pVAY2Mj73//+zn99NOJRodicO973/t2+5pkMsnixYv5yle+Qj6f54YbbuDyyy+npmaod05tbS2vvvrqAZjv4+PjM3EIIWio9lpF9iY9lV+FwtQDTIk0kpV9ZN0MXk6RTkALIPQAMhyAcAVIByFdUA6goYzQKLU7FUp4KwEB0j30PQXG5QTKy8sBaGlpGfeJTzvtNE477TQAIpEIb3vb27jlllv48Ic/POK4fU0x3VOHnL1RUzP+jj+Hkslq156YjDZPRptgcto1GW0aD4fL7tqaOLYjUXj9obe2DmA7krpQBU7cwXItklaSlJ1CExoaAke5oARCmICJwHMgCokhdELD9g00KXCFIp4IHvJ7HJcTuOWWWwBvdp9IJMZ14pUrV2LbNosXLwa8PYKpU6fS3T2k/93Z2Ultbe0+Gby/7SUna0u9yWrXnpiMNk9Gm2By2jUZbRoPk8nuirBJc3sKVypy2QK6EAS0OHEZJG2ncIGgFi1KWEgc6XqDv2agCZ2k3Ue/213SKtKV5wR6e1KYoYN7j3trLzmuPYGtW7dy5ZVXcuWVV9LR0cHll1/O5s2b9/iaVCrFf/7nf1IoFEin0/z5z3/m29/+Ns8++yy9vb3kcjn+/ve/s3Tp0n27Ix8fH5/DjGlozGiIU1cZJWjoKMB2FEKaxLQKYloFASJoMoCuQgRFlJCIY6gwQgaIyBqiooKcm/WUUJXAEZ7Q3aFmXCuBr33ta3zuc5/j29/+NnV1dfzDP/wDX/rSl/jtb3+729dceOGFrFq1iquvvhopJddffz0LFy7k5ptv5oYbbsC2bd72trdx8sknH7Sb8fHx8TlUGLpGQ02MoNj3yEQ279DZH8Duc8mqAYQSuDqoybox3N/fz7nnnsu3v/1tAN71rndxxx137PV1n/zkJ/nkJz854rGrrrpqt2mlPj4+PscCkZDBjPoEAUNjXVceTXkKo9I99H1axq28VCgUSpu4XV1dyMOgceHj4+NzNFFTHqHCrPacgCZwnEnqBK677jo+8IEP0NPTw3e/+13e+c53ct111020bT4+Pj5HNaahMbU6gYaOLQSuvW+KpAeDcYWD3v72tzNjxgwef/xxHMfhq1/9KkuWLJlo23x8fHyOeirjITR0HCGQTv6QX39cTuDmm2/m0ksv5aabbiIcPviNnX18fHyOVQxdwxAGtgDXtQ759ccVDrrooot44IEHeMMb3sBNN93EvffeSzqdnmjbfHx8fI4JdGHgaAJlTVIncNVVV/G9732Pxx9/nEsvvZT/+q//4pxzzplo23x8fHyOCXQMXCGQk3VPYMWKFSxfvpzly5fT2dnJ2Wef7e8J+Pj4+Bwk9GLnMtudpE7gve99L9XV1fzTP/0T73jHO0rS0D4+Pj4+B46hBQCwpIVS6pC27R3XaP7kk0/y1FNP8fTTT/Pzn/+cE044gSVLlvgNYXx8fHwOAoNOwHFtUGqU0uiEXns8B9XU1HDNNddwwQUX8Pjjj/Ozn/2MF154wXcCPj4+PgcBQ/fCQQXHAST7UMd74Ncez0Hf+973eOqpp+jo6OCiiy7is5/9bEkd1MfHx8fnwDCNEEhwHMdbCRxCxuUEcrkc//Zv/8bChQsPaazKx8fH51ggYATBAksdeicwrjXHv/zLv/Dyyy9zww03cN111/GjH/3I81g+Pj4+PgdMwAwB4EgXmIRO4NZbb+W5557jPe95D+973/t4+eWX+c///M+Jts3Hx8fnmCAQ8JQYXOUiD7Gc9Lizg/70pz9hmt7mxQUXXMCb3vQmPve5z02ocT4+Pj7HAqHiSsBVEild9EN47XGtBJRSJQcAEAgERvzt4+Pj47P/hAIRwHMCyp2E4aC5c+fyjW98g+3bt7N9+3a+8Y1vcMIJJ0y0bT4+Pj7HBKFiOEgqiasO7X7ruJzAl7/8ZZLJJFdccQVXXHEFfX19fPGLX5xo23x8fHyOCSIBr1jMReIe4j2BcTmBzs5ONm7ciJQS13Vpa2sjm81OtG0+Pj4+xwShohOQSFw5CcNB//Zv/8Y73vEOVq1axapVq7j00kv5/Oc/P9G2+fj4+BwTRMwgAK6QSDkJw0G5XI53vvOdmKZJIBDg3e9+N93d3RNtm4+Pj88xQcg0QSkkCjkZVwLTpk3jpZdeKv29YcMGGhsbJ8woHx8fn2OJgKljKHCFwnUP7UpgXHUCHR0dvPvd72bOnDkYhsGaNWuoqanhqquuAuDee+/d7Wu/9a1v0dfXxze/+U3Wrl3LF77wBdLpNGeccQb//u//7stS+/j4HPOYhoau8FYC7iQsFvuXf/mX/Tr5s88+y5///GcuuOACAD7zmc/wH//xH5x66ql87nOf44477uD666/fr3P7+Pj4HC1oQmBIgSsU6hCniI7LCSxatGifT9zf38+tt97Khz/8YdatW0dLSwv5fJ5TTz0VgGuuuYYf/OAHvhPw8fHxAW8lIA79SmDCRKu/9KUvcfPNN5NIJAAvzbSmpqb0fE1NDR0dHRN1eR8fH58jCk0KXMHk1A7aV/74xz/S0NDA4sWLueuuuwBPemJX9keWuqoqtt921dTE9/u1E8lktWtPTEabJ6NNMDntmow2jYfJaPfBsknHCwdFIuYhvc8JcQL3338/XV1dvPnNb2ZgYIBsNosQYkRaaVdXF7W1tft87p6e9H6lUNXUxOnqSu3z6yaayWrXnpiMNk9Gm2By2jUZbRoPk9Hug2nT4Eqgrz95UO9T08QeJ88T4gR++ctflv7/rrvu4vnnn+eWW27hjW98Iy+++CILFy7kL3/5C0uXLj0o11NK0dfXhWXl2Z0Wd2enhpTyoFzvYDJZ7doTk8NmQSAQoqKixm905HNUoCNwhCr2FDh0HNL8zO985zt84QtfIJPJMH/+fG644YaDct50egAhBHV1jQgx9jaHYWg4zuEeuEYzWe3aE5PBZqUk/f3dpNMDxOPlh9UWH5+DgaY0XF3gTMYU0QPhmmuu4ZprrgE8NdI777zzoF8jl0tTWVm3Wwfgc/QhhEY8XkFvb4fvBHyOCjSlURBMTtmIyY6ULrruF50da+i6ccgzKXx8JgodDVeAe4hb9x4VTgD2L9PI58jG/8x9jiY0dGwhkNI6xNf18dkDt99+G1//+lf2etySJWfQ398/4fb4+Byt6Og4GkjHPqTX9Z2Aj4+PzyRAF95KQDmHdiXgB9L3g5deWsmPf/wDampqaG1tIRAI8vnPf4Xq6hr+67++xcaN6xFCcPbZ53DjjR/lxz/+PqFQmBtv/Ag9Pd1cffXlfO97P2bhwjN54IH7eeKJx/na177Jfff9hbvuuhOlJIlEOf/8z/9CU9MMvv71r5BMDtDS0sI55yzhIx/5+G5tu+iic3jHO65n+fKnyGQyfOQjn+Cxxx5my5ZNVFfX8K1v3Uo4HGbVqpf57//+PoVCHsMw+eAH/4mzzz4Hx3H43ve+zQsvrKCiopKKikpiMS/HOJ1O8/3vf4ctWzbhOA4LF57JRz7yCV8E0MfnIKALEykErmujlDxkiS7+SmA/2bBhHdde+w/86le/58orr+JrX/sS3/vet0kkyvj1r//Az352G5s2beR3v/sNS5deyIoVzwKwYsWzVFZWsnLl8wA89dQTXHDBxbz88ov87W9/5cc//hm//OXtvOtdN/D5z3+mdL18vsBvfnPHHh0AgGVZVFVV8+tf/4G3vOVtfOtb/8EnPvEpfvObP5JOp3nqqScYGOjnC1/4LJ/4xKf51a9+z+c//xW+9rUv0trawl13/ZEdO7bzm9/8kVtv/W86OtpL5/7BD77LnDlz+dWvbucXv/gtAwP9/OEPv52Ad9fH59hDF95kynYsGENhYaLwp3D7yXHHHc8pp5wGwJVXvpn/+q//ZNOmDfzmN39ECEEgEODNb34rf/zj73jXu26gq6uTvr5eVqxYzg03fIC//e0+3v/+G3nppRf57Ge/yP/93/+yc+cOPvzh95eukUwmSSYHADj55FPGbdsFF1wEwNSpjcyePZuaGq8ye8qUKaRSA6xZs5rGxkYWLDgRgFmzZnPSSafw8ssvsnLl81xyyaWYpolpmixbdhmbN28CYPnyp1m79nX++td7UAoKhfyBv5E+Pj4AGJoJgK0cdlf0OiHXPWRXOsrQdb30/0oplFKjslWUkjiOg6ZpnHvueSxf/jSvv76aL3zhq/zmN//HY489zEknnUwkEsF1JZdeekVppi+lpLu7i3jcE+ALhyPjts00A8PsHP0RjyW7IaXCcRyEGDkJGf56KSVf+9q3OO642TiOJJVK+Rk6Pj4HCUPzfre2Yx/SlYAfDtpPNm7cwKZNGwG45567OOmkU7jooku4664/opTCsizuuefPnHnmWQAsXXoBt9/+a2bNOg7TNDn99DP4n//5ERdeeDEAixadzcMPP1jSV/rLX/7EJz7xTxNi+4IFJ7F9ezNr1qwGYMuWzaxa9RKnnbaQs846hwce+CuFQoFCocCjj/699LpFi87mD3+4vXR///qv/8yf/vSHCbHRx+dYw9A9J+BKF3eXqmFl5XCzAxNz3Qk56zFAZWUVP/3pj2lvb6WiopIvfvGrRCIRbr3129xwwzuxbYezz17MDTd44Z2FCxfR1dXF1Ve/DYCzzlrMo48+xHnnLS39/a53vYebb/4ImqYRiUT5+te/PSEz7fLycr72tW9x663fplDII4TG5z73ZaZPb2Lq1EZaWnZwww3vJJEoY9q06aXXffKTn+H73/8O73rXO7BtmzPOOIt3ves9B90+H59jEbO4EnBcd1QRpBzoAN2ESNlBv65QY2k8T2LGUhFtb2+mvr5pj687mHo3L720kltv/U9uu+2OAz7XZNDh2Vcmk82Dn/1kVJiEo1/58lAyGe0+mDb9+am7edh+hoszDVxx8T8Sinhy0srK4bS8jlbWgF45dZ/Pe1hURH0mjttv/zV///sDYz53/fXvZtmyyw+xRT4+PgeDgBkEG1wlR6wE5EAnUoErHfQ9vH5/8Z3AfnD66WcclFXA/nD99Tdw/fUHR33Vx8dn8hAIhCELUrk4yS5UOAqug8r10yscsFI0TMB1/Y1hHx8fn0lAyAwD4KKQ+RRO23rcvjZsIei0kkg1MSFYfyXg4+PjMwkIBb00cKkkjhFG6QZYaXqUU6wdmBh8J+Dj4+MzCQiHBp2AS2t3mh5hYmPTanehS4eIVmDft4X3ju8EfHx8fCYB0WAUAImkYGQRWoCsWyAcMHEKLu5+9FYfD74T8PHx8ZkEhIIhABSSgrTJyQIAYS1ImomTaPGdwASQyaT5n//5b1555UV03SAej3PTTTeTSCT42Mc+xJ133jvi+CVLzuDpp1dy//338sMf3kpdXT1KKVzX4dpr/4E3vvHNANx00428//03cvrpZ6CU4g9/+C0PPHA/4OUCX3/9DbzhDZce8vv18fE5cKKBIOBtDAeLOkKHAt8JHGSklHz605/g9NPP4Je/vB3DMHjppZV8+tMf59vf/v5eX79kyVI+//mvANDT0811172VCy64uCTnPMhPf/pjNmxYz49+9FNisRidnR3cdNONlJWVl6QqfHx8jhxCpoFQCpdDW4jpO4GDzEsvraS7u5sPfOBDaJqXgXv66Wfwuc99aZ/74WazWcLhMIFAYNTjd9xxO7/5zR9LzqG2to5///dvECwuKX18fI4sTEPHUCCF7wQOiGdea+PpV9tGPb6rOub+sOTkBs49ac/lGhs2rGfevPklBzDI4sVLaGtr3es1nn76Sd773utxXYcdO7bzD//w3lFOYPv2bUQiURoapox4fN68BeO8Ex8fn8mGEAJdehvDh5IJLRb7/ve/zxVXXMGVV17JL3/5SwCWL1/OVVddxbJly7j11lsn8vKHBU0T7E6OaaxOQbtKUC9ZspT/+7/bue22O/jLXx7gscce5qGHRspECKHt9ho+Pj5HLoYCRxza3/aErQSef/55nnvuOe655x4cx+GKK65g8eLFfO5zn+O2226joaGBD33oQzzxxBOcf/75B+2655409mz9UImezZ07nz//+c5Rg/tPfvLfLFhwEul0esTxvb29pZ4Bu1JeXs5ZZy3mtddWcckll5UenzFjBoVCnvb2durr60uPP/zwg/T29vKOd1x3kO/Kx8fnUKArkIewoQxM4Epg0aJF/PrXv8YwDHp6enBdl2QySVNTE9OmTcMwDK666ioeeGBsMbQjlVNOOY2Kikp+8YufljTBV6x4lvvvv4f58xcwbdo0Hn/8kdLxd999F2ecsWjMc1mWxWuvreKEE+aOeDwYDHHNNe/gu9+9hUzGcyptba385Cc/ZsaMmRN0Zz4+PhONrgTu0bISADBNkx/84Af84he/4LLLLqOzs5OamprS87W1tXR0dEykCYccIQTf/OZ/8cMffpcbbngnhmFQVlbOt7/9fSorq/jiF7/Gd7/7TX75y5/hODbHHXc8//zPny29fnBPQAhvA3jx4nO54oqrRl3nxhs/wi9/+b986EPvQ9cNdF3jwx++iUWLzj6Ut+vj43MQ0SWH3Akckn4CuVyOD3/4w5x55pls27aN73znO4C3P/Dzn/+cn//85wd0/tdfX8OUKXvuJ+BzdNLa2syCBfMPtxk+PgeFj//fRzBwefPct414fCDdT13VFM467dyDfs0JWwls3rwZy7KYN28e4XCYZcuW8cADD4zozdvZ2Ultbe0+nXespjJSyr3G+ydTI5ThTFa79sRksllKSVdXalI2HIGjvxHKoWQy2n2wbdKkhqu7ZDKFEY/n8zaZdH6/rrW3pjITtiewc+dOvvCFL2BZFpZl8cgjj3DttdeydetWmpubcV2X++67j6VLl06UCT4+Pj5HFDri6MkOOv/881m1ahVXX301uq6zbNkyrrzySiorK/nYxz5GoVDg/PPP57LLLtv7yXx8fHyOATSl4R78tuJ7ZEI3hj/+8Y/z8Y9/fMRjixcv5p577pnIy/r4+PgckehoOGIXL6AUiVQHIl41Idc86iqGfXx8fI5UdHScXXxAZdtqarevpD1SPiHX9NtL+vj4+EwSdDQcjZLGTax3OzXbV9Jb1ki+YtqEXNNfCUwAg9LQbW2tvP3tb+LWW3/EmWcO5e+/7W1X8cMf/oTbb7+N1atXYds2O3fuYMaMWQC8/e3XIoQoyUoP5zOf+RyVlZVcd901peOVkmQyGS6//I184AMfoq2ttfS8EGDbDtXV1Xzuc1+mtrYOgL///W/89re/xnVdNE1w0UWX8O53vw9d17n//nt5+eUXS2qmu/L000/y6U9/kp/97Dbmzp3HihXP8v/+3w8BaGnZQWVlFeFwhIaGKdxyy3dK99vQMAXHcfjFL37Ko48+RDAYJBAIcO217+biiy8B4Oc//wkPPfQAv/rV70pieC+9tJJf/OKn/OhHPz14H5KPzyTEWwkIQpkedCdPw6YnyEer2TZ9IZW7hokOEr4TmGAMw+Bb3/o6v/7174lEoiOe+9SnPothaOzYsZOPfexD/N//3V567v777x0hKz2ctrZWqqtrRhzf3d3Ftde+hYsvXkYwGBz1/P/8z4+49dZvc8st3+H+++/lD3/4Ld/4xneYOrWRbDbDf/zHV/jP//w6//ZvX9rrPd133z1ccMHF3H33n5g79wucddZizjprMTCy58FYfOtb/4FlFfjFL35DJBKlpWUnn/nMJ7Bti8suuxKAjo52fvKT/+bjH//UXm3x8TmaEFoAKQTTVt+LDmQDER6fOZ/N1gaW2PEJuaYfDppgqqtrOPPMs/jhD783odfp7u5GKUUkEhnz+VNOOY0dO7YD8Itf/JRPfOLTTJ3aCEAkEuVf//WLPPTQg7S3j1ZgHU5/fz8rVz7PRz/6CR577OGSbMV4aG1t4fHHH+Vf//VLJYc4dWojH/vYzfziF0Oz/De/+RoeeeQhVq16Zdzn9vE5GsgGPd2zH8xu4sfHzeJbTeU8mV9HUmbRhb6XV+8fR91KwN7wDPb6J0c9LsTu1T3HizlnKeYJ+16xd9NNn+SGG67lhReeGxEW2huDEhKl65sm//u/vwK8mf9733s9llVgYKCfuXMX8I1vfIfa2rpRktWO4/Doow9x0kmn0NfXR3t7G/PnnzjimEQiwcyZs1i/fu0ebfr73//GWWctpqFhCnPmzOfBB//GNde8fVz3s27dWmbMmEE4HB7x+CmnnE5rawvJ5AAA8XiCT33qX7nllq/yq1/dPtapfHyOSuqDs9jY2U6r7qAZNsKuhu5pZLLl5M+emF4hR50TmIxEozE++9kvlMJC42V34SCgFO6RUvKjH93K5s2bWLjwzNLzg04CwLYt5s1bwD/9002lamvXdUad03Hsvdp0//338o//eCMAF198CX/60x3jdgJCUBLV29t1ly69gMcee5if/OS/WbLk4KnM+vhMZpbOmUmmuxtFjGwGHBeIggxbREL+nsC4ME84d8zZ+uGWOli06OwJCQtpmsZHPvIJ3ve+6/nd727j3e9+H8CoPYHhTJ3ayOrVr5Xi+OCFeVpadjJnznxeeumFMV+3YcM6tmzZxK23fofvfe+7SCnp7u5i9epXOfHEk/dq67x5J7Jjx3aSySSJxJB89urVrzFlylQSibIRx99882d497vfOepxH5+jlXBAZ2a9TXlsZKQ+nbOIhSYmeu/vCRxCbrrpkzz//LN0d3cd1PMahsFHP/pJfv3rX9LT073X4z/4wX/iBz/4Li0tOwFPrfRb3/oaF1+8bER/gl25//57edOb3sLdd9/PnXfey113/ZVLL72Cu+++a1x21tfXs2zZ5Xzzm18jm80C0NKykx/+8L94//tvHHV8IlHGpz71r/zqVwcmMOjjc6QgtImJ+++Jo24lMJkZDAv98z/fNK7jd90TAHjnO6/n1FNPH3Xs2Wefw4IFJ/K///v/eM97PrDH877hDZei6zpf+tK/YVkFpJS84Q2XllYR4MX+h/c9uP76G3jooQf4wQ9+sos97+JDH3ovH/vYP4+Y3e+Of/7nz3Lbbb/kgx+8ASE0AoEA//iPH+bii5eNefzSpRdwwQUX09XVuddz+/gc6WhCwCFuKnNIpKQPJmOpiLa3N1Nfv2cp6cMdDtodk9WuPTGZbB787CejwiQcG8qXh4rJaPfBtimZTrLi1Ycoj1WPeDydS1JdVsdJc8duQLUnDpuKqI+Pj4/PviG0Qz8k+07Ax8fHZ5KgaxqHelj2nYCPj4/PJEGggzq0oVbfCfj4+PhMEoQuvIKaQ4jvBHx8fHwmCbqmHeLcIN8J+Pj4+EwaNKHhrQMOnSvwnYCPj4/PJEETAoWGOoT7An6x2ASwO/37sfoMLF58Tun5Qd19YES/gEGuuupq3vrWdwCeKNxb33olF1xwMTff/C+lY37+859w9913UVnptaKzbQtd1/n0p/+Nk08+dSJu18fH5yAhhEATGkoqJkg0dBS+EzhMDPYZ+O1v7yAYDI96fk/aPwDPPbecefMW8OijD/NP//RxQqEhhcE3v/kaPvCBD5X+vuOO2/nhD28tKZD6+PhMXnRNO2DF433BDwcdJgb7DPzgB/+1X6+///57Wbr0QubNW8DDDz+42+OklHR0dPgibD4+RwhCaMhDuCdw1K0EVrS9yLNto1UwhSi17dxvFjecyVkNCw/sJMO46aZP8p73jN1nYLgU9CBf/OJXmT37OPr6+njhhRX8679+EV3XufPO3/PGN765dNzdd9/FU089QSqVRCnFOecsGVfHMB8fn8OPpukHPljtAxPqBH70ox/xt7/9DYDzzz+ff/mXf2H58uXccsstFAoF/n97dx8VVdXvAfw7L4yioKK8aWCiV9RQhhQyVHBRoPKqizRJMRMVo6RMMzIxn+yaSr4kuBagYitZahESKg+RqGVq1zKXQGmJECpekAEEhkGBYeZ3//ByYnTAx2RgdH6ftViLmdn7nO/sc2b2nHNm9vb398c777xjyAhGrXdvC6xatQYbNvz3ffMMdHQ6KDf3W4wb544+ffrAy2syNm1aj8LCP+HsPBLA36eDqqur8PbbUXB2Hglra2u9y2KMGZe7RwJPwIXhn376CadPn8Y333wDkUiERYsWISsrC5s3b0ZqaioGDhyIJUuW4OTJk5g8ufMmDRk/cJzeT+vGNOhZW+PHez70PAP//vcRVFdXYubMYAB3B4jKzDyI995brVNuwABrxMTEYtmyNzBunIcwnSRjzHiZiaVQa5u6bH0GuyZgY2OD999/HzKZDGZmZhg2bBiuXr2Kp59+Go6OjpBKpQgODkZOTo6hIjw2HmaegcuX/4RCUYGDB7OQnn4E6elHEBf3GXJzv8Pt2w33lR8zRo5Jk7yRmBhviOiMsU4mEkuEr4gSANKoIdK2ANIeBlmfwY4Ehg8fLvx/9epVZGdnY968ebCxsRHut7W1RUVFhaEidKuCgjz4+XkJt6dM8W+3rL55BvRdE3BzexZEhICAYPTo8fe3gcaOdYej42AcPfqt3uUvWbIU4eGzkJ+fB7nc7R8+I8ZYVxCLxIBGA3VTE7QtapBYjNvmNrDp1c8g6zP4fAJXrlzBkiVLEB0dDalUipMnT2Lz5s0A7p4ySklJQUrKo80cdfHiJQwa1PF8AuzJVFZ2DS4uz3R3DMY6zZkL/4Oyyv+Fhbk5bGz7QdS7N/r07AvHfoPQ0wBHAwa9MHz+/Hm89dZb+OCDDxAYGIhffvkFVVV/T3+oUChga2v7UMvUN6mMVqt94Pl+Y70mYKy5OmJMmbVaLSor641ywhHANCZC6SrGmNsQmUg8ALI+Ulj1NwckYgwQD0Bv6oX6mmbUo/mhl/egSWUM1gmUl5fjzTffxLZt2+DpeXdCc7lcjpKSEly7dg0ODg7IysrCSy+9ZKgIjDH22Blqa4v/Etv9/9wChmewTiAlJQVNTU3YuHGjcF9YWBg2btyI6OhoNDU1YfLkyZg2bZqhIjDG2GPHTNq1k80brBOIjY1FbGys3scOHz7c6esjIoi6eBxu1r0es+mxGTNKT8SwEVKpDA0NSn5TMCFEhIYGJaRSWXdHYeyx9kQMG2FlZYOamkqoVLXtlhGLxdBqjeNiZlvGmqsjxpJZKpXBysrmwQUZY+16IjoBiUQKa+uBHZYxxm8WAMabqyOPY2bGmH5PxOkgxhhj/wx3AowxZsIeu9NBYvE//wbQo9Q1JGPN1RFjzGyMmQDjzGWMmf4TxpjbGDO19aB8Bh82gjHGmPHi00GMMWbCuBNgjDETxp0AY4yZMO4EGGPMhHEnwBhjJow7AcYYM2HcCTDGmAnjToAxxkwYdwKMMWbCun3YCJVKhbCwMCQlJcHBwQEZGRnYvXs3JBIJxo8fj/fffx91dXWIiIgQ6tTX16OmpgYXLlyAUqnEu+++i9LSUvTv3x+fffYZbGzuH164rKwMK1euRHV1NZycnLB582b07t1beDw9PR2//vqrMBPavbn27NmD7du3Q6PRwM7ODhkZGWhpaRFy1dbWQqlUAoBBc7Vn+/btaGlpwffff4+kpCSUl5dj8eLF0Gg0EIlEcHBwwOHDhw3alsXFxVizZg0aGhrQs2dP/Otf/4Kjo+N92zcpKQkKhQJmZmYYO3YsYmNjsXTpUmH5FRUVUCqVuHTpkkEyjRo1qt39rqamBg4ODjhw4ADq6uoQFhaGGzduwMzMDFqtFlqttlNyFRUVITY2Frdv30bfvn2xceNG9O3bt1vbSl+mp556yqj3uVY3b95ESEgIMjIy4ODgcN/23bdvHzZv3gy1Wg0rKyukpaVBJpMJuRoaGqBQKCCRSAyaqz33vs7LysoQGBiIwYMHAwCsra2RkpLSbv1HQt0oLy+PgoKCyMXFhUpLS6m4uJi8vLyooqKCiIjWrl1Le/bs0amj0WgoPDycDh8+TEREH330ESUnJxMR0TfffENvv/223nVFRkZSVlYWERHt2LGD4uLiiIiosbGRPv30U3Jzc6OYmJh2c40ePZr2799PREQvvfQSzZs3T6e+XC6nCRMmGDSXPkqlklatWkWjR48mT09PIXNcXByNHTu2S9syLCyMTpw4QUREP/30E/n6+urdvvPnz6esrCxau3YtRURE6DznuLg4GjlyJM2ZM8cgmYKDg/Vu30mTJtGyZcvI1dWVQkNDhbZKSUmhpKSkTm+r8PBwOnnyJBER7d+/nxYsWNDtbXVvpuXLl+utb0z7XOsyIyIiyM3NjUpLS/VuX7lcTlu2bCEioldffZVCQkKEuikpKeTh4UHjxo0zaC592nud5+Tk0Jo1a/TW6WzdejooLS0Na9euha2tLQDg8uXLcHNzE277+Pjg2LFjOnUOHjwIc3NzBAcHAwB++OEH4f+goCD8+OOPUKvVOnXUajXOnTuHqVOnAgBCQ0ORk5MDADh37hy0Wi1WrlzZbq5Lly5Bo9Fg1qxZAIC5c+fiwoULOvV9fX0hkUgMmkuf48ePY8iQIRg6dCgmT54sZD5//jxkMhkiIyPx+uuvQy6XG7wtZ82aBW9vbwDAiBEjUF5eft/2lcvlKCgowNSpU+Hj4wOlUqnznP/8808MGzYMjo6OBsukb7+zs7PDqFGjsGDBAgwZMkRoq99++w1nzpzBCy+8gKKiIri7u3dKrs8//xze3t7QarUoKyvDzZs3u72t7s3Up08f6GNM+xwA7N69GxMmTICVlRUA/e8rRIRXXnkFADB//nwUFhZCrVajuLgYxcXFCAwMhFgsNmgufdp7nf/2228oLCxEaGgoXn31VVy+fLndZTyqbu0E1q9fL7yoAGDkyJHIz89HeXk5NBoNcnJyUFVVJTyu0WiQmJiIFStWCPcpFArhME0qlcLCwgK3bt3SWU9NTQ0sLCwgld49+2VjY4OKigoAwKRJk/Dee++hZ8+e7eayt7cHEaGyshIajQY///wzmpubhforVqzA6dOn4eLiYtBc+syYMQORkZHw9fXFoEGDhPsHDhwIrVaLxMREeHl5IS4uzuBtGRoaConk7iTZ8fHxCA4Ovm/75uXlwdzcHCKRCDk5OairqxPqe3p6oqSkBAEBAQbL5Ovrq3e/q6ysRHBwMEQiEYqLi4W2srS0RHh4OMRiMWbPno133nmnU3JJpVIolUp4e3vjwIED2LJlS7e31b2ZXn75ZehjTPvc77//jp9//hkLFiwQyuvbvo2NjWhpaYFGo0Fubi5EIhFu3bqF4cOHY926dTh69KjQmRoqlz7tvc579OiBGTNmICMjAwsXLsSbb74pvOd0NqO6MOzk5IQVK1YgKioKc+fOxYgRI2BmZiY8furUKTg5OWHEiBEdLkcs1n1apGeg1IeZlN7R0REWFhZCLmdnZ536p06dgrW1Nfr27duluTqybds2rF69GlFRUThy5AgaGhp01m+otiQibNq0Cfn5+fjggw90yjk5OSEyMhK1tbU627e1fmsme3v7LsvUmqt1v8vIyMCAAQOE/W7dunWQyWRwcnLCsmXLUFRUhPp6/bOqPWyuPn364PTp09i6dSuioqKg0Wh0MnVHW3WU6UG6ep+7c+cO1q1bh48//vi+Om05OTlBIpFg6dKlQlu2Xc+pU6dgb2+PXr16dWmujkRHRyMsLAwAMHnyZPTq1Qt//fXXP1rWg3T7heG2mpqa4OrqiszMTAC4r3c+duyYzicfALC1tUVVVRXs7e3R0tIClUqFfv36Yfr06UKZ9PR0qFQqaDQaSCQSVFZWCoeK7Zk+fToqKiqwePFifP3111Cr1Th48CAkEgm++uor9OjRQyeXq6urzry7hszV6tChQ3rLEBESEhIQEBAgtKVcLhcuMrVm7uy2bGlpQUxMDCoqKrB3715YWloCgNCOUqkU27ZtQ48ePbBv3z4cP34cdnZ2aGxs7PJMbbdvZmamsN8lJCSguLgYMpkMWq0WycnJuHHjhk4uqVT6yLmys7Ph7+8PkUgEb29vNDY2Cp/0u6ut2svU9tOsMe1zv/76K6qqqhAVFQXg7qf3yMhI7NixAxs2bBDaMjk5GdbW1khOToa9vT2+/fZbAEC/fv2EXM8//zwKCgq6JJdCoQAA7Ny5E3Z2dnrbMzU1FUFBQcKpJCISjjg6m1EdCdy+fRvz58+HSqVCc3MzUlNTdXaavLw8ncM84G4v2brTZWdnw93dHWZmZjh06JDwZ2ZmBnd3d2RnZwMAMjMzhfPE7Tl06BDs7Oywa9cuqNVqaLVaZGRkoLm5GcnJyRg7dqxOriFDhnRZrta/9ohEIuTm5mLOnDlQqVRIT0+HTCZDUFCQQdty06ZNUKlU2LNnj/BmC0Box71792LhwoVwc3PD4cOHkZqaCktLS6F+V2Zqu33b7ncajQb5+fkICAiAWCxGbm4uTp8+DXd3d2RmZkIul8Pc3PyRc+3Zswe5ubkAgLNnz8LKygr9+/fv1rZqL5Ox7nNeXl44ceKEUM7W1hY7d+7E0KFDsWvXLqEtLS0tUV9fj7S0NDQ3NyM+Ph7Dhw8Xjvby8vLuOzoxZK7W+9vrAIC71wrS09MBAL/88gu0Wi2GDh3abvlH0iWXnx/Ax8dHuHqelpZGAQEBNGXKFIqPj9cp5+rqSo2NjTr31dTU0JIlSyggIIBmz57d7lX4GzduUHh4OPn7+1NERATV1tbqPH7w4MH7voXTNtfOnTtJLpeTi4sLvfjiizr1XV1d6csvv9Spb8hc+sTHx1N8fLyQubCwkHx9fWn06NE0ZswYWr9+vU75zm7L6upqGjVqFPn5+VFISIjwd287pqWlkZ+fH40ZM4bGjx+v85xbM7V9zobKpC9XQEAAeXh40Ny5c4UyhYWFNGLECJo2bRqFh4dTWVnZI+ciIrpy5QqFhYVRSEgIzZ07lwoLC7u1rTrK1J7u3ufu1bbt7r39xRdfkJubG7m4uJC3t7dOOVdXV/rxxx8pPDy8S3Lpc+/r/ObNm/Taa69RYGAghYaG0h9//NFh/UfBM4sxxpgJM6rTQYwxxroWdwKMMWbCuBNgjDETxp0AY4yZMO4EGGPMhHEnwJ5oERERuHXrFhYvXoyioiKDrqu0tBTR0dEGXQdjnc2ofjHMWGc7c+YMAGDXrl0GX1dZWRlKSkoMvh7GOhP/ToA9sVatWoWMjAw4OzujqKgIaWlpuH37NrZu3QpbW1tcuXIF5ubmiI6ORmpqKkpKSjBlyhRhfKETJ04gMTERarUaPXv2RExMDJ599lkUFxdj9erVaG5uBhFh5syZCAsLw7Rp01BRUQEPDw+kpKQgKSkJx44dQ1NTE+7cuYOYmBj4+fkhISEB169fR2lpKRQKBVxdXTFx4kRkZmbixo0bWLlyJYKCgpCQkIArV66gqqoK1dXVGDlyJNavXw8LC4tubln2RDHYz9AYMwLOzs5UXV1NPj4+VFBQQGfPnqVRo0bRxYsXiYho4cKFNHv2bGpqaqLq6mpycXGhmzdvUklJCQUFBdGtW7eI6O4vhydOnEgNDQ20atUqYax5hUJBy5YtI41GQ2fPnqXAwEAiuvtL0nnz5tGdO3eIiCgrK4uCgoKIiIRf2SqVSrpz5w55eHjQhg0biIgoNzeXpkyZIpTz9vamyspK0mg0tHz5ctq4cWPXNR4zCXw6iJkcBwcHPPPMMwCAwYMHw9LSEjKZDP3790fv3r1RV1eHc+fOQaFQ4LXXXhPqiUQiXL9+HX5+foiJiUFBQQE8PT0RGxt732iRTz31FDZt2oQjR47g2rVryM/PR0NDg/D4hAkThLGMbG1t4eXlJeSpra0Vyk2bNg3W1tYAgJkzZ+KTTz5BTEyMIZqFmSi+MMxMjkwm07mtb3RGrVYLT09PnQHD0tLSMHz4cPj4+OC7776Dv78//vjjDwQHB+P69es69S9evIiwsDCoVCpMnDgRixYteugMAIS5EFoz/dOhiRlrD+9R7IkmkUjQ0tLy0PWef/55nDlzBsXFxQCAkydPIiQkBE1NTVixYgWys7MRGBiItWvXwsLCAuXl5ZBIJMLsU+fOncPo0aOxYMECPPfcczh+/PhDjc3f6vjx46ivr4dWq0VaWhp8fHweehmMdYRPB7Enmp+fH+bMmaNzKuY/0Trj1PLly4Wx3BMTE9GrVy+88cYbWL16Nb766itIJBL4+vriueeeg1KphEQiwcyZM5GUlISjR48iICAAZmZm8PT0RF1dHVQq1UPlsLa2xuLFi1FTUwMPDw+8/vrrD1WfsQfhbwcxZqQSEhJQU1ODDz/8sLujsCcYnw5ijDETxkcCjDFmwvhIgDHGTBh3AowxZsK4E2CMMRPGnQBjjJkw7gQYY8yEcSfAGGMm7P8AlXddWy1kiA4AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"power_draw\", hue='power_model', data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 127, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 127, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEJCAYAAABR4cpEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACAc0lEQVR4nO29d5hc5X3o/zlt+mwvklYNCYEAGTARYHCQDMZIQhJ2BNxghIkrNrnXARdiMAaFXHOxCXEjwbjnXn7BiYKxwCCEbQjGGNMNGBAIVFBZbS+z0059f3+cmbM7W7R9tSu9n+fhQXt2zp7vOTPzft9vV4QQAolEIpFIxoF6uAWQSCQSycxHKhOJRCKRjBupTCQSiUQybqQykUgkEsm4kcpEIpFIJONGKhOJRCKRjBupTCQSiUQybvTDLcDhorMzg+eNrcSmujpBe3t6giUaH9NRpuGYrjJPR7mmo0wwfeU6FNNR5ukoU39UVaGyMj7k749aZeJ5YszKpHj+dGM6yjQc01Xm6SjXdJQJpq9ch2I6yjwdZRoN0s0lkUgkknEjlYlEIpFIxo1UJhKJRCIZN1KZSCQSiWTcSGUikUgkknEjlYlEIpFIxo1UJpIjCiEEtuMebjEkkqMOqUwkRxSm7bKvJY0nZ75JJFOKVCaSIwrHFXRlLLJ553CLIpEcVUhlIjmicD2B63i0d+cPtygSyVHFpCqTdDrNunXr2L9/f8nxf//3f+djH/tY8HNjYyMbN25k9erVXH311WQyGQBSqRRXXXUVa9asYePGjbS2tgJgWRbXXXcda9as4a/+6q/YuXPnZN6GZAZh2y7RsE4qa8rYiUQyhUyaMnnllVf46Ec/yp49e0qOv/POO/zgBz8oOXbLLbdw+eWXs23bNpYtW8Zdd90FwHe+8x2WL1/OI488wqWXXsqtt94KwD333EM0GuWRRx7hq1/9Ktdff/1k3YZkhmHaLpqmgFDozliHWxyJ5Khh0pTJ5s2b2bRpE3V1dcExy7K4+eabueaaa4Jjtm3z/PPPs2rVKgA2bNjAtm3bAHjiiSdYv349AOvWrePJJ5/Etm2eeOIJLrroIgBOP/10Ojs7aWxsnKxbkcwgLMdDUxWiEY3WrpwMxEskU8SkdQ0uWhF9+ed//mcuvvhi5s6dGxzr7OwkkUig674otbW1NDc3A9DS0kJtba0vqK6TSCTo6OgoOV48p6mpiTlz5kzW7UhmCJbtomsKmqaSzbtk8w6JqHG4xZJIjnimrAX9H/7wBw4ePMgNN9zAs88+GxwXg+wcFUUZ8u+o6uDG1FDHh6K6OjGq1/entjY5rvMng+ko03BMpMxCCPZ15EhGDRRFQTFMyitiVJVFDqtcE8V0lAmmr1yHYjrKPB1lGg1Tpkweeugh3n77bT784Q+TzWZpa2vj2muv5Z/+6Z9Ip9O4roumabS2tgausbq6Otra2pg1axaO45BOp6moqKCuro7W1lYWLFgAUHLOSGlvT495fkBtbZLW1p4xnTtZTEeZhmOiZXZcj67OLK7lWyI9WYu2kIZr2odVrolgOsoE01euQzEdZZ6OMvVHVZVDbsKnLDX4tttu45FHHuGBBx7g61//OsuWLeM73/kOhmGwfPlytm7dCsCWLVtYsWIFACtXrmTLli0AbN26leXLl2MYBitXruSBBx4A4IUXXiAcDksXlwTXFdDHqFVQZMxEIpkipkWdyaZNm9i8eTMXXnghL7zwAtdeey0A11xzDS+//DJr167l3nvv5eabbwbgYx/7GJZlsXbtWm699VZuv/32wyi9ZLrgeh701R0KiBk+vU4imSkoYrCgxVGAdHMdfiZa5lTGYk9TirJ4CIBM3qa2IkpdReywyjURTEeZYPrKdSimo8zTUab+TBs3l0Qy2Tiuh1Li5pr5c7UlkpmCVCaSIwbTdtH6ZPUpioLwDqNAEslRhFQmkiMGy3HR1NK0chmAl0imBqlMJEcMlu2VKBNFAenlkkimBqlMJEcMluOh9lUmKIMWxUokkolHKhPJEYHnCTxPlCoTRQbgJZKpQioTyRGB4w4eaZeqRCKZGqQykRwRuJ4AIfh/j77FC2+2AEXLRKZzSSRTwZT15pJIJhPH9chbHnsO9tDckePkxdWoiiItE4lkipCWieSIwPUEqYwJQM50eOGtVlBAGiYSydQglYnkiMC0XVI5f7JiRSLEH19rwnE9WWcikUwRUplIjggs2yOV9pXJmvctIJN3+NOONtnoUSKZIqQykRwR2I5LKmsRC+ssmVvOgllJ/vh6M9YQWV4SiWRikcpEckRgOR5daYuKZBiA05fWks7ZtHRkD7NkEsnRgVQmkiMCx/Ho6jGpKiiTWNhPVLQcIavgJZIpQCoTyYzHEwLXE3Rlei0TXfM/2p7nyfRgiWQKkMpEMuPxPEEqayIEVPZTJo4njtgy+EzepidrHW4xJBJAKhPJEYDnCbozNtBXmfg9ulznyE0PzuUdMnn7cIshkQCyAl5yBOAJQXchLXigZXLkZnOZtisbWUqmDZNumaTTadatW8f+/fsB+M///E/WrVvH+vXrueGGG7AsfxHYvn07F198MatWreLGG2/EcRwAGhsb2bhxI6tXr+bqq68mk8kAkEqluOqqq1izZg0bN26ktbV1sm9FMk3xPEF32kRTFZJRA+ijTNwjNwBv2h6W4x5uMSQSYJKVySuvvMJHP/pR9uzZA8Du3bv5yU9+wn/8x3/w4IMP4nke9957LwDXXXcdN910E48++ihCCDZv3gzALbfcwuWXX862bdtYtmwZd911FwDf+c53WL58OY888giXXnopt95662TeimQa4wnozlhUJEJBC/rAzeV5HKG6BMtxsewj1/KSzCwmVZls3ryZTZs2UVdXB0AoFOIf/uEfSCQSKIrCcccdR2NjIwcOHCCfz3PqqacCsGHDBrZt24Zt2zz//POsWrWq5DjAE088wfr16wFYt24dTz75JLYt/ceThWlN3x2wb5lYgYsL+lsmh0uyyUMIge14ON6Ra3lJZhaTqkxuvfVWli9fHvzc0NDA2WefDUBHRwf//u//zgc/+EFaWlqora0NXldbW0tzczOdnZ0kEgl0XS85DpSco+s6iUSCjo6Oybydo5qDHZkhZ4YcbhzXoztjligTrWiZuAJxBKZzFVvuU0iLlkgON4clAN/c3MynP/1pLr74Ys4880xeeumlAa9RlMFHriqKMuBYEVUduW6srk6M+LWDUVubHNf5k8FkySSEoD1jUVOTwNC1Cf3bEyFzynQxbY85dUkqK+LBcV1T0AyN6uoEsYgx4XKZtm+thY2JfSZD0VemnOlQ0ZUHoKIyTjR8+HJppuN3YTimo8zTUabRMOWfwJ07d/KZz3yGK664gk9+8pMA1NfX09bWFrymtbWVuro6qqqqSKfTuK6LpmnBcYC6ujra2tqYNWsWjuOQTqepqKgYsRzt7ekxZ8LU1iZpbe0Z07mTxWTK5HmC9o4MiZA2oQvnRMn81i7/sxPRFTq7MsFxTVXJZi1aW9PEIiP/qI9UrpbOLJ4nmFUdH/a146W/TOmcTVdXFhSF5pYU8VEoS8f1AjfgRMs1E5iOMk9HmfqjqsohN+FTWmeSTqf51Kc+xTXXXBMoEvDdX+FwmBdffBGALVu2sGLFCgzDYPny5WzdurXkOMDKlSvZsmULAFu3bmX58uUYxuh2n5KR4QkxrbOi2roLO/Q+bi7wLRPHFUxW1WI275DKHp44ne/a8q131x35/dmOx7tNKdwjOGVacniYUmVy33330dbWxk9/+lM+/OEP8+EPf5jvfve7ANxxxx3cdtttrFmzhlwux5VXXgnApk2b2Lx5MxdeeCEvvPAC1157LQDXXHMNL7/8MmvXruXee+/l5ptvnspbOarwPEHRRT8daUv5yqQyEcZxPDpS/pAsXVNxXY++BqhfLT8xVeM5yyFnOtjO1C/Mtu2iqKAqCnafWJYQgnRuaAXnuB7dWTuoy5FIJoopcXM9/vjjAHz84x/n4x//+KCvWbp0Kffdd9+A4w0NDdxzzz0DjldUVHD33XdPqJySwfGEwJvGg6Y6U3liYZ2QoZHNO4RDGqbtomuq306lD5bj0p22KIuFxnVN1/NwXIGqKOQtB0Mf398bLabtoqkKQoBluyXH27pyJKKDW+muJ1CA5s4sFckw6iFikBLJaJDtVCTD4nngiOmbYpvOOSRi/uLpuoLyWAjTctE1pWCZ9AouBDju+NOcHcf/m5oGmUNYApOF5XhoqoKmKiWFi7bjkTtEGrfreuiqiu149GSkdSKZOKQykQyLQOC5Ytq27sjm7aDlPPixE0NX0TTFT2fuI7ZfnzH++7ALfzdkaEFfsKmkaJmoqlLiZjMtF/sQ/chMx0VVIRLSaOnKTds4mGTmIZWJZFj8mMn0DMB7QpCz3N5sLQXChkpdZQxVUXC90ioTT/hxg/Hei2m7KIofl7FsB3sK25oUCxbVomXSpwo+Y9oIMbTit23fogkZGjnTIZN3pkpsyRGOVCaSYfGEr1CmY3Gc5wlypkMsrCOEQC0s8OXxEJqmDshCE2JikgnyphO0bAGF/BR2CHA9gRAee5p6UBRKXHk500XAkMrStL2g5Yyhq3QUkhckkvEilYlkWIq73Ono5nIcD9N2iUZ0HNcjEtJQFAVdU4mFdWyntLOuJ5iQZIKc5QS1GpoG6SlMEXZcj4PtOe55dAe7GlOgKLiuwPU87EIsZajMX8vx3WPgu7pSGUumCUsmBKlMJMPiuh6apkxLyySdtxHCH9PruKKkEjxs+JaJ188ycQ/hBhoJQgjyphu0bAkZGt1TOKTK9QRdPX76c1chxbeoSIoMpiz7usfA7ybhCTEqV9dwqceSoxepTCTD4nj+AjQdd7DFSYOxsI7jlCoTQ9cKPax6X+95AuGNLzPNcf1RwMXWPrqmYtvulMVNHLe3ViaVtaBQVGoVlYkQgyoTfzMgSloShXQ1UEwjIW+5NLVnhn+h5KhDKhPJsLiuQFMUJiCjdsLpyfq76mhYB8W3EorouorreSUWVbEAczxuLtvxBimqn7q4iW27gTLpKbjXXE9gWX5SAMrgVqRfKV9aVxIOaXSPwtWVNR2ypjtta44khw+pTCTD4npi+lomuYJlUsjmMvr0nArp6oCuwW4hM23cyqQfqsaUjdC1HJeeQjpyT8ZCURVsxyVj2v79CxCDKJPBpk4qigKCEbu6UhkLz/NG1cJFcnQglYlkWFxPoGkK0zBkEgS+IyENBT9DqYih+xXwJQH4woI6nphJznLQ+vW7DOlaYCVMNqbtlVgmmqpg2y450y/UVBSGtkwGUaKGrozI1eV5frxEU9VpmYwhObxIZSIZFtfz0BRlWs4zKcZMQoZGJKyVxAMMXR2Q0uwKUBTGpRj9Rbv0q2PoKnnLnZJnlDMdUpnemImqKuQKxYqapqKolPTrKmI57qAjHEbq6spbjq+LFKallSo5vEhlIhkWx/XdXNPRT57O2qiqgqpANFTaai5UMB9su6+by/OzmMZjmZh+WnBP/wyufn2yJgMh/Hn3noDyeIi85eK5oiReoyp+G5n+2H1qTPriu7rEsCOAM3mH4un9e55JJFKZSIYlm3fIW860dG2k8w7RsIbnMWBAVNHlZbm98QDPA00dfLEdCY7rxwsOtGX49uZX2X0wFfxOUfwA9WTieoKuglUyr96fLZHJ2yWWllqoO+mPafemMw9k+NTvVMYkHFJRmJ41R5LDi1QmkkMihOCxF/dz3xO7pmWdSSZnEwv51e+hfoO7DMNfOO0+O24vSHMe2704rgcK7G32Bxm9/HbvUDfDUCc9buK4XnCNeXW+MknnHVzXo+jBUpTBLQfL8XjprVa27+kc8m8f6rrZvO/eU1RKmktKJCCViWQYhPDjEqmMNWT9wuEkazpEI35acN/gOxCMGLb6NHZ8+Z12OlPmmGMbxdnrB9uzAGx/tysY3xvSVdI5e1KfkeOKIF4yt9ZXJj1ZC9f1gkw2dYj4lu24/P7Vg/zXEzt5dWd7ye/UYRSE70bza1RURSlR0BIJSGUiGQZP+P74vO0C44s1TAa9HYOVPr2yfEIF5eL0cXM9+txe/ryrfcz34RTcR03tWSoSIRzXC3b6iqIg8Dv3Thau65HKWCSiBlWFyZI9WZtEzCBk+PerqAOzuVzPI2s65C2XkK7ywFO7+fOuXoXSv2Fkf9I5C7XwfLV+nYolEpDKRDIMnucrE8v2O+1O1qbbHGPgOmf6HYMV/JnvfQkVLBOzEIB3XH+glWm7Yw4g246L5Xh09Ji8d0kNVckwr/TZ5SvCz3qaLPKWX7BYnggRDmmEdJWejEXI8DPZfrb1TZ56tQmn32LvuoLujJ/+e+FZC5hfn2TL73ezvyUN+PO9D61M7OB5qqrSW20vkRSQykRySIqWCfizMCarDf3BtsyoXU+u55GzHCIhDV0f+FE2Cjv1YrC9mGllWu6YU1st26WtOwfA7Oo4Jx9bzbtNPXSl/YXa0JVJ7V1l2S6pjEVFwrdKkrFQMIc+Zzrsa0nT1JFFUFrl77vH/NfVVkS57IPHEjY0nn2jGWDAkK3+FIeNwcBRwRIJSGUiGQbL9oJF3rInZ3SvJ/y+UqMNimdyfpPHaFgPXFp9CQXZXL78uUKmVd52A3fVaLFsj9Yuv2377OoYJy+qBghiECHDL16cLKWbtx1SWZuKhD8mOBk3ghTlxja/Z1a2UM3e15Xnel4w970yESJsaJxybDXb93YF6dW2M/icF6fQZblYo6KqyrQe4yw5PEy6Mkmn06xbt479+/cD8PTTT7N+/XouuOACvv3tbwev2759OxdffDGrVq3ixhtvxHH8L0RjYyMbN25k9erVXH311WQy/hcmlUpx1VVXsWbNGjZu3Ehra+tk38pRSbpPixDTdifFzeV5wl+wRqlMijvtsKGVtFEpUlQmxWBxruB+ylvumGMmluPR3JGlLGYQjxpUJMMsqE/w553t/jyVQqbYWN12w9HebeJ5gvK4r0zKYqEgu6uoTIpKs69isB0/1hIJaUQKKdR/cXwdnif409utBUUx+MyawXuRIVuqSEqYVGXyyiuv8NGPfpQ9e/YAkM/n+epXv8pdd93F1q1bee211/jd734HwHXXXcdNN93Eo48+ihCCzZs3A3DLLbdw+eWXs23bNpYtW8Zdd90FwHe+8x2WL1/OI488wqWXXsqtt946mbdy1JLuU5hn2pPT4M91xYBW8SOh2FIkEtIHVKRDbzZX0SWTN93C/51xWCYuTZ05ZlXHg2PLFlXTnjJp6fLdX4jeBX0icVwvaDlfkQiTydmE+1hCjW1+hlmvZdJ7ru14dGctKgtBe4Ca8gjHzE7y0o62gnIdvD7Fdj2Kecdbfr+bZ15vKjSTlK4uSS+Tqkw2b97Mpk2bqKurA+DVV19lwYIFzJs3D13XWb9+Pdu2bePAgQPk83lOPfVUADZs2MC2bduwbZvnn3+eVatWlRwHeOKJJ1i/fj0A69at48knn8S25ZyFsdKRyg/q4kjnehdF05ocy8T1PBx39NZCMTYRDqlBfKQvRpDNNdAyUZTRF945rj+Iq707z+zqWHD8+HkVALz5bhcAIUMN0ncnErdP6/mKRAjXg0TUCGaSNBZaw+cKBaZ9rYz+sZYiy5fW0Z2xePtAt3+NQRSEZbso+MWRr+5s550DfqHmdMvskxxe9OFfMnb6WwstLS3U1tYGP9fV1dHc3DzgeG1tLc3NzXR2dpJIJNB1veR4/7+l6zqJRIKOjg7q6+tHJFt1dWJc91ZbmxzX+ZPBeGTqzrtUVScG7PDV3R3BvzVDo7IyTkUy3P/0MVNbm6Q7bRJry1FZFacyGRn5yTv8gsHaqgSz6sqoLCs9VxTaqYRCOjU1CYzCIuh6glgsTFV1YkBtSl+5+pO3HHKO705dMr+KygrfOqmsgEUN5by9v5uPfGAJnhCksxZV1YlgquFEUFEZwypYDgvmVpIzHepqfBlSeZeerE1VWYSOVJ5w1KCqOk4y5rvDWnpMutMWpyypDeQGOLMsxq+f28cr77RzwqIayisGvr9ZV1AtFLbv8T8LmbxDeXmUisr4kM9qujMdZZ6OMo2GSVUm/Rls56soyqiPD4WqjtzQam9Pj3lnVVubpLW1Z0znThbjlamtI01MVwYsrgebe/9mV3eetrYe7PzE7LqLMnelTbq6s7S0hHBG0cb9YIsvm205dHVlcczSc4NmiD0mLa09tLalg9+1tKdpaYkOqJrvK1cmb6MAsYgB+DUtu/b5NSXJiEpnV++QqGPnlPHr5/exc287VWURejIW+w90Ba3xx0ttbZKmlh5a2jPEIzqZdJ6ejIVasCRefKMJgEWzk3Sk8jS19tBSGSVfUCa79nbieoKoUSo3wIkLK3nmjWa6OrM0Gyp2vlQpNzZ1IwS8+ravSDt78qS6czSHNMoT4Wn3XRiOI/H7OxWoqnLITfiUZnPV19fT1tbbfqKlpYW6uroBx1tbW6mrq6Oqqop0Oo1bmMpUPA6+VVM8x3Ec0uk0FRUVU3czRxiiMIWvP6UBeGdSWqpYjutnE40y3TSdtVAUfzyvOsgmoxiUd10/SynXp/4jPwKXXXfaLHFXOZ6guTNHPKKTiBolr126oAKAN/d2AaAqKtkJnm9i2g49WYvyRBhRyK5KFJTFjn1dKAosmlMGQN70gpkmfqzFT13u7+YCqEyGEcLPchus4NK0XDQVdhZcYZbtYbku9nScliY5bEypMjnllFPYvXs37777Lq7r8tBDD7FixQoaGhoIh8O8+OKLAGzZsoUVK1ZgGAbLly9n69atJccBVq5cyZYtWwDYunUry5cvxzCMQa8rOTRCFKcPDvxdJmejKMU6hNFnXI2E/S1pmjuyo26+2J21iYZ0FEUZtIFh0cqyPQ8hSivTTcsZNuCfNZ2SgVeO49HalWN2TXyAhVyRCDO7Osb2d33LxTCUoEhwojAtL0gLdl2/F1lZzEBRfNdTbUWU8kLKcN5yAuVsOx7dBaVYOYiLsugKy+ZtrH4KopgW3NKZJ5N3WFxQVtmcI1uqSEqYUmUSDof5xje+wec//3kuvPBCFi1axOrVqwG44447uO2221izZg25XI4rr7wSgE2bNrF582YuvPBCXnjhBa699loArrnmGl5++WXWrl3Lvffey8033zyVt3JEIQSF6vaBi2s27xAJ6URCGpbtTopl8tiLB3j8pQOjTjVNZayCG0kMGpsoFtm5rl+5nytRJofOTBNCkDPdkhG1ubxDR49JXUV00HOWLqjkQGuGVMbC0FWy5sTON8mbNt0ZPyPL8TyiIY1o2CBRcMPNqYkXWsv4yqT4PB23t8akqGz6koz552fyzoA2KX7tCexs9K2SU5fUFF5ryyp4SQlTEjN5/PHHg3+fddZZPPjggwNes3TpUu67774BxxsaGrjnnnsGHK+oqODuu++eWEGPUgR+Wu5ga2vWdIiGNBTFd2+MJh20K20S0rVh4wapjEXecrFHoUxsx/P7ckX8vlxDzenQtWLTQ1FimeTtQ2ePFRdVgcC2PcIhjfaeHJ4nBt3dA5wwv4L/fukAb+7t4owT6qDQPSARnZg9W1vKDK7vuoJwWAPLIxEz6MnZzKmOBc8636fK33L8yYzJmDFoCnVZwTLJ5GzMftaG7frzX945kKK+MsqcQsA/nbNlfy5JCbICXlKwTAZPkMiZDtGwTjikY9ouo9lod6dN7BG0Ks/kbUzbHdFri5i2S85yiYZ1NFUZNGYCoGtqoYbFVyBFpeNbJkP/fcvxQAgU0dtmpKXTr3yvSA7c3QPUVESpTIbZ01SYcaJM3LAs1/XoSPnXr0yG8TxB2NAJh1TifSwTQ9fQteLUR/8GTcufzDiUEoxHdVRFoSdn+5Xt/VKKbdtlX0uaxQ3lgRWTzjmypYqkBKlMJIBfMzDY4po1HSJhjUhI85XJCBcQIQTp/PALjuf5NRL+tUYesM7kbPIFq2mw3XYRXVN9mYW/qCaihZ37MJaJZfsjblW1tz6lvbiYF4LY2bxDJldanNhQG+dAq58tpaoKzgQV9tl9guh+t2AFQ1MI637cRNdU6it991ssopPrU5iZt/yYyWDBd6AQyDf8Svp+xYim5XKgPYPnCRY3lKFrKrGITk/WwnNH3wJHcuQilYkEMUTwHfyq8WhYJ2xomLY34tnptuMVlM+hT8iZve6SbH7ku/jujEnOcomEBu/LVcTQfctE4LucwobfadcaRjFm8g66pqDrKpmcXwTYlfazx4pxB6dPgLtIQ02cnqxNKmP5Ew8nyBXkuILutIWmKpTFQ6D4ilLTVE4/oY6/WXM8WkGpxsJFZeJfuydn0VMI3Ash6Mn4P/dkrCBbLRkr9PgSoqQ7QM5yaGzNoKlKMIyrrNhcUlHwZBW8pMCIlUk+n+ett95CCEE+n59MmSRTTDH43j8gXUynjRUaKY6m267leLiON2wAuqOnN+Mpb45sNLDj+llNnicIh7QhCw/BD8I7xWwu2yWkF6ws69DJBDnTQddVDE0la/oWVnfGoiwWQlPVIDV3Xl2CnOkELsKGWj+mcKAt4088nKD+Vbbj0pU2qUyGg5ntuq6iqQqxiEFDTW8hom+Z+O+V63l09/RmctmORyxisGRuOSceU0UyFiKbtwvKpGiZ9MpsWi4dPSY15ZHAAiyLG/QUlNBE3Z9k5jMiZfLyyy9z/vnn89nPfpbm5mZWrlzJSy+9NNmySaaIYrvy/iETx/WwbI9o2B+La44imytvOqjawPG4tuPS2pUNfu7OjL73l2m7Qe+rSEgbtJVKEUNXg2wu03LRdQVD9112Qym6YqNGTfUD+57nB++702ZQHW45HomoQVk8RGUiTK5gVc2qiqGpCgdaM/7EwwlyA5m2F2RyuZ6Hrvu1Nbqm0v+NK1omrisKPbkKNSbJMI7rEY/qREI6mqrSUBNHCIVEpKBMRG/L/mJacHt3nury3kJG3zLx37fRpnNLjlxGpExuv/12/u3f/o2KigpmzZrF7bffLhsrHkkMkRpc7H0VMTQiIR3b8UacwdOTswnr2oCdq+14dPZYwbWKKavgxzFGoqxyeSfIzAqHNAxtYBV7EUPTcDzfzWXaLoauEQ5ph+wcbBeC5kEtiRBkzUIQu+Dism3PdzcBs6pjCPy+VrqmUl8V40Br2ndzTZAbKG/adKUtqgqZXJGQH/vRNQX6dYuIhjWyBWvJcvq2ng/juhAN9WbXhQyNhto4IaOoYF3MQsJB8f3uTPuWSZFkPETOdCctVVwyMxmRMsnn8xx77LHBzytXrgyq0iUzH4G/c++/uKazxUaKGpGwv2CPpBuuEIJs3iYcUgfs/j1BIdPIP96d7nVzWSNsDZ/K2kHmV8TQBi1YLGLoSqEC3t/dhzSViOG7uYayGkzbLW0GoEAqbZHJO4FlIoQIajoMXaO+MhrEfBpq4jS2ZxGICXMDtXblsB2PyrIwjiuIFNrAFNOfixadP2FR8VODRcGiyvixlmImVn+3YHk8FCiLrOkXIxbfw+6MPzOm+HvPE+iFjLhMXqYHS3oZkTLRdZ3u7u5gp7Zr165JFUoytRRTg/u7mIKuvIZKNOwvRCNxRdmOH6jXVHXAzrzoQirWM3Rl+lomgw9n6ouf/WXTnvKVUHVZ5JDNFHVNw/V8q8uyXQxdJRLWClbQ4AthznIRCN58txMhBIauBq65ykIrExSFcJ++XhXJMEpBvobaOLbj0Z4yJ8QN5AnBgcJ43SAtONR77bChB4kOedstqTXJ5h06e0yqyyP+91cZqEwURaG60CQzk3fIWS7vNvWwrzVDJue/P9XlfqaYZbvEC38/ZzoT3jJGMnMZkTL53Oc+xxVXXEFTUxNf/OIX+ehHP8rVV1892bJJpoji+j1AmRRSdsOGRqxgmVjW8KN7i5XRijJwgJLr+kHhfMHC6U5bhA2/KNK0hu/9ZTkuAn8QVFUyTKRQZzIUvmXiJxdYtoehq0TDekkdRn8yOYu9zT1s/u+dvL2/u6BM/FklFclwEC/pWyipayp1lVGyeScIwje2ZQptasZnnZiWS0e3n/RSlYwgoKRBZdhQA4tOFNrSg5+Jl7ccOlJ5assj/vCuQhZYf6rLe9OdU1mLrOlQkQjR2VNU2v7vbVdQVVA8OdOlI5WXExclwAgr4M877zwWL17MH/7wBzzP42//9m9L3F6SmU06Z9GTtwe4mDIFyyQU0hD4C6ffnwsOUdpB1vT7eRU7P3tCBEWFjuf5A50Kf7sna5GI6qRz/uI0nJfLcQWKEDS2ZZhf77fsPrQy0XBcD8fx5TB0FU1VDxn/yeYcmjt85bH7YIrj5lUEWWeViTC27VFTPrClSmUyQktnjop4iGhY40BrhiUNFXieQD2EK244LNuloyePovhzTLJm7zx28OenFK0vFCgruLNypm9ldKUtTjm2Bsf1iIS0QTtvF+8nnbMLdSw+bd15yuOhEuVVdPWlczauJ8ib7oR1R5bMXEZkmTz//PO0tLSwZMkSjj/+eLq6unjttddIp9PDnyyZ9vziyV385rl9A2MmRWWiq72uE9sZ1DIx+1gsmZzTW/vRbwiV43iEDb+jrhCCdM4mHjEIh1TytjOsW8j1/HNSWTto7aEdYvSAofsLbb5QeBjSVaIRrSDnQBeN63lYtktzp+/W2nPQbwtuOX5wPR71M9v6BrH7XqumPELe8phTE+dAWwaUwTsLjIZ0zqarx/TTkjUVRKl1EdI1PK+QYRYxguLEnOXQXrCoaisiOK4gGh580fdridRgBHCRvplcfjq071qMhDRSWQsVhXSu11Xper2V+pKjixFtJ2677TbefPNNlixZgqqq7Nixg9raWnK5HLfeeivnn3/+ZMspmUTSWZus6QxolVLsmBvqM2PdL1wcmO779n5/dsecmjjZvN1n0SrNNHJcgab6RYOm7ZLO2cyujpG3/DjKcBXjtuPS3OkvkHNqYigwaF+uIkahAr44kz1kaMQKisAsVMH3Pd9xBCiCg+1ZFAWaO3NkcnZhSmGhjYqiEAkNnkFWmYzQ2p2noSbO7189iGW7jDehqydn05kyqSoLB1ZOX2XiFyuKwGIqdv7tWwVfWxEtyQLrj6aqxKOFwsUCQgjauvNBc0fL8XzFb2jEowapjP8+t3akqav0J0929pgcbM9SnggdUslLjjxG9G7PmTOHn/3sZzzwwAP88pe/5Oc//zmnnXYaDzzwAP/6r/862TJKJhnfDTQw+J3JOYQNDU1VAz+8NcgckGKbEdN22bG3C9ctXaD7xkFau/KFwLdCNu+QzTvEIkZvV+JhsoNs26OlM4eiQF1FFP0QBYtQqID3RJCFFjI04rGhkwlczyOVscnkHZYdUwXAnqYeOnvMoOgvHjGGVGAhQ0VV/D5ZQkBr9/hiCo7rYTsu7al8UGMS6VdXU0wP9vBrTCrivmWSN/1Yi6ooVCX9xIHBhoEBaJpCImqUWCY9Wb8zcNEysW2PZMwgZGgkC4pH11Vsx5+D4rgezR05PM/DtGSW19HGiJTJvn37OPPMM4OfTz75ZPbs2cOsWbMmTTDJ1OG4HrY7cFZJJm8HO/DizIv+lonjerR15YlFdKJhnUTMINrXfy4IduaeEPz4odd55o1mVBU6urNYjkcsrBMJ64UFaZgAvOtysCNLfWUMTVMO2UoFeosW84W6lJCuBl1yzUFSkV1PcLDN76313uNqCRsauw+m6Er7va0c1yMWGbquRVEUomEjeG65vD2uWgzTdjEtj0zO9lvPD+Kq0lQV4Qk0xU/jjkZ0DF0lb7t0pn2LRtNUUIZ+XppaLFzstUzaCkH/IC1Y+MpKU33FEwwOEwrpvE1Hj684NVUN3IqSo4cRpwY/9dRTwc9PPfUUhmHQ0dGB48gPzUzHcQWOM7BpXzbvBPUlkZCOrvnuqb4b7Y6ePEL0uppU1R/929iW4V/vf41cnyFU2byDaXscbMsWXuPHJaJhjWi4UDQ3zMJrWi5N7VkaauK4ngjcb0Nh6CqeEGQLmWmGoRHv0+yxv9HgeoKD7b4ymVMdY8GsBG/t7cK0XSqSIVyPQeMlfYlFet2C+WHmpgxHLu8EQ7aqyiKDuqqKlklZPORXxasK0bBfZNqRMqmtiAZW51CWnKYqQSv7ooJt76NMivGSYvuaRNQgk/f7f4VDKh2pPC0d/hRKXVcGjUdJjmxGFDPZtGkTf/d3f4dSaOwWDof53ve+x49//GMuu+yyyZZRMsk4roeAAWNYs6YdLJwhQyVs+EHyoEDO9fwFJDrwY/Ty2220p/IlqaPFXW9zZw5DV2nv9hfJWNggFi62oT+0e6SlM4dpu8ypjeN5Qy+ORYo78aL7JqSrlEX7WCb9FnrH8Whqy1BTHiFkaCycVcaOff5gqGK3YEMf2jIBiIaNoJbDtNxgfO5Y6MnZQSJE5RCuKqUQwykruLc0VSVaCJB39OQ58ZhKf/6JoQ3Zql9RFMrjIYTwLdJkLERbd56Q4SsO3yLSgr5kxQLIVNoiZGh0pU10VUVVFUK6n61X7F8mOToYkTI5+eSTeeyxx9ixYweaprF48WI0TWPp0qWTLZ9kCnAKC3h/P3c271JbYaDgZw+FQxqW3TtbvKvHRDAwAC6E4K19XYCf7lt8fVGZpHN+TKJYYR+N6MQj/oJl2kNbup7npwSDX2VeTPU9FMWFtzjL3tAVkvHe0bb9lYnluBxszwQdco+Zkwx+V5EMw0iuqauEdNWvnRmBtTUUQvgFmkVFWJkMB4WX/alMhoJaIE3zLZPG1gxCQG15tNCTa/A5LMH9FZRlT7aoTHLUFIodLcehpsxPH1YUhcqk7/rqSptUxnXfIilYY37rfb8v2FAxGsmRx4iUSUdHBw8++CCZTMavG/A83n33Xf75n/95suWTTAG9cy9KF/Kc6RAOacGCEDZ6u+26nkdLZ3bQ+MHB9mywAPadRZ7q449v7sjSU5gFEgvrxAsB/lx+aGXieh5NHbkgBTeds4e1EooLbzF9NaRpRMM6uqb48Z9+C31XxqQ7bXHGUj87qa4iSiyik807lMdCeAysIB/smoqiEgnp5A/RUHI4LNtvA9PSmaM8ESq0zvcGvX5tRSz4t6YqRMJ6UDzamxZ86GdVWVZUJhYQpz1lsnCWr0w9j+A9Aqir9JXJwfYMlfHyAe+DUuzSLJXJUcOIYibXXnstTz/9NL/4xS9oampiy5YtqONI+3vggQdYu3Yta9eu5Zvf/CYA27dv5+KLL2bVqlXceOONQSymsbGRjRs3snr1aq6++moyGX9nmkqluOqqq1izZg0bN26ktbV1zPIc7RQXu74zvb1C+/lIn+FT0bAepNN2ZyxcMXiNx1v7uih6N/rOIk9nexVFU0c2qCmJRfQgWyxnDd2G3nEFTR1Z5lTHUFUFBUYUgAeCIVbFNOdwSMO0nAFFkvtb/M/XrGp/cVYUhUWzy4hHdH/HP0y8BHwrTteV3lb3Y1Qmpu3X7uxrSbNwdvmwrqoiiqIELU+KdSEChkwLLlJdsDZSWRvLdkllrN4aE0qf9ezqGMmYwY69nYP+LVUliFNJjg5GpBEaGxv54Q9/yIoVK7jiiiv4+c9/zt69e8d0wWJtyj333MMDDzzACy+8wNNPP811113HTTfdxKOPPooQgs2bNwNwyy23cPnll7Nt2zaWLVvGXXfdBcB3vvMdli9fziOPPMKll14quxiPg0CZ9J2RbvrB6UhIDxZkf9qiV0gBzRIdotZix94u5tUlMHSVnNm7My9aJtGwRnNHjp6sTdjQ0AvZQTB4HUsR2/Vo7coxuzi7Q1FG5HICf2FTKKTuqgoRQytpOFmkOCVxdlXvTv+CM+ZxxQXHDZvJ1ZdYRO8znXJsbq6saZMzHbozFsfMLsNxvSGLDvuTLDzPykQ4iCsNl6zgz0qB13d38JOHtwN+S/1iG5a+zzps6CyoT7Jjb+egyj9kqCXFjJIjnxEpk5oav2hp4cKF7Nixg/r6+jFncbmui+d55HI5HMfBcRx0XSefz3PqqacCsGHDBrZt24Zt2zz//POsWrWq5DjAE088wfr16wFYt24dTz75JLYtM0jGQtGnb/WpNSkWLIYNFUP3d8KxsI5VKDS0HS9YpF7d2c5jL+7H9Tw6e0yaO3McP6+CWKEHVjFLLJ2zUQsT+5o6fFdYImqA0tvRdrB03SLdaQvXE4UUWY+QfuiRvdAbM8nkbQxdDdwxkbCOaXtYfVx7QviWT3V5hEifRTsRNaivig1o334o4mG/HiNvudhjVCaZnB1U4h8zp2xErqpA5kL6c02fTK5DzX0Bv21OImqwtzmNrql85JxjOLahoMT6tWEJGRoL6pPkLdev9O+HrqlkzZF1gZYcGYzom1FdXc2Pf/xjTj31VO68804SicSYW6kkEgmuueYa1qxZQyQS4YwzzsAwDGpra4PX1NbW0tzcTGdnJ4lEAl3XS44DtLS0BOfouk4ikaCjo4P6+voxyXU0U9ydO4VW7YrSR5mENEKFBThWcHP1TRnuzlg89PS7fmZXZy4IXB8/v4LXdneUVGGns37dSn1VjLf3dxOLGMSjOqqqUBbvLSQcqi6j2Lm3LB7CdjySwwSUoXc3nTPdQl8u/3g0pNGdsYLuxeCnBbd05lg4pwyATMEtF4/5nz/B8JlcRcIhjYihkcpYY5pp4glB1nRpbMuiayoNtQkONKVKOhUfivKCcvbjJR6x8PDuMV1VWHf2QiIhjbm18UB52I4I5rgU0VSF+fUJFAV2HugO3nfPE0FfNvDfz5FaU5KZzYje5X/8x3/k4YcfZvny5Sxbtow777yTL3/5y2O64JtvvskvfvEL/vu//5tkMsmXv/xl/vCHPwx4XbFJ4GDHh2I0cZzq6sSIXzsYtbXJ4V80xYxFpuKsDwAjrFNTk0DTVPZ1+At3TVWc+royKpJhaqpiuJ4gFg9TW2if8dAf9wKCVWcu4NfPvsvb+7upr4qxeH415YkDZHI2ybIItbVJLFeQiBosnFPB7185yL6WHt6zuIa6mgT1BbeSqmlUVScCt1df8ttbAJg3q5xYzGDerDJqKgY2XCx5JoVW9X78R6emOkFtbZKKsihtKZNoPBw8t6zpd8ytq4xRWRFHNUyE8Od9KIqCapjMmV02IoWSNB3KkmH2taZJJCOjfm/ypkN5eY6mjhwLZiXRNJWKiiizZ5WPaHGeO6ccgAVzKojFItRVx6itOfRnPm86LD3GDlKMi6hpk3kN5VSW9Q7Ish2P1rTF/Pok7zanqayI4zge3/6Pl1g8t4INHzgWVTeJJyNB+/rpxJHy/Z1OjNgyufLKKwG47LLL+MAHPsDpp58+pgs+9dRTnHXWWVRXVwO+6+onP/kJbW1twWtaW1upq6ujqqqKdDqN67pomhYcB6irq6OtrY1Zs2bhOA7pdJqKiooRy9Henh6zCV5bm6S1tWdM504WY5XJ7BMn6e7O09Lag66pHDiYAsAxHTo7M9h5i2K02jQtOrv8wPBLb7XwlyfP5swTaklGNX755G5OWFBBZ1cGQ1PoyVq0tadpTYToSOUwNBWvUM8iBOgqZHrymIVYREd3jpaWFLnYQKvj3cYuABTPJdXtkImHEIdIJQbIFIZv5UyHeFgn1Z2nNdKDofrV6S1taVqSvrJoas8UlEeYjs406ZxDIqrT1JonpGvkTIeuzuwhr1fE8/zuxjnToa0tTWufSYUjIZW1aG3LsL8lzdnv8a3tru4c3V0Z0iPYNNUmQiw/vpa51RE6urIkIxqtI5hD09WVxe33THuyNqlECMfsdSMLIUh15zhufiW/fX4vjc3dPP9mC41tGSzbpfPU2eRNh3f3uXhDVMN7hemeU93D60j6/k4lqqocchM+onfx3nvv5Utf+hIdHR1cdtllfO1rXxtzWvDSpUt5+umnyWazCCF4/PHHOeOMMwiHw7z44osAbNmyhRUrVmAYBsuXL2fr1q0lx8Gf9rhlyxYAtm7dyvLlyzGMgbtZyaGx+wSgbbc3ZlLsZRU21KDFe7FzcLFD8KPP7SURNfjL9/htdU5cWMUX//oUzjl5dvD6rNmbzZXJ+W6uqkSIUMF/HwvrQSxDU5VBM6yKdPZYhHTVb1UiCP7GoSi6uYTw/100KmIRvZAa3Fv531kYb1t0o8XCOjXlUexC0kFxQNhIUFWFeNQIJjyOtgo+bzm0dmXxhGBubQLX9dA1ZcQLbySkc87Jc4iEdASC8AisKa1QST8QMeBZK4pCyNBYMq8CIeClHa38/pWDGLpKW3ce0/LTgruz1pD33p02aSk07ZTMfEb0ybzvvvu44YYb2LZtG+eddx4PP/zwoK6pkfCXf/mXrF27lg0bNnDRRRfhOA5XXXUVd9xxB7fddhtr1qwhl8sFltCmTZvYvHkzF154IS+88ALXXnstANdccw0vv/wya9eu5d577+Xmm28ekzxHO32zmRzXCxbyoDFiqHf4VKywmOYtlz/v6qCxLcsH/6KhpJYgGtYDV2QsYmDZfqPCYkuTSFgjHNKpK7inomE9CAxHCrPZh0ql7U6blMVDeJ7A0JVhg+9QmoGk62rgCk0U4i1WQVGAX4QJUJYoxGTiBvGIgaYqWLY3ZPbaUFQUiyPtodOdhyKTc2gquBrn1SWwXW/EwX8oKgYKmVjKiBSvqihoqlIiq+f5lsNgzzqkq8ypiRM2NB578QCaprDmzPmAX3+iFv5WX+u3Lz1Zu2RsMxCMJZDMPEb06VQUhZqaGv74xz+yZs0adF3HG0df7auuuoqrrrqq5NjSpUu57777Bry2oaGBe+65Z8DxiooK7r777jHLIPHpq0zsPtlcuWJjRE0NZqzHo4Vq8pzN4y/uZ3Z1jJMXVw/5t4sz0osKImv6cYuQrlJbEWV/a4ZYWA8C/NFChtVgbeg9T5DKWJQXrIb4IDGVweibDmvofe4lGG3bmyDQWVjYyuIhUqkcsbDfHbiqPMK7TWnm1Y0ukFwe723b0j/+5wlBU3uWWEQPKs+LFOevNxbaukTDOo4z8rRgKAwMExQsLGPEbU2Kg7aKXQ0OlY4cCWt4KBwzO8mbe7s477QGjpvnx2oa27MsnF2GqihkSkYS9N5jT9bPzrMdN4hD5S2XpvYMx86tGPG9SqYHI7JMQqEQP/rRj3juued4//vfz7333ks0Ov2CapLR07dLr69M/H/nTAdD92syiq6VeMRfwJ98pZFU1uaC0+cdcpEqusVylksm5yCErzAiIT3oRBvr0849qMtwBu7iXc+jJ2tTngjhuCKQZTj6WiaG1ptKHIy2tbzAEiqOqC2LhVAgyJyqSISJRfRha1r6U0zP7ZseDb5i3N+SprE9M+gMdbvQdHN/a29bF8cberDVYGiqgqIUBmYN0jttKAxdLbFMbNcLNgX9Cekajudxxgl1/MVxtSw/vo5YxKAiEQra3oQMdYD1Ab5F6F9GCTYu4GcR5kYwGloy/RjRt+PrX/86e/bs4Zvf/Cbl5eW8+OKLfP3rX59s2SRTgNOn6t0qUSYuIV0taaRYbOjY2Jbl+PkVLJh16OyT4iKUK2RJQUGZhDWOn1/BX61cTH1lFL2gTIoV9oO1oc+aDlnToazQjDA8QpdTiTLRlSA9tmgNZPI2puMvZt1pk1hYR+C7xHqLNXVqKiKjbg1SVHh5yy1pw/9uc8p32cUM8oPM/bAK3X7zlhsoE0UZvo1LXxRFQdfVUVs0IU0riXG4h5jOWHwPFs4uY+3ZC4JNwZyaeKBMDF0lmx9YHJq3HBACXVOCHm3g9/oSnhjQzVky/RnRp2zx4sUlFeayJ9eRg93fzUVvny5DVzH6zBovxhlUBc7/i7nD/u3AMsk7QZPHeEQPqupXvHcuexu7+lg+Oi2dOaxBCmKLHYbL4yFQxIjrLfouwCFdCxa8usrCzPOsFTS47M5YJGIGlu0GM0+KzK6Kj+h6fSkpxCysjqbl0pO1g9hPbpBMp7zZGy9pqI37u/QRJhz0JaRr5PLOqJSgUXBzFVEO0WUgEtJQMgO7A8+pifPGnk6yeZtYxEDgz5NJRHv/TipbLCJVSWUt5hDHdlxyeQdF8ZWuiuw4PJMYkTIpVpr351e/+tWECiOZegbGTPx/5y2XkN5bsAh+9XfY0Dh1SXXQs+lQ9HVzpTL+7jMS0goZYoVFS/T652MRA9N2yeQdbKe0oWFxrngyZqBpgweEB0Mr9PAS9E5BLP6dkK4W+lD5C3pP1iYZNbBdj/IRxmQORSJWtEx6uxNbtotC7+wXzxU4rldyP5m8TUtnjrChUVMewXY8KspHPwbX716gDtu/rC+GriH6GUtDKbHiqN+2rFmisGYX+po1tmc5tqEcXVNIZazAtViMl0RCvnLP5C1sxyVrugh8K8zzBMgekTOKESmTm266Kfi3bdv89re/Deo9JDObvm4u2+lNYc1bbiGVtnchUVWFqz9y0oCsJiEE6awTVLMHf69QXZ417RI3l6aqxMI6tuOCogRBcT9d1/eXZ02Hcr3XOmjr9lNIY2EjCJ6PhKK7p6icijtoXVMpi4dKquDTOZtZVVE/XjLKzK3BiBYWy3yfmSaZvI2u99lxKwyiTByaO7P+jHvFzySrSIbBHTwraigMw1/sRzNTRFP9LDDwC1qNYdKRy5MRDjb1EOqje+dU+1ZcY1uGYxvKCRsa3RmT2dWF+3H8OJWqFppRFuIm3WmTkK7iuGJcA8Ukh4cRfSvPOOOMkp/PPvtsLrvsMq6++upJEUoydRT7RqmKUqgz8Y/nLX/+e/9+TmWxEJZTuqhlcg6xqF5w3/iLV7rQdysS0sibLj2FdM9oyFc4sajfnl1Ve+MYsbDuT21UFDpS+SAbCnpHyEYjGhFjdFlVuqZgO35AvajsFEWhPBGiK23iuh6W7ZLJ2YE1MRGt0zVNDZIKiu7EdM4pbbhYyLiKFG7VdjxMy6G5I8dZy/xiRU8IEjGDTM/olElY11Cjo3MV+VlgxeFnonQE8yAkogYepQt/OKRRXRYJJmlqmoqd92Ne8YjhK9e+19QUejKW72aM6tg5h3Eki0oOE2MqPe3s7KSlpWWiZZEcBopurkhI862UfpZJ/9Yher9sH9Py0zoXzkpSXxWlJ2OTN30/fW1ljGhYJ2f5MRNF6Q3iR0L+WNm+C2vR4nALtQZ9XXCdPSbxiI6KGsxXHynFaxi6WtKfqjIRpittIYCOlD/oKxbWSURDw/axGglqYQJisXOw63mYtls6HVIRJdMlTdultSuHJwQNNX68RFWVYdvHD0YyHqJqlJX3uqYggJ6CxZYcpBNBX4rB+f7ZV3NqYsH4Y/A/X7saU6RzNumshaGrHGzPFCZHqnRnrEJfOF+ZSctk5jGmmEljYyN//dd/PSkCSaaWvsrE7lO0aBZiJppauqgamhp80V3Pw3Jcjm2oQFNV6itjOI5HZ9riuNlJBH6X27zpBk0ei4HzsK6BECX+/Fikt9ljLKT62VuxEEIIOnvMwFIZbYpucfH2LZPe41Vlfjwib7m0Ft1oEZ3wCDvzjoRoSA9a3fvDrkoXSV1TS4aS5U2HpkJVeENtHNN2ScaMAdMsR8JYFKKuqSycXYahqQPcnIOhqQqJiIHt+BXvtuOSzjnMqYnz510d9GQtkrEQIcPvOryrsRtNVelM5/npw2/ykXOO4eTF1b7FVnT/KYpUJjOQUcdMFEWhqqqKxYsXA7Bnzx4WLlw4KcJJJp/irjgS1gtFZP6Cl7dcQqHegHURTVPwvGKcxGZefTLYnSqKwpyaBLUVHuGQVmhdrtOdsYjm/YLFvlaCoWtofaK9xULEvOlSHgvR1WNSFgvR3JGlO2MxqyoGCiMOvhcpXjPcr416daFxYXfGCir+o2GDcEgfdXxiKKJhnVShOM+y3QEpr7qmBNcGf+Z7a2eOZMwgGQvRk7Wprxy+O/JEoSjKgEy24UjGDZra/eysrOkrlVmFbLl9LWlOXFgFFCdQ6uTyLrsa/d5v7+zv5uTF1URDGkYhtuW5MjV4JjKmmElfvvCFL/DLX/5ywgSSTC1FyyQa0uhI+QH4YiA+rGsDgreGpiIEZHMulckIlf2qt1VVCYLXmurPIm/qzBaq37WSOeHRiI7okwZcLK7LWy7hkN++vaUzy8GOLD1ZK6iu1kdpmRQtmZBe2oa9pqKoTEw0pZCeHPazzZwJUiaxiE5LVw7H9ee5//y3bxMNa1zygcXEowaaqgZFe15h5vvBjiwNhQFgAhG0sZmuxCOGL3vOobY8iuv5DRzjEZ3XdncEygT8jUAyrrLzgK9MdjWmEEIQDmkIIfi/j7xJdXmEv6lferhuRzJGxt2uU1aqzmyKBYLRsO5n0Xi+VQK9Uwn7oqoKjuuhqn49waEyhRTFb3aYM/0ZKOHC7rNIImqUpB4nIsWKcQel4Oo42J7B0PwMn7JYaEDcYyQUlU+kn/uqrtBGvydr01Wo0o5FQ8NOJBwN8bCOWXBztXblaenK8W5zmh8/tJ2m9myhf5XfH8y2PXKmQ0fKZE5NHNfz0FV11PUlU03R4tN1lfqqKImojvBg2aIq3t7XXWJ5gf/+7m9NU5kMk+1TU9PcmaOx3bdC5VCtmce4P6WjSTuUTD/sQmZWJOy7pVzXCwrpBps3rqkKmqYyf1ZyRO6mspgRzIyPhkpbktRUREuqq4uZVMXFJxEtuHoKmWDxqDHiSYN9KSqH/o0SkzE/26wn66cu+2nLI2sgOVLiUb92xrJdGtv8gXLnntaAEIKfPfJmoX5G8bO4bDdYWOfUxP2U4ERo2n/HVEWhpjzC/LoEmqoSNnRAcPLialxP8PrujpLX7znYgxBw3mkNAIGV8tou/3Wm7Q7an00yvZneWx7JpGMHAfhCe3nHI2/6CqZ/wBr8lNmFs5Mj7o1VVgiaO64XFKkViUWMkkr2oqLI5H1loqoKqqqQyvhWQzyqjzotGHrdXP17TBmFWpNUxqInYxfcbGLUbrRD0VdBthWq+E9ZXM3lHzoO2/HY0+QvrI7rkS7ES8DPhrIdL+g6MN2ZXR0PEijCIRUUhVlVMWorIry6q73ktTsbU4R0laXzK6ivjAaurqLSsWwvGFsgmTlIZXKU4xSaKhYLES3LDbKLwqGBlomuqaMK0Jb1qRUp7vyHQlNV5tbGeeWd9sBiAj9ADpAIG6NOC4belOBQSB1wvDzu15qkc35djMLEWiaxPt2J27tzREIayZhBbUUEQ1dp6cyhKv5uPJ2zae7MUVMeIRLyW/mPtG3MdMK3TjRc17dO9rdkgg4GQgh2Huhm4ewyNE1lUUMZe1vS7GxM+dZrsT+btExmHFKZHOXYrovWp44hb7nk+lgm43Wx9A3QxwrV74fivNPmks7ZPPtGbx1Td9ryg/kRfUxWg6Fphfnvpedqql+42J22AmUyEcWKfSm2EMmYDu0pk9qKKIqioCgKtRVRWjpzaJpCNueQtxwOtmeZU6gvUZTR9+OaLiSiBpbj8Z5F/oiCV3f61klnj0lX2mLxnDIAFs0pw/MEjz67F11TOemYSkzLnahkOskUMu5PqkwLntnYTrEozv8omLYXWCb9A9ZjIdnHMomF9WHrJY6Zk+TYhnL+8OcmsnmHg+0ZXtnZHizCYwmOV5eHKU+EBlhFiqJQXRbB9fz2LYmYMeGLd9EdaJou7al80GASoL4ySnNnDk1VyJg2qYxNOmfTUBvHcjzikdG1QplOxKMGjutRFg9xzOwkf3q7jR37unjnQDcAixt8ZTK/LomuKbSnTI6fV05ZLITrCcxhxjFLph8jckD/7Gc/G/T4Jz7xCb797W9PqECSqcVxPbQ+6bym3TtmdyJSUiv6urkiA4sg+6OrKuecMot/e+QtHnhqN3ub00RCGhevXOT/fgyWyYf/8hgWzSkLeoD1pba8d3GPhY0JdysVOwdnLYe85ZYok7rKKH96u62QPafQXJgv31ATx7a9oA5mJhI2evugrTx1Dr/43S7+47F3AKhMhqkq3Juhq8yvT7KrMcWyRdVBD7dsXiqTmcaIlMmOHTuCf1uWxYsvvsiZZ545aUJJpg7H9dA0hUghOG3afp8qGBiwHguxiB8ncT0RTC48FLqmUl0W4ZTF1bz8Tjs15RE2XnAciYiO7YoxV3Ubuoo+iIttVqHDrS+rNuHKpOjm2tvkZ3LVlkfImw6RsB4oltauHBWJMM2dWXRNob4qSjbvjmoOyXSj6C4UQjC/PsnfXfIe3trbxUs72lgy168XclwPzxOcsrianOmwuKGM7Xs6AX9+jWRmMaJP62233Vbyc0dHB3//938/KQJJphbH8S2TaOHLb9kuectFASLh8bt8DF0jGtZJ5+ySYPxQaJqCJ+CDfzGXRCzEmSfWES+0po+McaFXFFAVdVCrqL6qV5nEI8aEZnKBnxoNsLelB4CKZJis5ffnqi8ok5bOHHPrEjR3ZJlVHUNTVYRwZmTwvYiqKMQK6eaGrqGpKicurCopYCxaH+9ZXM17CuOfixZy3nQHzEmRTG/G9M2pqqriwIEDY77o448/zoYNG1i9enUwsfHpp59m/fr1XHDBBSWus+3bt3PxxRezatUqbrzxRpxCxXRjYyMbN25k9erVXH311WQymUGvJTk0tivQVDWwTCzbI5t3MAwVQ5uYxSwW0VGU3l36odA1FeH5PvfzTmsIYg6HmkU+HIripxgrgyiTWFgPGkwmosag1st4CBl6oWWKSyJqYGgqFfEQ2bxDLGKQiBq0dObQNYWD7VkaahI4rkfI0CY0q+xwkIiGsJyhs7KUQmsct0/mVlGBmrYr+3PNMEb0af3Zz34W/PfTn/6UL37xi1RXV4/pgvv27WPTpk3cdddd/OpXv+KNN97gd7/7HV/96le566672Lp1K6+99hq/+93vALjuuuu46aabePTRRxFCsHnzZgBuueUWLr/8crZt28ayZcu46667xiTP0U4QMzGKysQtDMZSB40xjIV4RCdiaCMa0qSqSjDtsS+ey5jSgsHfJavK4I0PDV0NLKZYxECfoHvuSzFTrq4yiif8BpO+68+jrhCEb+7wW67MrY1jO15g0cxk4lFjyHoRy3aJhXWqkuGg4wL4Cgb82J3UJTOLESmTHTt2BP+9/fbbzJ49mzvuuGNMF/zNb37DhRdeyKxZszAMg29/+9tEo1EWLFjAvHnz0HWd9evXs23bNg4cOEA+n+fUU08FYMOGDWzbtg3btnn++edZtWpVyXHJ6LEdP2YSLmRzWY7f0sPQtQnbpc+qjFGRDA+YjTIYaiFtdgDK2ILvRbQ+c1P6H69MhElEdbQxNJEcCcVizGKMJBLSqK+Mkc35AfnWrhz7W/2YSkNtHMfxiM+QYsVDEYvo/rCrQawT0/KoSEZIxkMlacBFV6Rle7KlygxjVDGT7u5uNE0jkUiM+YLvvvsuhmHwqU99itbWVs4991yWLFlCbW1t8Jq6ujqam5tpaWkpOV5bW0tzczOdnZ0kEgl0XS85Phqqq8d+D/41k+M6fzIYi0xKwSqZXeenamqGhiME0bBOXV2S6j7ZTmPlkx9ZxmvvdFBfW0ZtnxjFYDLHcjYdGZvyPvUpQggU3WL2rPIxxxHaszZ1dclBXWUXrVxMV49FdVWC+vqyQeUaD8l4mNauPAvnVFBeFmXOrHI8ATm3jYVzynnm9Wbe2NNFMmawsKGSnqxFw+zyAbJOx88cHFouLWTwblOK8ni/hqC6ycK5lYQMjc6cQzxs4Hgecwu92oSqUlWdmLQkhOn4LKejTKNhRO/Url27uO6663jzzTcRQnDaaadx++23M2fOnFFf0HVdXnjhBe655x5isRh/+7d/SzQ6cMFSFGXQJpKHOj4a2tvTY9751NYmaW3tGdO5k8VYZcrlHb+dfNqvUO7pydOTsVAV6OzM4lnjz6rJZC1M0yLVnUXpsw0dTGbbcenpyZFO54hFDFzPoyfrUFcRpbszM+aAbDqVo8NQBy1KjOoqpgG5bJ7W1p4Jf3/DhTkdUR2yWZOODj++Z+AFX8C9zT0cN6+Czq4M6ZxDqjtLus+9TsfPHAwvl+t59KRy2KYVFI06jofrQU/Kbx2D7dKUyuE4HrML3ZK7U3laWnqCDgJTKfPhYDrK1B9VVQ65CR+RTX/DDTdw6aWX8vLLL/Pyyy8HwfCxUFNTw1lnnUVVVRWRSIQPfvCD/OEPf6CtrS14TUtLC3V1ddTX15ccb21tpa6ujqqqKtLpNG5hYSoel4yeYsxEU/2CQNv1gkmJY5jHNCi6qqLrKtoIXEiGrnHs3ArikRBdaYtMzmF+XWLYDsXDkYiGhnRhRcMapu0RGkPfr5FQ7FlVngwR77PTTsZDVCTDQZygoRAvSUSMCZn0OB3QVJXayijZXO8mwrQ8Kst63XjliRCW7YGiUJEIEzY0GYCfgYxImeRyOS677DIMwyAUCvGxj32sZJEfDeeeey5PPfUUqVQK13X5/e9/z+rVq9m9ezfvvvsuruvy0EMPsWLFChoaGgiHw7z44osAbNmyhRUrVmAYBsuXL2fr1q0lxyWjp6hMFBQMQw0mDxr6wPbzY0XXFELayFvHhw2N+fUJFs1OcmxDRVDgNh6qyyND3k+o0EcqHJqc7KmTF1Vz8uJqVEUtcduEDQ1dU4LixIYav/I9eQQE3/tSmYigKL7V6XkCV3gk+8SE/Gw//znomj+W2bSkMplpjGgrNm/ePF566SVOO+00wA/Iz507d0wXPOWUU/j0pz/N5Zdfjm3bvP/97+ejH/0oixYt4vOf/zymabJy5UpWr14NwB133MHXvvY1MpkMJ554IldeeSUAmzZt4vrrr+f73/8+s2fP5lvf+taY5DnacRwPLab4faB0FavQCr3/IKnxUCwaHK76vS+KolDWz88+WRiaiqEp/ijhSeDUJTVUJMIIr7eOAvznEjZ0aioitHXnA2USnQTXzuHE0FXm1MTp7MmTs1wihl6SmaepKrOqokFWnW8puggZgJ9RjOhT29zczMc+9jGOP/54dF1n+/bt1NTUBLPhf/WrX43qopdccgmXXHJJybGzzjqLBx98cMBrly5dyn333TfgeENDA/fcc8+orisZSLHOxG8qqOEUlEnYGP0QqqFQVYWQMXGpxhONrvmxlJG44cZCoEQVSoaBASRjOu89toaGmjjhkFZ49jO3WHEoqsoigYXpDVKMWBxUBn4qtWW7uFKZzChGpEw+9rGPcffdd3P99dezc+dOdu7cyZe//OWSTCvJzKTYTkVRFEK6Ss708/tDg8wyGQ9zahLTtghP15TA5TQZqKoCCiBEyXAwgHg0xKyqGEvmVRRmvujT9jlNFMNtUoodE1zZhn5GMSJl8stf/pKPfvSjnHHGGZxyyimYpsmWLVv40Y9+NNnySSaZ3piJ78Nv6/azuiai/XxfpvNuW1EUKpKhAQv9RNHbm2xg1+SwoQaVesXJikc7sYg/00RaJjOLEX17Ojs7g1hFOBzm4x//OK2trZMqmGRqcF3hKxPF7xxcbLDnT1mcnm6pyaC2IjbsrJWxoqoKKgxaM9G3IaLjzpzJipNJNKzLaYszkBF9e1zXLSkKbGtrG7TWQzKzKC5gejFmomvYhWrlcGjiYiZHO6ri9wUbrGZCVRTiEQPb8VCU6W3BTRWxsI7luNhyQtaMYkRuro9//ON85CMf4ZxzzkFRFJ5++mnZNfgIwPX8Llh+zKTgcikQDh1ZGUWHE0VRiISGbm+fiOk0d1j+aOEZOllxIomFdYSAvCljJjOJEa0Yl1xyCcuWLeOZZ55B0zQ+9alPcdxxx022bJJJxnH9L2tfN1eR8CTFD45WomG9RFn3JRY2MG2P2vKIbLkOgQWXyduHWRLJaBjx9nPp0qUsXbp0MmWRTDFOwSetaWoQgC8yGW0sjmbm1g7dhiJsqOiqQiIm4yXQ+9nLTUArH8nUIbefRzFFy0TX1AGWyUhmj0gmBkPXSERDRKVrESCYL5OTo3tnFFKZHMUUW4MX6yv6WibFoVSSqWFOTZxIWAbfofezl7Ncmegzg5DK5CjGLsZMCkVyxSFOqsKk9amSDE4sosvsuQLFxpiyP9fMQq4YRzHFmIleqCcp9ksydG3CRvZKJKOlWI+TtxxkEfzMQSqTo5ggm6ufmys0wnbxEslkEAuUiXRzzSTkinEUYzu9AXggqHEwjIlrPy+RjJZQocmonGkys5DK5CgmyOYqtBEpWiaGrk7YYCyJZLQUizxN25NurhmEVCZHMb2pwb7mCAVurqOrL5dk+hEJaVjSMplRSGVyFGM7/he12C03iJlM4CwTiWQsRMO6dHPNMKQyOYopWib9lUl4AqcsSiRjISKnLc44pDI5iikqk+L0v2IAPmSoEzoYSyIZLbGwjm17cqbJDOKwLhnf/OY3uf766wHYvn07F198MatWreLGG2/EcfxWCo2NjWzcuJHVq1dz9dVXk8lkAEilUlx11VWsWbOGjRs3yvkqY8AOlEmxaLFgmUzwYCyJZLREwzqW4+HICPyM4bApkz/+8Y/88pe/DH6+7rrruOmmm3j00UcRQrB582YAbrnlFi6//HK2bdvGsmXLuOuuuwD4zne+w/Lly3nkkUe49NJLufXWWw/Lfcxkiu1UjGJKsK5xyQcWceIx1TIALzmsxML+HPiuHutwiyIZIYdFmXR1dfHtb3+bz33ucwAcOHCAfD7PqaeeCsCGDRvYtm0btm3z/PPPs2rVqpLjAE888QTr168HYN26dTz55JPYtmxZPRqKFfBFNxfA2ctmU5kIyZiJ5LBStExypkPOlA0fZwKHRZncfPPNfOELX6CsrAyAlpYWamtrg9/X1tbS3NxMZ2cniUQCXddLjvc/R9d1EokEHR0dU3wnM5vAzdWnD5eqKJM2C10iGSnFNvSu59HZYx5maSQjYcp7Xv/Xf/0Xs2fP5qyzzuL+++8HGLRlgqIoQx4fCnUUUePq6qHnS4yE2trkuM6fDEYrU6jQtmJWXRm1VXH/WNTEUdQpu7/p+Bxheso1HWWCyZFrVmH+S3lZDFdVqKqKT2iLn+n4LKejTKNhypXJ1q1baW1t5cMf/jDd3d1ks1kURaGtrS14TWtrK3V1dVRVVZFOp3FdF03TguMAdXV1tLW1MWvWLBzHIZ1OU1FRMWI52tvTeGPMFKmtTdLa2jOmcyeLsciU6smjqQrdXTnUgpWSyduke/K0tk7+R2M6PkeYnnJNR5lg8uTybH/+e1tHmljYYM++TsriEzM8bKJkdj0PzxMY+vibok7X97cvqqocchM+5f6Mn/3sZzz00EM88MAD/N3f/R3nnXcet912G+FwmBdffBGALVu2sGLFCgzDYPny5WzdurXkOMDKlSvZsmUL4Cuo5cuXYxhyBsdocByBpiklrVOkm0syHSi6ufK2Syik0tqVO8wSDSSTc2juyB5uMaYN02bVuOOOO7jttttYs2YNuVyOK6+8EoBNmzaxefNmLrzwQl544QWuvfZaAK655hpefvll1q5dy7333svNN998GKWfmdiui6aqKH20SSSkUVsRPYxSSSS9M01yeYewoZExHcyCtTJdyJo2qawtOxsXOKxzQjds2MCGDRsAf8b8fffdN+A1DQ0N3HPPPQOOV1RUcPfdd0+6jEcyjiPQVIW+UShFUYJeXRLJ4aIyGQbgQFuGpQsqQQhMyy2ZBnq4yeZdLNvFdrygr93RzLSxTCRTj+166JoiCxQl046KRJglc8v50442HMf/nGZy0yf1XwhBzrSDVvl9yR6ls+ulMjmKcVzPd3NJXSKZhpx5Yj1Z0+G13R0YukbGnD7KxHY8POEPlutbB2PZLvtbx57cM5ORyuQoxnY8NE2h1NElkUwPjptbTk15hOe2N6OpkDPdabNIW44HQmDoKj3ZXiWXztmk8zaWM73iO1OBVCZHMb5lokjLRDIt0XWV046roakjx74WvyffdAnCm7aLUsh8zJpO0Cq/s8dEeMJXNkcZUpkcxTiOVCaS6YuuqRw/v5JoWOPZ7c0gBNYUKpPOnjxNHZlBs7WyORtd7403mpYfiM/kbcIhDcuaHkpvKpHK5CjGDmImUptIph+6pmJoKu9dUstbe7twPH+xHg1jSdv1hOBge4a9zWlaOnIcaEsPGNKVMR2MYkV+QcllC7ETQ1OnVXxnqpDK5CjGcUUhZiKRTD9URcETgmMbyhACmtpzpHMjz5RKZSzebe4ZlULxhGBvUw+tXXnK4gbJuEFHymRfc29Q3fU8bMfjQFsGx/XQdZVM3qarJ0/IUNF15ajM6JLK5CimN2Yi1Ylk+hENaygKzKmJo6oK+9vSmLaLO8IZJ509edq787Sn8iO+Zjpnk8palMUNFMX/bpTFQ3SlTdpSfhW+ZXt0pvL82yNv8eJbrRi6SirrFzCGDQ1NVXFcMWI5jxSkMjmKsR0PXZOpwZLpiaFrVCUjOK5HQ02cvU3pgktp+EXacT1SWZuKZIjGtsyI2tgLIWjuyAZD4vqSjBk0t+cwbRfL8djbkgZg98EUuqZi2x6I0ka0I5HzSEIqk6MYmc0lme5UlUVwXMGCWQkOtmexHI+8NbxiyJoOCNBUlXBIY29zOhhTPRSZvD87ZbBqdlVV0DRo6siSy9scaPOzy95t8t1fnhAYRu8XSQgRjHg4WpDK5CgmiJlIbSKZpkTDOsloiNlVMTwhaOnMjSge0Z020XX/cx02NBzX450D3Ye0UFo6s4ds1xIN63T1mHSlTfa3pImENEzbpakjS3kiRCSk8/b+bnY1ptBUZURK70hCKpOjGMf10FUZgJdMb2oro9RWRFEUaGzPkMpahyxe9DxBY1uGnzy8nR37ugCIR3UUBd7Z30VrZ25AUD6bt0nnHMKDuLiKKIpCNKyxryVDJu/wvpPqAdjT5LeOdxyPLb/fxaPP7UXXVbKjSBY4EpDK5CjGcT00TaYGS6Y38YhOWTxMfWWMfS1pHMfzW5YMkaWVNR227+mkI2Xy8B/fDQodw4ZGLKqz+2A3B9rSQYA8m7fZ25ImpA//PQgZGl1pf/LjsmOqqC6LBMrkzb2d5EyX1q48lu2Sk5aJ5GhACIHr+l2DJZLpjKIoVJdHaKiJc6A147ub0hZN7Vk8IcjmbZo6MnT15HFcj+50njfe7aQ8HqIna/PEnxqDv6WpKpWJMF09FrsaUzS2ZXjnQDeqApHC5FEhBO829fDfLx0YdF7J3uYekjGDymSYhbOT7G3uwfMEL+1oC75PjW0ZHEcMG6c5kjisLeglhw/XEwiQ7eYlM4J4RGdObYwXdwgOtGeYX5egrStHZ9rEcz1UVcH1BAoKBzsytHXnufB982nuzPHc9mZOXlzN7OoY4CunRMwgbzp0pkySMSOwzl95p40nXzkYzJ3/4+vNXPT+BSxbVA34imZPUw+L5pShKAoLZyV58a1WXt/TwZ6mHs45ZTZP/7mJd5vTzKmOBxmTRwNHx11KBlDcMWmq/AhIpj9hQ2NBvT8jfW9TD4qikIwbREMayXiIeNSgLB4iEdPZeaAbTVU46ZgqzjutgVhY56Gn95DtVz0fCevEY3qgSPYcTPHgH/YQDWt85Jxj+F8bljG7Osb9T+7m0ef24rge7d15MnmHBbN8WYr/f/TZfSgK/MVxtTTUxtnbnEbgp98fLciV5CjFcX1/s3aU7JokMxtFUZhTE6euMsqf3m4jbzkoioLaz03rCcHrezo5bl4F0bBONKyz9qwFNHfm+P6W13lrb+egfz+Tt7n/yd1UJSNcuep4Tl5cTVVZhCtXH8cZJ9Tx7Bst/OSh7bzwVisACwtKJBE1qK2IkDUdlswtpyweYn59ouCCG337l5mMXEmOUoo7JhkzkcwUkrEQH3hvA90Zi4f/uHfQNinv7E+RzTuccmx1cGzpgko+ve4EElGD/3x8Jz976HXa+syUF0LwwO93kzMdLv7AopI6E01VWX3mfC774LGkczbPbW8J4iWeJ7AdL1As7z2uFoD5dUk8IWjvNmnpzAUB+yMdGTM5Sim6uWTMRDJTiIZ0GmrirDx1Dk/8qZFFc8o4aWElb+/vZl9LmmzeYX9rmnhEZ3FDGdm8g+N4lCVCzKqK8el1J/CH15p4+rUm/ryzjZOOqUIIP1je2WOy5n3zmVUVG/Tax82r4OqPnMRjLx4opCkrpHM2nif4i+PrUBSFJQ3lAMyrS6AosK8lzcJZfoBe11QSUWMqH9eUc1iUyb/8y7/wyCOPALBy5Ur+/u//nqeffprbbrsN0zRZs2YNX/jCFwDYvn07X/va10in0yxfvpxbbrkFXddpbGzkuuuuo729nWOOOYY77riDeDx+OG5nRiJjJpKZhqoqVCRC/MVxtew52MPWP77LI8+8i+MKQrq/WMejBqcdV4uqKDiuR1k8RDprk4gZaJrKilPmcN7pC3j4qV28+FYr8YjO7JoYZ51Uz18cX3vI68ciBuvfvxDwrRkhBIahkojqrD5zfvC6cEijvjLG3pY0mqYSC+vsPphiydxyIqEjd/8+5Xf29NNP89RTT/HLX/4SRVH49Kc/zUMPPcQdd9zBPffcw+zZs/nsZz/L7373O1auXMl1113H17/+dU499VS++tWvsnnzZi6//HJuueUWLr/8ctauXcu//uu/ctddd3HddddN9e3MWIpuLn0EufUSyXShPB6ms8fir1Ycwy9/v5va8ignLKxkfl2iJH6SzTtUJMLMqYmz80AK03KDgsRkLMSqM+Zxwelzx1xjlbdcyuNhIiGN1u4ciWjppmx+fYI/vd2G6/ldhXXXo7EtwzGzy47Yuq4p35bW1tZy/fXXEwqFMAyDxYsXs2fPHhYsWMC8efPQdZ3169ezbds2Dhw4QD6f59RTTwVgw4YNbNu2Ddu2ef7551m1alXJccnwpHN+Tn6x1YO0TCQziVhER1UgHjG4ctXxrHnffBbOSg4IxDuuR21FFF1TmV+fwHLcATUf41nUbUdQXR4hGQuVVON7nqC7x2JeXQLb8TjY7tepRMM6qaxNT+7IDchP+UqyZMmSQDns2bOHrVu3oigKtbW9JmZdXR3Nzc20tLSUHK+traW5uZnOzk4SiQS6rpcclwyNEIK27hw7G7tp68qzqzEFgCFjJpIZhK6p1FVFSR9iUc6bDmXxENFCEWI0rLOgPkk274wqVbfoyuqP63nomkIsohMNa+iaGlTTZ/IOkbBGfVUUTVXY9uw+8oV+YPGIRmNretLn2Hek8mMaCjZeDpsD7+233+azn/0sX/nKV9B1nd27d5f8XlGUQR/IoY6PhurqxOgE7kdtbXJc508GQ8lkOy4HWtL0WB7zZlegqQppy//w11TFD+u9TMfnCNNTrukoE0y9XFXVCYTajoIyoJeW6wnImJxwTHVJwLu2NkldXRk79nZhWi6VFYPHV/OW46cdo6CoCsITqKpCPGqgFtaYVNZiQUOM+hp/DbFQaevMEgnphMIGC+eU8/a+Tv5m7Yn834ff4N7H3uFzG06mMmLQnTERhk7tIIH+iXiOOdPhYLdJZVUCQ59aW+GwKJMXX3yRv/u7v+OrX/0qa9eu5bnnnqOtrS34fUtLC3V1ddTX15ccb21tpa6ujqqqKtLpNK7romlacHw0tLePfYdQW5uktbVnwHEhBHnLRVGYtEBbJm/T2pljdnW85Is0mEyeJ+hMmzS1Z0FALKqRSvl9ilI9fmqka7uD3stUMNRzPNxMR7mmo0xw+ORKhFR27u8mGTdwPf9753kCXVOpKguTS+fJpQcOxapNGKRMl72NXYAA/EanmqZgOx6xsM6s6jhhQ0MvHGvrzrOvscvfsAoQCOoSoeC+HdOmvTMLQjCvPoGZNTFzNvXlIf7HeYvZ/PhO/mXzn/jEhUvRVJXtb7fgzKso6VBcfI5CCNI5m2QsNKbn0taV42BLD5VRbcLXIFVVDrkJn3I318GDB/mf//N/cscdd7B27VoATjnlFHbv3s27776L67o89NBDrFixgoaGBsLhMC+++CIAW7ZsYcWKFRiGwfLly9m6dWvJ8cOF6/mVsTv2dfPOfv+/yWg/7XmC/S0ZenIWO/Z30ZHKD9nszvMEe5t7ONCaIRJWSyp9fZn98wwZgJfMQOIRg5qKKKmMheMKasujHNtQwdIFlcyqGjqrMxrWWba4hpMWVnH8vEoWzSljTk2csliIBfVJFjeUk4gaGLrfADVkaMypiXP8/EqOmV3G/FlJFs4uK9nIRcM6qgrRiE55IoyiKNRVRMhbHkvmVvA/zltMU0eOx188gKr6hZaNbelBPSyt3Tn2NPWMuXK+PWWioARFyVPJlFsmP/nJTzBNk2984xvBscsuu4xvfOMbfP7zn8c0TVauXMnq1asBuOOOO/ja175GJpPhxBNP5MorrwRg06ZNXH/99Xz/+99n9uzZfOtb35rqW0EIQU/W4kBrBtv1dzXJuIFpuexp6mHxnPIJNTU7evKYtktZPITr+Z1TmzqyVJWFiScjJXIVW3WXxf0dju24PP9mK398rQnL8YKBWIY+dMttiWQ6U18Vpboscsi28UOhqgohVSNkaMQjw9d/hA1tyFknqqJQXxklHul1hSXjIZTWDJ4nWDK3gtOX1vHs9haWLqhkwawk3RmLzrRJVZ/vbSpjcrAtCwpYjjvqtSNvOZiOg66ph6XBpCIOR6RmGjBWN1feckiZLu0dGTzhZ43EwvqANz5bmI1wzOyyAZkmY8F2XN7a20U0opVkYLmeR950SSSj2HmLuqoYOdOmqSNHWczAE34309+/cpB0zmbRnDLqKqP+mFEEl33wOKrLI0NfeBKRrpuRMx1lgukr16GYKpmbOjK0d5nEYzqW7fKDB98A4LMXnYimKuRMl8UN5WiqQllFjBf+3EgkrJE3XWZVx6gpj47qem1dOe773U4A/uqcRdQPUYA5VoZzcx25FTSThOsJcqYT7FJUdfBHGIvqpDIWzZ1ZZlePvZhSFGZet3T5KYZv7OmkrjJKfaX/QdFUlXhUpSIRpjlnsudgCgEkozo79nXz2xf2057KM78+wcUfWBQ0ywPoydiyAl4imSSqyyJ0pMygxuWi9y/k/257i988v5+1Zy9A0zze2d8FCpR151FU+OWTu4lHdNaevWDUyuTd5h5eeLOFRMzAKsxwmUqkMhkDgzWYA3/h7077riVVVUjGDFo6s8Fwn6EQQvgt4YX/b8cV5C2HdM4mnbNxPA8EPPN6M8+84adAHz+vgr88eRZzauJBLCRk+Ga7EIJfP7+fZ99oproswl9/8FiOm1s+MONNATkAXiKZHAxdY9GcMnYeSKHYLgtmJTlrWT1/fK2Z2sooZ5xQB4VlIRkN8fNH32THvm7ChsY5p8xBCDHiLFXTcnnhrRY8AamMTfowNJiUymQC2NWY4s+72tl5IEU6Z1NbEeFDy+dx7Nxy4hGDfS0Zjpunl8QnHNcLmsC5nigmllBIF0FRFHRdIRLSUFWdJ/50gGfeaGb58bXEowbPvtHMWw93UVsRYdkxVZx1cgO64iuj37zgK5LTl9ZxwRlzhyxM9DwhdYlEMolEQnpBoXSjKAofPG0u7d0mjz63l8pkiCVzK3Bcj3t//RY79nWzuMFXPi2dWex5FSVNJw9FR0+eP+/qIKSrWI5Hc0eWExZUDfpa03YPOet+rEhlMg46e0x+/dw+3trXRSSkBZkhL73Vyr2/fZtj55Zz8YpFqArsbU5TVxklEtKxHY+9LT04judX9A4RU3Fdjz1NPby+u4M/vd3Ge5fUsOZ981EUhfedWM+fd7Xz2u4O/vtPjfz3nxqpLotQVRbm7f3dnL60jtVnzhuws3FcP8biCUEiahCZhA+VRCLpJRrWWTi7jF0HuonHdDasOIZ/e+QtfvHELiqTYVq7/KzMD/5FA8sWVfPd/3qV/a0ZLMcbkTLJ5m2efKWRbN7hvNMaePylA3SkTBx34GAuzxMcaE2zcHZZkCwwUUhlMgqEEPzu5QNs39tNV0+epvYsqqpw3mkNvO+k+uCNO/OEOp7b3sJjLx7gPx9/h8vPX4Jp+xleCN+dFQlraJpCW3eedM7GtF1c18N2PFq78zR1ZGlsy2DZHpqqsHxpLavPmB8oh3BIY/nSOpYvraM7bbK3LccrO1rY09TDmSfUccEZvYpECD/O47gCQ1epr4pRFguNKQtGIpGMnkTUYG5dgr0tacrjBh89/1geeGoPigLHzi3npMW1zKrwfV4ViRAH2jJkTXvYTsNWYV15dWc71WURTj+hjsdfOkBn2sR1Bf2TNU3bxbInJ9NLKpNR8szrzWRMh4p4iL84vpazl80K0m+LaJrKWctmkYga/PL3u/nFk7u49AOLUVW/en9nY4qnX2tiz8HBM0o0VaG+Ksp7FlVzbEM5x8xOBjsU2/EVTjSsBcqiPBFmxdwq3rOwosTP6isRvydRVVmYqmSEaFg/YhvNSSTTmaqyCHnLpbUrR3kixBUXHBf8rrIiTmdXBiEEc+sSvss8a1NXMfTfcz2Pvc1p9rX0cLA9y5oz56MpCvGITmePieN5hCnVJnnLwZ6ktGGpTEaBoih8ZeNptGdsPHv4osT3LK4mazo8+tw+7tryGoamkrdcujMWyZjBylPnUFMeIRE1CIc0dFVB11SScWPQOEcu7+AByahBZ9okEhqY+15UFJ4nSGVtKpNh6gvuNYlEcniZVR3DcT06e0zK4kZpIbHr0ZNzmF0V57VdHexrSQ/ZZbhYwLynKcUDT+2hIhHi5GOryZhOIYssjzNI4WM65zBZxSByhZlgPE8EBUMCOOOEOlRV4Z39fgCuSlVYeeoc3rOoasQjcy3bJW+6JOMGc2sTGLpGdXmExrYMPRkLFAVFN8mbfm2LJwTprM3cmgQ1FaNLL5RIJJOHqijMrfP7ZjV35ohHdLxC8XPOdFlYnyCsq/zmhX3sa0lj2d4Ad7QQgsa2NDv2dfLL3+8mEtK4cvXxeJ6gLBaivirGn3e1YzoD04PTOWvS7k0qkwnAK9SeeJ5AURXiEQMFX5mkMhbvXVLD6UsH7x3muB6W7eJ6QNFFpYDw8P8vBPGIwfz6KGWJcBA0i0cMjm0ox/UErisoq4jx1q5WUhkLAcyrS1JVdniKESUSydCoisLs6jghQ6O9O09Y16ivilGb8OOYIUMjHtU50JbBdHrnsAghMG2XjlSel3a08cize4mGfUVSFguRydnMmR1nTk2c599soTNlUlfRW7hoOx6O402am1sqkzEihL+I5ywXBagpj1AWDxMJayVZEqmMyd7mNLrmD8kRwjdnrYIJGjZ0qpIRYlEDUaioF55A11VUVSFSqB0ZDEVR0DUFXYOyeIiFs8rIW75Si42gRYREIjl8VJdFqC5s+GprE0FVfjSsM682wf7WDM0dWTp78ggPsqaD7br86a02fv/ng9RVRLnsg8dSFg+RytjMro4RDmnMqfEVSHNHluPnVwbXM22X599sAUXhpGMGTxseD1KZjBKl0GU0k3MIGRqzq2JUJMMDUvCKlMXDHDtX40BrBtcTaCpEogaz42FiYW3Ce2PJ2IhEMrNRFIXj5lXw5t4u2rpzVCUj5G2HXY0pXtvVwe6DPZywoJIP/+VCDF2lJ+tQlQwHFfMNhdb4TZ25kr/blTb9/mDzKyZFbrnyjJJYRGfunHI6OjIjPicS0lncUD6JUkkkkiOJZYuqePAPe/jJw2+iqf48eyGgPB7iQ8vn8r6T6gFIZW1qyiLMrokHHpG6yiiqAu3deVzPC5J5Xtjegu14nLhw4q0SkMpkTIw0cC6RSCRj4ZhZZZz73gbyloMAQrrKknkVzKn2XVjFurH6Qp++vnEQXVOpLIvQ0ZPHcX1viCcEL73dSlVZmNnVE9sAMrjupPxViUQikYwZTVP5wHsbSOcsomHdb/jqeKSzDiAoS4SZVRkbsvC4vjJKc0cO1/XA0GhsS7O/NcO5750jA/ASiURyNFGRCNHVk6fHsVAUf3RwbXmUWEQfNjY6qyrGjn1dmI5HDPjDn/0GsScvrp40eaUykUgkkmlIImpwwsIqfzrjKK2J2dVxHFfQ0pHFdT2eeb2JBfUJyhNherKT01FYKhOJRCKZhhRT/8fCnBp/htLvXmkkbzp0ZyzOO60B2/FQVT8jdaKRykQikUiOMIpB9mdeb0ZVFJbMLWfJ3HJypsOiOYPMNpoApDKRSCSSI4yKRJgrPnQcGdPm+HkVaJpCOuuwcFZy2E7EY2VG57j+6le/4sILL+RDH/oQ//7v/364xZFIJJJpw+kn1DG/Lolpe+TyLvMKMZPJYsZaJs3NzXz729/m/vvvJxQKcdlll3HmmWdy7LHHHm7RJBKJ5LATjxgsmVuOrqkYujrpoydmrGXy9NNP8773vY+KigpisRirVq1i27Zth1ssiUQimRaoqkIsYhAytCmZYTRjLZOWlhZqa2uDn+vq6nj11VdHfH51dWJc16+tTY7r/MlgOso0HNNV5uko13SUCaavXIdiOso8HWUaDTNWmYhBJryMRvu2t6fxvLFNiamtTQYdPqcL01Gm4ZiuMk9HuaajTDB95ToU01Hm6ShTf1RVOeQmfMa6uerr62lrawt+bmlpoa5u8JkhEolEIplcZqwyOfvss/njH/9IR0cHuVyOX//616xYseJwiyWRSCRHJTPWzVVfX88XvvAFrrzySmzb5pJLLuHkk08+3GJJJBLJUcmMVSYA69evZ/369YdbDIlEIjnqmdHKZDyo6vhS5cZ7/mQwHWUajukq83SUazrKBNNXrkMxHWWejjL1ZTj5FDFYWpREIpFIJKNgxgbgJRKJRDJ9kMpEIpFIJONGKhOJRCKRjBupTCQSiUQybqQykUgkEsm4kcpEIpFIJONGKhOJRCKRjBupTCQSiUQybqQykUgkEsm4OaLaqaTTaS677DLuvvtu5s6dy/3338+Pf/xjNE3jzDPP5Prrr6e7u5tPfvKTwTk9PT10dnbypz/9iVQqxZe//GX27dtHVVUV3/nOd0oGcBVpbGzkuuuuo729nWOOOYY77riDeDwe/P6+++7jhRde4Bvf+MagMt199920tLRgGAannXYaX/va1/hf/+t/Bec3NzeTSqV44403JkWmQ/Hd734Xx3H47//+b+6++24OHjzIZz7zGVzXRVEU5s6dy4MPPjipz3Hnzp3cdNNNZDIZIpEI//AP/8AJJ5ww4Fn+9Kc/5bvf/S6u61JfX8/999+P4ziBXF1dXaRSKYBJk2vevHlDfubC4TDnnHMOV1xxBZ/85CfJZrPs378fTdNwHIe/+qu/4oYbbhiXTO+88w5f+9rXyGazlJeX841vfIOGhobD/qyGkms6f+6KNDU1cdFFF3H//fdTUVEx4P394Q9/SEtLC5qmsWTJEv7hH/6B6667Lji/ra2Njo4Otm/fPikyzZ07d8jn2P973tjYyNq1a5k/fz4ANTU1/OQnPxny/HEhjhBefvllsW7dOnHSSSeJffv2iZ07d4pzzjlHNDc3CyGE2LRpk/jpT39aco7ruuKKK64QDz74oBBCiFtuuUX84Ac/EEII8ctf/lJcc801g17rqquuEg899JAQQoh/+Zd/EbfffrsQQoh8Pi/+6Z/+SZx66qniK1/5ypAy/c3f/I146KGHxKZNm8QnP/nJkvNvv/12sXTpUnH55ZdPikxDkUqlxA033CCWLVsmzjrrrEDm22+/XZx22mlT+hwvu+wy8fjjjwshhHj66afF+vXrB32Wy5YtE/fee68QQoiLL75YfOxjHyu551NOOUWcffbZkybX+eefP+j7u2vXLnHDDTeIE044QXziE58I/u5PfvIT8f3vf39Cn9UVV1whfve73wkhhLj33nvFF7/4xWnxrAaTazCm0+eu+Dc/+clPilNPPVX8+te/HvT9vf7668UPfvADsWnTJvGFL3whuI7ruuLHP/6xOPHEE8Xq1asnRaZ9+/YNev5Q3/Nt27aJm266adBzJpojxs21efNmNm3aFAzIeuuttzj11FODn88991x++9vflpzzi1/8gmg0GnQefuKJJ4J/r1u3jieffBLbtkvOsW2b559/nlWrVgGwYcOGYPb8888/j+d5wS5lMJlOOeUUXn31VVatWsW5555LKpUqOf/NN99k8eLFzJs3b1JkGorHHnuMhQsXsmjRIlauXBnI/OKLLxIKhbjqqqv43Oc+xymnnDLpz/HSSy8NZtMcf/zxHDx4cMCzfOONN3Bdl0svvRSAjRs38qc//ankns8//3w0TZtUuQb7zL3yyissXLiQVatWsXv37uBv//nPf2bLli28/vrrPPzwwxw8eHDcMv3sZz9jxYoVeJ5HY2MjZWVl0+JZDSbXYEynzx3Aj3/8Y84++2wqKyt5+OGHB31/n332WdavX8+5555LU1NTcJ2dO3fy2GOPcdxxx1FTUzMpMg3FUN/zP//5z+zYsYMNGzZw5ZVX8tZbbw35N8bLEaNMbr31VpYvXx78vHTpUl555RUOHjyI67ps27atZDKj67p8//vf50tf+lJwrO9ceV3XSSQSdHR0lFyns7OTRCKBrvsewtraWpqbmwH4y7/8S/7+7/+eSCQypEwvv/wy0WgURVHYtm0b3d3dwflnnXUWu3fv5sILL5w0mYbiIx/5CFdddRXnn38+c+bMCY7Pnj0bz/P4/ve/zznnnMPtt98+6c9xw4YNaJoGwPe+9z3OP//8Ac9y1qxZCCFobW3FdV2effZZLMsK7vlLX/oSTz31FCeddNKkybV+/fpBP3Nnnnkmn/rUp9i9ezeZTCb4fTweJ5VK8fOf/5yVK1fyhS98Ydwy6bpOKpVixYoV/PznP+d//I//MS2e1WByDcZ0+ty99tprPPvss3ziE58A4Mtf/vKg729zczNVVVVs27aN9vb24DqLFi2iqamJjRs3TppMQzHU9zwcDvORj3yE+++/n0996lP8z//5P4P3fqI5YpRJf4455hi+9KUvcfXVV7Nx40aOP/54DMMIfv/73/+eY445huOPP/6Qf0dVSx+RGMfs+WOOOYarrrqKrq6uEpmK5xdlmjVr1pTJNBzf/va3ufHGG7n66qv51a9+RSaTKbn+ZD1HIQTf/OY3eeWVV/jqV7864LXz5s0jkUgE7+9xxx1Xcv7vf/97ampqKC8vnzK5+n/mqqurA+UDcP7553PCCSdw/PHH89GPfpR33nln0OuNVqaysjKeeuopvvWtb3H11Vfjum7Jaw/XsxpOrkMx1Z+7XC7HP/7jP/K///f/HnBOkeL76zgOV155ZcmaoqpqIFPRqzAVMg3H5z//eS677DIAVq5cSSwWY9euXWP6W8NxRAXg+2KaJieffDJbtmwB4Ne//nXJm/zb3/62xAIAqKuro62tjVmzZuE4Dul0moqKCj784Q8Hr7nvvvtIp9O4roumabS2tg47e765uZnPfOYz6LrOt7/9bcLhMP/+7//OY489Rn19Pfl8fspl6nv+Aw88MOhrhBDceeedXHjhhcFzPOWUU4Jg3mTJ7DgOX/nKV2hubub//b//RzKZDGQuPsv/+q//wrZtfvGLX6BpGv/5n/9JOBwukevkk0/G87xJl6vv+3vnnXcGz+raa68NFmjP87jzzjuDL/ZEybR161bWrFmDoiisWLGCfD5Pd3c3n/jEJw7rszqUXEWm0+fuhRdeoK2tjauvvhrwLYqrrrqKf/mXf6GtrS14fzdt2kRDQwPf/e53efXVV5k7dy7PPfccFRUVUyrTbbfdRktLCwA//OEPqa+vH/RZ3nPPPaxbty5wkQkhAgtoojlilUk2m+Vv/uZvePjhhwmFQtxzzz0lpvbLL7/MZz7zmZJzVq5cyZYtW/jc5z7H1q1bWb58OYZhDPjQL1++nK1bt7J+/Xq2bNky7Oz5+vp6fvSjHxGPx/nIRz7CqaeeyoMPPsj9999PVVVVcH5RphdffHHSZRrqi9wXRVH4zW9+w//3//1/PPbYY2zbto1QKMS6deuC10zGc/zmN79JOp3mpz/9KaFQqETm8847jx/96EfYto3nedx///18+MMf5gc/+AGnnXZaiVwXXXRRyS5ssuTq//4WP3OvvPJK4DpSVZW3336bbDYLwJYtW4IFcjwy/fSnP0XXdS644AKeeeYZKisrqaqqOuzP6lByDcfh+Nydc845PP7448FrzjvvPH74wx8yd+5campqSt7fc845h1/84hc888wzHHvsscF1ijI1NTVNukw/+tGPhn2O4MdS8vk8n/nMZ3juuefwPI9FixaN6NxRMyVh/ink3HPPDTIeNm/eLC688EJxwQUXiO9973slrzv55JNFPp8vOdbZ2Sk++9nPigsvvFD89V//9ZCZE/v37xdXXHGFWLNmjfjkJz8purq6Sn7/i1/8oiSjor9MH/rQh8R73vMeceaZZ5acX5Sp7/mTJdNQfO973xPf+973Apl37Nghzj//fLFs2TLxnve8R9x6660lr5/o59je3i5OOOEE8aEPfUhcdNFFwX+DPcsf/vCH4pRTThEnnXSS+OAHP1hyzyeffLL4j//4j5J7niy5hvrMbdy4seRz9573vEdceuml4sILLxRXXHGFaGxsHPf7+/bbb4vLLrtMXHTRRWLjxo1ix44dh/1ZDSfXYBzuz11/+j67/u/vqlWrxCmnnCLOPPPMkusUZXrmmWfEFVdcMakyDUX/73lTU5P4+Mc/LtauXSs2bNggtm/ffsjzx4OctCiRSCSScXPEBuAlEolEMnVIZSKRSCSScSOViUQikUjGjVQmEolEIhk3UplIJBKJZNxIZSKRjIBPfvKTdHR08JnPfIZ33nlnUq+1b98+Pv/5z0/qNSSSieaILVqUSCaSP/zhDwAjLhYbD42NjSUNIiWSmYCsM5FIhuGGG27g/vvv57jjjuOdd95h8+bNZLNZvvWtb1FXV8fbb79NNBrl85//PPfccw+7d+/mggsuCHp3Pf7443z/+9/Htm0ikQhf+cpXeO9738vOnTu58cYbsSwLIQSXXHIJl112GatXr6a5uZnTTz+dn/zkJ9x999389re/xTRNcrkcX/nKV/jQhz7EnXfeyd69e9m3bx8tLS2cfPLJvP/972fLli3s37+f6667jnXr1nHnnXfy9ttv09bWRnt7O0uXLuXWW28lkUgc5icrOaKYtHJIieQI4rjjjhPt7e3i3HPPFa+++qp45plnxAknnCBef/11IYQQn/rUp8Rf//VfC9M0RXt7uzjppJNEU1OT2L17t1i3bp3o6OgQQgixY8cO8f73v19kMhlxww03BLMuWlpaxLXXXitc1xXPPPOMWLt2rRDCr4z+2Mc+JnK5nBBCiIceekisW7dOCCGCivFUKiVyuZw4/fTTxW233SaEEOI3v/mNuOCCC4LXrVixQrS2tgrXdcUXv/hF8Y1vfGPqHp7kqEC6uSSSMTJ37lxOPPFEAObPn08ymSQUClFVVUU8Hqe7u5vnn3+elpYWPv7xjwfnKYrC3r17+dCHPsRXvvIVXn31Vc466yy+9rWvDegO29DQwDe/+U1+9atf8e677/LKK6+UtLU/++yzg4aTdXV1nHPOOYE8XV1dwetWr14dzNi45JJL+D//5//wla98ZTIei+QoRQbgJZIx0rfZIzBoN1bP8zjrrLN44IEHgv82b97MkiVLOPfcc3n00UdZs2YN27dvZ/369ezdu7fk/Ndff53LLruMdDrN+9//fj796U+PWgagpBW+53ljbmkukQyF/ERJJCOgOLd9tLzvfe/jD3/4Azt37gTgd7/7HRdddBGmafKlL32JrVu3snbtWjZt2kQikeDgwYNomhZM43v++edZtmwZn/jEJzjjjDN47LHHRjUXpMhjjz1GT08PnuexefNmzj333FH/DYnkUEg3l0QyAj70oQ9x+eWXl7iYRsKSJUv4x3/8R774xS8GsyS+//3vE4vF+Nu//VtuvPFG/vM//xNN0zj//PM544wzSKVSaJrGJZdcwt13382vf/1rLrzwQgzD4KyzzqK7u5t0Oj0qOWpqavjMZz5DZ2cnp59+Op/73OdGdb5EMhwym0siOcK588476ezs5Oabbz7cokiOYKSbSyKRSCTjRlomEolEIhk30jKRSCQSybiRykQikUgk40YqE4lEIpGMG6lMJBKJRDJupDKRSCQSybiRykQikUgk4+b/B8dfKnCRHoIcAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"cpu_usage\", data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 128, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 128, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEJCAYAAABR4cpEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAACA/0lEQVR4nO29eZxcdZnv/z5b7b1v6XQWErYAASKGVUlEwYQsgkEUCDC4oYw/LzLKCCpk8MqADiMqXsB1Xne4MhoRw2IIKIioCCYoIBAIZE866b27utazfX9/nKrTXb13eqtOvu/XixfJSZ2up5Y+z3m2z6MIIQQSiUQikYwBdaoNkEgkEsn0RzoTiUQikYwZ6UwkEolEMmakM5FIJBLJmJHORCKRSCRjRjoTiUQikYwZ6UwkEolEMmb0qTZgqujoSOK6hzZiU1UVo60tMc4WjY1itGk4itXmYrSrGG2C4rVrKIrR5mK0qS+qqlBRER30349YZ+K64pCdSf78YqMYbRqOYrW5GO0qRpugeO0aimK0uRhtGg0yzSWRSCSSMSOdiUQikUjGjHQmEolEIhkz0plIJBKJZMxIZyKRSCSSMSOdiUQikUjGjHQmksMOy3an2gSJ5IhDOhPJYUXGtNl9sBvHlQ5FIplMpDORHFbYjqAzkaWjOzvVpkgkRxQT6kwSiQSrVq1i3759Bcd/9rOfcdVVV/l/b2xsZO3atSxfvpzrrruOZDIJQDwe59prr+XCCy9k7dq1tLS0AGCaJjfeeCMXXnghH/7wh9m+fftEvgzJNMJxXHRN5WBbCtuR0YlEMllMmDN55ZVXuPzyy9m1a1fB8XfeeYcf/OAHBcduu+02rrjiCjZt2sTChQu59957AfjOd77D4sWLeeKJJ7j00ku5/fbbAXjggQcIh8M88cQTfOUrX+Gmm26aqJchmWZkbQdDVxBAe3dmqs2RSI4YJsyZrF+/nnXr1lFbW+sfM02TW2+9leuvv94/ZlkWmzdvZtmyZQCsWbOGTZs2AfDss8+yevVqAFatWsVzzz2HZVk8++yzfOhDHwLg9NNPp6Ojg8bGxol6KZJpRNZ00VSFaEinuT0ti/ESySQxYUKP+SiiN//5n//JJZdcwqxZs/xjHR0dxGIxdN0zpaamhqamJgCam5upqanxDNV1YrEY7e3tBcfz5xw8eJCZM2dO1MuRTBMs20XTFFRVQQhBKmNRFgtOtVkSyWHPpKkG//nPf+bAgQPcfPPNvPjii/5xIforZSqKMujPUdWBg6nBjg9GVVVsVI/vS01NyZjOnwiK0abhGG+bD3RlCOoamqai6jqVVTEqS0NTbtd4UIw2QfHaNRTFaHMx2jQaJs2ZPP7447z99ttcdNFFpFIpWltb+cIXvsB//Md/kEgkcBwHTdNoaWnxU2O1tbW0trYyY8YMbNsmkUhQXl5ObW0tLS0tzJ07F6DgnJHS1pY4ZMnnmpoSWlq6D+nciaIYbRqO8bbZFYLWtiQlEQOA7pRJS0DFyVpTatd4UIw2QfHaNRTFaHMx2tQXVVWGvAmftNbgO+64gyeeeIJHHnmEb3zjGyxcuJDvfOc7GIbB4sWL2bhxIwAbNmxgyZIlACxdupQNGzYAsHHjRhYvXoxhGCxdupRHHnkEgC1bthAMBmWKS4LjFN4cKIqCO0DkK5FIxp+imDNZt24d69evZ8WKFWzZsoUvfOELAFx//fW8/PLLrFy5kgcffJBbb70VgKuuugrTNFm5ciW333473/rWt6bQekmx0LcVWAHpTCSSSUIRAxUtjgBkmmvqGW+bE2mLnQfifporlbGoKg1TVxmZUrvGg2K0CYrXrqEoRpuL0aa+FE2aSyKZaBzHhYL7A5nmkkgmC+lMJIcNWdtB1Xr+rigDdwtKJJLxRzoTyWFD1nTRerWVezWTqbNHIjmSkM5EcthgWjaa1mtGSQEhvYlEMilIZyI5bDAtF63X8KqiKLhIZyKRTAbSmUgOC1whsF2BqvZJc0lpLolkUpDORHJY4ORmTLKW47d8ewX4qbRKIjlykM5EclhgOwLhutz769f47Za9uaMKrpChiUQyGUhnIjkscFxB2nToTllsebOF7pQp01wSySQinYnksMB2XLoSJuA5lr+83sQQ4tMSiWSckc5EcliQtRziKc+ZzKyO8NJbLSSzFo4MTSSSSUE6E8lhgWm5vjNZefZRWLbL5q0tsgAvkUwS0plIDgtMyyaeNCmJGNRXRTjxqAo2v9lMOmNPtWkSyRGBdCaSwwLTdulMmJTnVvSecnQVpuXSGs9MsWUSyZGBdCaSwwLHEXQmslSUeM4kYHiKj44rpNijRDIJSGcimfa4QmDZDvGk5TsTPafRZduOrJtIJJOAdCaSaY/rCrrT3p73HmfifbW9FSeHpzfJmDbprKwJSYoDfaoNkEjGiusKf8akPBYAepyJ7R6+kUkibeE4gnBQ/hpLph75LZRMexxX0JX0nEnfNJfjHL76XKbl4MgxGkmRMOFprkQiwapVq9i3bx8Av/jFL1i1ahWrV6/m5ptvxjS9i8DWrVu55JJLWLZsGV/96lexbS98b2xsZO3atSxfvpzrrruOZDIJQDwe59prr+XCCy9k7dq1tLS0TPRLkRQpQniRia4pxMLe/nc/MnFcOEzTXFnLxbRkmktSHEyoM3nllVe4/PLL2bVrFwA7d+7kJz/5CT//+c959NFHcV2XBx98EIAbb7yRW265hSeffBIhBOvXrwfgtttu44orrmDTpk0sXLiQe++9F4DvfOc7LF68mCeeeIJLL72U22+/fSJfiqSIyUcmFSVBlJyGSk9k4h622xZNy8W0ZWgiKQ4m1JmsX7+edevWUVtbC0AgEODf/u3fiMViKIrCcccdR2NjI/v37yeTybBo0SIA1qxZw6ZNm7Asi82bN7Ns2bKC4wDPPvssq1evBmDVqlU899xzWJY1kS/niMYu4nyKK6ArkfVnTKB3zUQcloGJyHWw2Y7APVzzeJJpxYTWTPpGCw0NDTQ0NADQ3t7Oz372M+644w6am5upqanxH1dTU0NTUxMdHR3EYjF0XS84DhSco+s6sViM9vZ26urqRmRbVVVsTK+tpqZkTOdPBBNp0479ncytiaJp43v/MS4260m6UiYL5lVRUR4F8GdLdEOjsipKJGRMvl3jTG+bLNultC0NClRWRjF0rSjsmi4Uo83FaNNomJICfFNTE5/61Ke45JJLOPPMM/nb3/7W7zGKogw4bKYMIQWrqiO/0LW1JfwlSqOlpqaElpbuQzp3ophom5pbEoRUBUMfP2cyXjbvONCFablEAiodnUn/uK4pJJMmra2JUXU8jdSuVMaLhEfrqA6FvjZlTJuueBqAg01xQoGp6aUpxt+F4ShGm4vRpr6oqjLkTfikz5ls376dyy+/nA9/+MN87nOfA6Curo7W1lb/MS0tLdTW1lJZWUkikcBxnILjALW1tf45tm2TSCQoLy+f3BdzhOC6Asd1i3aSvKXDu6iWlwQLjuua6i3NmiCzuxImbV1TI9fi5G+EhMB2ivNzkRxZTKozSSQSfPKTn+T666/nE5/4hH+8oaGBYDDISy+9BMCGDRtYsmQJhmGwePFiNm7cWHAcYOnSpWzYsAGAjRs3snjxYgxj4u8Qj0RcIXDc4s3N5y/oFbH+zsRx3YKhRVeIcRv0S2VtOpPmIUe4Y8F2RK7nWelxLCM6z2VfS3fRfpaS6cukOpOHHnqI1tZWfvrTn3LRRRdx0UUX8d3vfheAu+66izvuuIMLL7yQdDrN1VdfDcC6detYv349K1asYMuWLXzhC18A4Prrr+fll19m5cqVPPjgg9x6662T+VKOKIQQuG7xzmu05p1JiTewmL+465rSLzKxLJe2cRB/FEKQMW1c1/v/ZGNZDoqqoKjen0eK7bi0dmWIJ7ITaJ3kSGRSEq3PPPMMANdccw3XXHPNgI9ZsGABDz30UL/jDQ0NPPDAA/2Ol5eXc//994+rnZKBcV2vxbZY01zt3VmiIR1D18haDomURWVp0ItM+tjtCoE9Du20tiNwXYGuKSTS1qTUTXpj2g6aqvh/Hil553qgLUVpNIiqynWUkvFBanNJhsUVAkeIop3XSKYtf1jRsl1iYYOM6XiRiVuozCWEwBqHNmfLdgCFoKHR0W2O+eeNlqzloqkKmqqQtXpej+24NHekBj3PcVx0VcVyXDpkdCIZR6QzkQyLK7y78GLNs6ezNpGQF2QLF8pjQWzbRfMjk57HuoJR1RgGw8pFN7qukrUdsqNINY0HWcuLTFRV8W3JH89vnBzwPNtBVSEa0jnYnirq+SHJ9EI6E8mweN1cAlGEoYnrCtKm09P6qwjKY0ECho6m5orTvbyJEAJnHAb9UhkbLTdlrwhBOjN5A7PewKKLmotMzF6OLGM6ZLKDO7asmYtoNBXX9XbASCTjgXQmkmFxBbnIZKot6Y/jet1ZkaCeq40oGIZKTXkIVckV4Hs9XuQik7F2YKVzaTTwFnF1JiYv1eUt/HLZ0RgHvNeUjzBSacv7rAZ5fabV4wRDAY2ObulMJOODdCaSYRHCuyA7bvGlRGzHJWs5hEM6tuMSCWqoikJpNICmKdiOW3BhFULgCndMnWlCCNJZy5dsCRiqJwc/Se+P7bgcaEvzs9++zfbGOCg9qbtExkLAoJGXabloueFeQ1dJZ+2CyGYkFGsjhmRqkc5EMiy246VGnCIcjkukLYSASFDHsl2ivVSDI0HPwbgF3VyMuf6T7+TKd0IpioLASyFNBrYj6Oj22pu7EiYIr7Bu5/5TVQaMTFwhsHvZDaAAyVGk6CzbZV9zYsyvQXL4IZ2JZFgcR6CpCm4RRiaJtJdeigT13KKonhbdoKH1E0L0JvkHvtiOFMt2ERS21CpAepLmTRxX0J3yHEA8ZYLiCVpmLQchFEAZ0Fk6AxTbAwGNzlF0o6VNm+60JaMTST+kM5EMi+N6xd5iVDvPX1TDQR0UhUAv7TBDV3ONAz2Pd3K1n7FcDC3boe90hqGrJIboohpPrF4dW4mUBXizMxnTQVWAQZxlz9R8DwFdJZmxRtzV1Z00sWy3aDv7JFOHdCaSYfEjE1F83sR3JrnW4IDR25louX0mvdJcjsjVTQ79OdOmjdZHpNfQ1VzKbeIvslnLKYhMNFXBtB1SaQtdV0BhwNfnOYw+EZWiIIQnDTMcQuQ2WioUZcpTMrVIZyIZlnye3ZncUYoR0Z32LqpBXSVoaH5xGXpFJn0m4Hv//1BIZRy/+J5HVRUcITCtiXe4WcslnltT3J2y0FTVczAZy1N1FgN3c9mOYCDRbV1X6BpBi3DWcnAcgcr4zOpIDi+kM5EMi+O6xRuZ5C6qhq72k5nPO5PeFz7XFaiKMqaZmbRp93MmAAgmZXgxY9rEc5FJd8pEVRUyWQfH9jq1FGXgmolpeQOLfQkaGl2J4QUrUxkvelGU0YlLSo4MpDORDIvrUtTdXKoCqqIQDfV3JuCJO+ZxXIGiHvqdtWW7uI5XQ+qrPqxrXqprIhHCGzR0XUFpNEA66+C6IjdPkws7FAbUHzMtpyByy6OquW60YRxhV9IkEFC9wU/pTCR9kM5EMizprI3luEV5AUlmLL/4HgwUFjICuejB6pWfcwVoinLIMyG244KisL8lwV0/f5k9TT0LjfLzJhOJ7Qg/xTW71ltUlMxYOI5LvuNXVcAe4PV5UiqDCzsO9fk6rksibRHQvchnoM4wyZGNdCaSYXn6pX388vfbp2Rvx3AkM7af3gr02QKZj0xMu3fNxIsqDtUx5s/b3ZRACPj72z1L3XRNJWPaE6p3ZTuun+KaXeutKO5Oed1Yuu45ClUZOIrMWi7/2NHG9v1d/X/wMJplGdNBCC/FpaqjUyqWHBlIZyIZElcIuhJZOhMmguKbfk7lnYkQ/fbTG7nOrt5CiG/s6iCeNA85Zec4Lgg42OYp827d1ZFTEPZQFGVC6yaOK/w60awaLzLpTlnEIgZBw4vMFFXpp4zsuJ4SwNMv7eN/fvcOb+3pKPj34faiJFIW+bdXVRUsq7i+B5KpRzoTyZC4riBjOWRN70IzUfMFB1qTh5R6SuUUg1VNRe3TqmTo3sW19130b57fzcvvtA6YBhoJlu2iqN4+kJKIgWm7vLmn0/93BUhnJm540YtMTKIhncrcmuLulEnA0FCUfGTSf0DRcQRZyyaddVAU+OWzO9i2t8fufHvxYCTSpt92rQ3grCQS6UwkQ+JtFPQk1oU7Nk2roUhl7VGn0URuBW84oGEM0F0V7FOAd1wXy3HJms4htzl77bHetsbTjquhLBrg1Xfa/H8PGKrfrjwRZEyb7pRJWSxIMKBh6Kqf9gL4xdPvsPnNFvpe621HEE94j7vwrDnUVYT55e+3c7Ddi7A89eHBHYS3H8Z7P1VlaMcjOTKRzkQyJK7rXUjAqz1MRN3EdT1J9dHWMRzXJZO1CQX0AVt1/TRX7sqafx1ZyznkNmfTdv01wfVVEU4+uoodB+J0p3palBM55d6JIGs5xJMW5bEAiqJQEjH8586YNm/t7WRfc6Jf3cZ2Xbpyj5tRGWHtBcehqQovvtEEeKkrcxCJg7y+mR/5qMq4bKuUHF5IZyIZEtN2/JqDp/00/s/huG5OFn5056UyDq6AUHDgyCRfkM/XNPJ7PjKm40mLHAKW7dLSkQY8Z3LK0VUIAa/taAfyE+ViwuommawnpVIW8/bdl0QC/jT8gVwdJ5W1+zlm23b9ve8VJUEiIZ2T51fx+s520lm7316U3li2VyfKkxe2LEYVacnUMeHOJJFIsGrVKvbt2wfA888/z+rVq/ngBz/I3Xff7T9u69atXHLJJSxbtoyvfvWr2LaXd25sbGTt2rUsX76c6667jmQyCUA8Hufaa6/lwgsvZO3atbS0tEz0Szki6d3qalr2hNRMHFf0U/cdCfkW2ZChYRj9W17zNZO840hlvdeSMZ1D7uYyLYemjjTRkE4sbFBdFmJmdZRXt/ekulRFGZE8yaHQ2Z3FdgTl0SC24xIL6X5kkncm+fmX3tFR1nSIpyyChkYo10L97gU12I7glXfahnQQluMy0LtVjN19kqljQp3JK6+8wuWXX86uXbsAyGQyfOUrX+Hee+9l48aNvPbaa/zhD38A4MYbb+SWW27hySefRAjB+vXrAbjtttu44oor2LRpEwsXLuTee+8F4Dvf+Q6LFy/miSee4NJLL+X222+fyJdyxJLsVUzOWu6EdHM5ruinoTUS8o4uGNAGTHP1RCaFaa6M6RxSN5edWwF8sD1FfVXET/ssnFdJU0ea9riX/jJ01e+4Gk9spydVVRYLkM44hII63SlPE6yx1bvRSmVs6DNY6KXHTD89Bl66q6EmyktvteQ+VzHg+2KaPfMpv92yl5dz7dCHGt1JDk8m1JmsX7+edevWUVtbC8Crr77K3LlzmT17Nrqus3r1ajZt2sT+/fvJZDIsWrQIgDVr1rBp0yYsy2Lz5s0sW7as4DjAs88+y+rVqwFYtWoVzz33HJY1eatTDzcGS8skehV3s5Y7IdsWnfwE9yh/eF45NxTQB05zGfnIpK8zsVGU0d9Ze9IsLs2daWZURf3jC+aUA/hdXQFDJZGxxj2Ks50eTa7yWBAQlIQNf9uk70yyNoJC6X3TdulKmpTnOsDyLD6+hrZ4hl0HuwFlQAeRNh10VSFj2rzwehOv72r39L+KrE1cMrXowz/k0OkbLTQ3N1NTU+P/vba2lqampn7Ha2pqaGpqoqOjg1gshq7rBcf7/ixd14nFYrS3t1NXVzci26qqYmN6bTU1JWM6fyIYi03b93VSXxntN6uh7uqZR9AMjcrKKGWxYN/TD5mamhK0eIZIa4rSsgg1lZGRn7zNu0OuqYpSW1tKaTRQ8M9uTtrXCOhUV8cI7PfW3NqOIBIJUlkV8wcbB7KrL4m0RXp7G0LAsXMqqCj3HEpFeZRZtTHe3tfFyvceDYCWzFJSGiYSMvr9nEOlvCKKmbvYz20oJ2s51OacWsoSdCbMXA3FJBgOUlkZJRIyEEKwpy1FPGly0vwq326AcxaF+O2Wfby6o4MPv6+S8opIv8+3JWESjgTZust77amMQ1l5hPLczynG34XhKEabi9Gm0TChzqQvA6VI8gXL0RwfDHUgFbtBaGtLHHLOt6amhJaW7uEfOImM1abWtgQRXemXLmpsivt/7uxM09KawEyPTwonb3NzZ4p0KktzSzfKKHp2D+Zer5216WhPkk0VKt925wrO3d1Zmpu7aW7t2RDY0pagpSXs11UGsqsv8ZTJjtxsRklQo6Mz6f/bsQ2l/P7vjexp7PAu6EmLvfs7qSwNjfj1DEVNTQlNTXGa21OEgxrpVJbulIWS60p76Y2DAMyfWcIr77TR3JKgpdJzZrbj0tjUhWm7hANqgd0AJx1Vwd+2tfL+RfU0B7WCz1cIQVNzN9GwzqvbvLpkR3eGrq4UTYZKeUlN0f0uDMfh+Ps7GaiqMuRN+KR2c9XV1dHa2iM/0dzcTG1tbb/jLS0t1NbWUllZSSKRwMldYPLHwYtq8ufYtk0ikaC8vHzyXsxhhusO7OwLaia2MyE1E9vOrQUeZXdQd9JEUbyaiTaA5lQ+6rBdF4Hw01yQaw8e5uk6uzMF0uy27dLckSIU0PxuqjwL5lYAhamu7nFyur7NtkN3yqQ8FvTXBsfCgYLnPXpmKeAV4fM1Ey895qUryweIKqvKwl6qzLL7pTvtXsX37Y2eDIvXwOAWKAtIJJPqTE499VR27tzJ7t27cRyHxx9/nCVLltDQ0EAwGOSll14CYMOGDSxZsgTDMFi8eDEbN24sOA6wdOlSNmzYAMDGjRtZvHgxhjF+KYUjCSEELgMvjMrvB1dzraMToTvV0Z3N6UuNzlF1Jk1CAc1TDR7KmeQK572dScZ0hs35d6ftgoFA03Zo7kwXFN/zVJeFqCoN8ebuDv+5u1PjWzfJZr2OrLJYAMd1CRmqXwPpSppUlQapyEVC6aztf562k1tqxcDOpDTi/d6k03a/wUUrp2vWHvckdWbV5NJqaXvQuRTJkcmkOpNgMMidd97J5z//eVasWMH8+fNZvnw5AHfddRd33HEHF154Iel0mquvvhqAdevWsX79elasWMGWLVv4whe+AMD111/Pyy+/zMqVK3nwwQe59dZbJ/OlHFYI8IrfA1z3UmmbUMBrJzVHOViYNZ0R3b0+uXkvj/9l96iciSsE3SmTSE7kUdP6O5N8ys52RM6Z9O5MG96ZpLM2yV7RhWk6tHVlqK3oX9dRFIUFc8vZdbCbdNb2ZN0Fg85uHAoZy6YrkaU8FsR2BKGATjig+9L79dVR//1Im7YvqWI7LvFE3pkE+v3cWM6ZJLN2v8/LclwQwo9KFh1bDUAiM7GClpLpx6TUTJ555hn/z2effTaPPvpov8csWLCAhx56qN/xhoYGHnjggX7Hy8vLuf/++8fX0COV3E70gS6uqaydu/tXyJoO7igu+G3xDNGwQZne/wLWm87uLKlR7CEHz1Glsw7hoI6qKv10ucC7wGtqvkOpcJAwaw49gOm6+cd7MzC6ptKe8GY8KksGbkBYMLeCP//jINv2dnLqMdXesizTIRQYn1+z9m4T2xFUlARxXEEwoOEKiIUNkhmbmVUR35nkU1H515rX88p3uPWmNOJ9PsmM1U8mJWt5jnH7/jgVJUHm1pX4j5VpLklvhvyW59NIg3HxxRePoymSqULkUlwDXVxT2R6Jd9Me3SxIIm35A3KDPrcQJDN2bip95HfxGdMmk7UpLwkOWETPY+hqbobFu6gqeAFYxhxaCyx/oRTCUwHWNZWWTm+OpLxkYOc4sypCLGywvTHOqcdUe0q843T37jiuP8dSUeLVTAK6huMKomEDOtLMrI4SMFRv86LpYOdSVD0zJgM7wVjYi0wSaS8yEb2kU9JZGwXYdbCbU4+poiSSf6xVoJYskQzpTPIzHS0tLezYsYOzzjoLXdd58cUXOeGEE6QzOUwQOUciBshzpXPOxM1JhIw0zeW6gkzWHrao7gpBKmN7aajsyC9OibRNxnIIDTKwmEfXVC/iydVMomGDRNrKzcwM/lpM2wHhKfDm34OObq8YP9hFWVEUZtVEaWzxuqU89d7xqZlYtktXLlVVWRJEUbzOu6ChEQsbKIo3hKgoCpGg7u1VyX1WGdOhK2nSUDNwJ45XyDdyM0XesKOeSxtmTZfG9hSW7XL0zDICuQn67pSnP1aMC9MkU8OQziSfRrr22mu5++67mTNnDuBJnNxyyy0Tb51kUhDCu6gPdG1NZ20qc2mVZMoa8cUjazlYjjtsHSSTdfz0U8b05FoGSln1pTtlks7aOV2uwR+va94iLFcIMpaXssvmJPWHSqtlTQdFBUNTSWYsyqLBPgODkEzbuK5LSa/5loaaKG/u6SSVsVEUZdzqCrbj0pXIoije9Hsq42DoCo6rsOiYao6eWeqnsCIhnXTW8WsmmaxNV9LkpHkBXFcQT1qEg1rBZsqSiJEbAvUWa+ka/g3E3qYEigJHzfBSXKW51mcUBVfqc0lyjKgAf+DAAd+RAMycOZODBw9OmFGSycabPh+o7TdjenWJYO4iPNL2XdN2/d3kQ9GV7Gm9zVjOiGZ/TMshnRNrDBk6+iCDh5CPTHLpHtMlYKi+QxnKMSazFrqmYOgqybRXbO5KmsTCht8l5gpBMKB78iU5ZlZ73U6NrUlURTnkvSl9sRyXzoRJWTTgO1tNU9E1laqyECcfXeU/NhLUSWdtbNf1NzMK4aXHLMclFjHQVJV4wvSdXUnE8NUO8p9xXhm4tStDZUnIdz4lUcNXH5CSKpI8I3ImNTU1fO9732Pv3r3s3buXu+66i9mzZ0+0bZJJQghw6R+Z2K7rO5OQoY1oNiNPOmv59YreuK4ouPh29dKwGqn2V8ZyyOTEDEO5nR6DYeiq3xqcr30Yuuq9liEuhOmsg6Gp/orfjNmjbZV/HaoCc2eUIETPhXdmbiJ9f2sSReGQ96b0JWu5dCazfr3E0L1lYAPN13iRiY3teFpbvSMq23EpCRsc3VDKnLoYqYyXiiyJBHJOp+cGIF/vae1KU13WM3xZGgn4P1PugpfkGZEzufPOO9m2bRsXXXQRF198Mfv37+ff//3fJ9o2ySThpbn6Dy0mc0KK4aDXNZS13BEXXZNpi4Ch9otkspZDc0fK/3tndy9nMkI131TawrTzzmRgXa48hqbiuAKBIGs6BHSvzpA1Hb+m0BfH8QbyekvLpDIWXYkebSvTciiNBAgaGnNnlJDM2AjhdVjVlIfYn49Mxulim8nadCVMKktD2K4glEtpDdQSHQ7m0lyuwHJcP/qrKAniOp4DVhSF8pIQs2tjJFI2JWEjN+jo+jZ3JUyEK2iPZ6nq5UxKIl73mOOMfgeN5PBlRD2LtbW1fP/7359oWyRThMDzJn2vC3ln4i2f6pFvHw5XCFJZh3BA67fxz3FFwVbF3mmukUY+8ZTldyqFAlo/PbHe6LpKOusV+E3bwdCDBA0tt/Nj4CfrOwWuKF59pDvd0xFl2a7f2ZSXou9MZomGDBqqo2zb2wXKwCq8h0J7PEPGdLy2YEcQjOSciap6HWq5DizXFeiq4s+Z5MUhFQVKowaptFMQyVWUhDBth9d3ecfSWQfTdogns7R0przoxhWFkUmuRpSSsyaSXozImfz1r3/lnnvuoaurq+Du9bHHHpswwySTRz4y6XtxTaS8u/9gQPXbg9NZp6B1dCC8tlpP7sPs43zyRd38Bbt3mmskU+m246XeOnKdTb1rCANhaCqJXD0oa7kYmooIQHt3dtD6jJlLtyXSFrGwQUBXaelMe3WHXkN/4WCP4kJlaYi2XOtuQ02Ml99pI560CAzRtjxSXFfQ2OzpNlXm0lzBXvMiAUPzO7BMy8EwVC+tZzpkTJuO7iwVsSCaqoLi9EsL1pZHfGeRztikMjZtXZ5j3HHA02bLRyZCCD8STGYskmmL8BANEJIjhxE5k69//etccsklnHjiiUNeRCTTm75prkSmZ/lUz6yJN+w31NfAtBwQyoAy764rMK0eXad40sTQVH83+3DOxLJdFLzdHdGQnismD+FMdBXH8ZqeTcvB0L1IJjvEtsVM1uJgW4qfP/MOn1y5gJnVUVo6ve2K5SVBHNdLgQWMnotyOOgty8qaTq8ifIq5M0pG3KE2GFnLoS3ek6oSUDB8GNBVsraDruUippxeV9p0SGUd2ruzVJeH/c+3byu1oijU5JxFImORSHn1Ll1XfQdZlZNpyVoOwUDPYGR7PMPM8pC8LkhG5kwMw+DjH//4RNsimSJETkmlb9Ynv3wqYGhEcpIdeRkSL7kyMGnTRlV71rvmRQnBiywUpWfrYXfKIhrW/cHF4XaaOK7nyRpbkzTURFEYWJcrj6F7dRs7JwVjGCqqkmtdHqT+k0hbHGj3ZkXe2R+noSZGKqdRVhELYlpuwZKpPLUVEXY0xqmtCKFrCvtbk8ytK/Fe/xju3jOmTUevgcWM5fpzIOCJSqbzUjGK4n9W6axNOmPR3p3luNnl3tS8oQ144c9HHqmMjWH0RKJtXRkiQd3/mZYtqC7zUn2JjOU3J+QfLzlyGVEB/thjj+Wtt96aaFskU8SGP+7gqc17+kURecXggK4RzUcmg3Rc9Y4okmm7p123j0yL7bgEA5pfj+lOW0TDBkFDI2PZw06MO7kIprUr40cAAxWh83jdXF7xHSCgqX56Kj3Iat1k2qa5w7t478yleTKWk6s7BLAd4av19iYS0gnoCsKF+qqot6xKGViNeTQk0zadiawvh6IgCqKLfJrLdQWaqlCRaxJIZy3a4l46r6Y8hO24hAdRJCiNBtFUhe60VeAYWrsyBcV3gSAaMjA0lXjSQlEU/7PMI5dmHZmM6HZi7969XHLJJcycOZNgsGf6V9ZMDg8aW5O0dmb6XQSSae9ia+gqiuJdgL3IpPB81xW8s6+L0qhBdXnYk2DJX7Tyqa7cXy1bENBVUllvpiSRtqgqDZIxHbKmO2zB2rJdmju9brCZVZFBdbny5Lu5MrkajWFohAN5McT+9R87V7RuaveeY19LEst26OzOzXio3p6dcLD/RVlVFGorIuxrSTCzOspLbzXnLvJDvqRh6U6bXt2jNOjZS+HeGUPX/AaDkrDhO6901hOmBKgpD+M4YtAIQtdUYmGD7lShY2jtynD87HKAXHFfpSwaIBYx6E55qs3tHSmqy8O557Q50JZk/syysb1oybRjRM7khhtumGg7JFOI7Xpihv0jEwtDU9E01b94mgMMFsaTWdKmjWk7fmG7J/WkFDgpx3Fz4osupuWQyljMqY2STNuYIxiKNG2H5g6vflFXGRlSlwt6tLnyUUjQUImGcym7XI1G6+NMspZNe3eWOXUx9jQl2NOcoCORpbzEm9MIGNqgz1sSCaAoMKsmyotvCNq60riifEgbh8KyvRRde1eGOXUxb1CzjyPTVAWEwLIgVmH4KbWM6fgzPdVlITKmM6DQI3hKAZ4z6WmISGe9Ynw+MslaDiWRAMGAJ+EST5oEdI2MaefqUSoH272NjpbtDPvZSA4vRpTmOuOMMzjuuOOYPXs2s2bNor6+Xu5bP4zI3433nRlIpi3/wpWXKe+rtiuEoKkjQySkEYsYGLpSkM9H9NRihBD88LHX+ceOdhAK3WnTV/4NBb2hyOEmqk3L5WB7iooSr8V3KF0uyNdMhC8/b+ia39KbMe1+g5qOKzjQ6kUlZ5xQi6oo7DrQTWeuI8q2XaJDrOI1dBVNVf322cQYd5pkcztkuhLewKKXqiq8B9Q1xeuIULxGgGjIa0pIZ23a4hnKooFceoxBBzw1VSUW1gsik9ZcVJPv9LJsl9JoAEPXCqIYBW8OJ5mxiSdNNFUha8mW4SONEUUm3/3ud/nhD38IgKZpWJbFMcccI9Nchwm27WLZA8yZZDwtK4QgGtJzFwmnQBAymbHJWrZ/8TR0DUP3tLP+9OpBzjqpzr+YZkyHg+1pdjR2cfyccg7mBBHDQYNw0KK5Iz28M7FdDralmF3ndUkNpcuVt8dxhR+ZBAzVV8nN5qOsXjfQjis40ObZNaeuhIaaKG/v7SKZsSmPefWS/P6QwYiGdOJJ76I9UomYwchkbeJpCwH+jEnfVJWmqohckT9gaNi5x6RNh/Z4tmBGZDDnq2ne1sbtjXE/9ZdPkeUjE0VRCAU0hBC+M3FdQcDQ6Og2vbXAQQ3Tcslkbf99lhwZjCgyeeSRR/j973/PsmXLeOqpp7jzzjs55phjJto2ySRhO56Cbl8J+HTW9nZxKAoBXSVgaL7mVp6WjnRBi2yev21rZfObzbR0pv3H5wu1TR1pDF31O6bCQY1IUB9WfBGgM5ElnrJoqI7gugypywVe26z33PnIRKEkt78ja/ZXDnYcwYHWJCURg1jY4Kj6Epp7tQULIQZNFeWJhgy/KcBLpQ358CHpTlkkcxHAQG3BkGtAUBRKIoYvseJd1B3auzN+W7Cq9okae6EqCiVRA9vpWW/c2pVBVRUqYkEcx+sgy0eDsbCnJN2dNr0VxSkT0/LSaIah+p2AkiOHETmTyspKamtrmT9/Pm+++SYXXXQRu3fvnmjbJJNEPhrImIUX8lQmV0gX3h1+0FAL0lzprDcVPtDyp7dyO8kz2Z7Zkbw4YHs8i3AFnTlJ90hQJxLUMe2h5Vpc15svAU9Q0RViSF0u6EnrdOfbnDXNXwaVGWDi3nIcDrYlmVHpbVOcl1PKhR614MAwzxkMaP6wYsZyDlm/yhWCRMby37fKkiAC0U8+RlUUz0nmOsx0TSEU0DnY7k2we51cntTLUPMg+YHMfPqqrStDVWkQNZe2Kovmf35PGq8rYaIoCoahEgl7r9nQVJJZe8xdbJLpxYicia7r7Nmzh/nz57NlyxZs2yYej0+0bZJJIh8N9JURSWVtQkEdRcG/KzV7rbttj2cGbMvtSpoczHVDZcweyY3e+fiWroz/90jI8BY8MXi7LngT+k0daRQF6qsiCJchdbmgx5nkoyLD0IiEvL3xWdPut8MlmbZo6Uwzo8pzJrNqYv7dfHksAIoygmhIQ1HxNcBGqrTcF2/4E9riWYIBb9ZHEQPXPUoihp/+0jSVcG7nCHg1j4FqLX2pKPE6svLOq7Ur4w8r2o4gFulph84fb8tFbeGg7k3Ygy+OKTcxHlmMyJl85jOf4ZZbbuF973sfTz31FO973/s488wzD/lJH3nkEVauXMnKlSv55je/CcDWrVu55JJLWLZsGV/96lexc0J+jY2NrF27luXLl3PdddeRTHp3pvF4nGuvvZYLL7yQtWvX0tLScsj2HOnkL/a995ULIXJpLg1dV1EUhWBA8+60XeGlUOIZf01sb7bt7fT/nDEdv923d6dQU3vKT4VEghqxUI8zGaxgbTuCg+0pasvDXqeQwrAdQ/kUXCI3dBjQVS/KCujegqw+Oaj9LUmEgPpcZKLrKrNrY/4iqnBQG3aa3TBUFBRf6v5QZdqzlotAsL8lwZy6ktyqXn3AIc36qqjvTFRFIdKrSSDfFhwaZrCwqjQ3jJiycFyXju5sr3oJvrgkwKzaGAFd5e19nQP+LAWvviU5chiRMznvvPP4v//3/xKJRHjkkUf48Y9/zO23335IT5hOp7n99tt54IEHeOSRR9iyZQvPP/88N954I7fccgtPPvkkQgjWr18PwG233cYVV1zBpk2bWLhwIffeey8A3/nOd1i8eDFPPPEEl1566SHbI+lJc/V2JmZuYjwU6Fk+FQ7q/k6T9ngGVVEHTJts29NJZS49kjZ7BBXjuTtlVVVo6kiTNR10TfGK4vluMXvwgrXjCpraU/6wIoIB6zW9yTubfM0kYGhoqkLQUHPLuAofvy/XFFCfi0wAzj2lnvMXN+C4YkDn2RdV8WoWoYA2pGzLcKTSXoG7qSPNvJllQw4d5qOCPLGIZ2cs7EUsQgyfnsvPijS2JXn6pf24whN49AryhRFRJKQzuzbGW7s7BkxnKcrQUabk8GNEzqSlpYUf/vCHfOtb3+Kee+7h0Ucf5Vvf+tYhPaHjOLiuSzqdxrZtbNtG13UymQyLFi0CYM2aNWzatAnLsti8eTPLli0rOA7w7LPPsnr1agBWrVrFc889J9uVDxE/Mul1J5mfTwj1yv/nO3VMy6W1K0M45B1vj2fY35IAvILzzoPdHD+73Fsfm+2JTPKRSEN1lIPtKVIZi2jIQKFXh5XZP1rIk0hbZEyHqrIQjuNiGOqwrcF529NZ2192pSiK5xjN/pIqB9qSREK6XxMAOKq+lDNOqPPu7odJFeWJ5BaKjaVmksxavsDkvPpSv0trJOTfT7+TSxm8LThPKOA5wC1vtvDiG02cMLeCE+ZW+E6s941D0NCZUxejPZ7x1xn3JqDLIvyRxoi+mddddx0zZswYl4VYsViM66+/ngsvvJBQKMQZZ5yBYRjU1NT4j6mpqaGpqYmOjg5isRi6rhccB2hubvbP0XWdWCxGe3s7dXV1Y7bxSCM/X2LZri9KmOyly5WvEURykUn+31RVwXFcHvzt23Qksqw8ey6hgIbrCo6bU872xjipbM8u8kTKIhTQqK+K8Pe3W4mGDaJhryaTLx5nrcHFHltz+fmyaADLdokOIGnSl/wFNJW1CeiaLwoZDuqkMlZBNOYKLwqYVRtDURR/J31+1kZAwarboYiGDAK6Rmcie0g1E1cI0lmHxjav9jR3Rikt7QkCI3z+fJNBTXnIjxyGUzDWVJXFC2pxHJfFC2p9WZZUxi5QSwYwNIW5daVAIzsa41SWhgr/XVdJZaxhFaYlhw8jciaWZY3bPpM333yTX/3qV/z+97+npKSEL33pS/z5z3/u9zhFUQYJnwf/YqrqiAItAKqqYiN+7EDU1JQM/6BJ5lBscl3hO5NAUKe6KoamqRzMqdRWlkeYUVtKTWWEqvIIlu2i6jozq8NoqsKzf9tLe3eWhpoYjz+/m4qSINGQzsnH1vHnfzRhOS4lJSFqakqwXEEkZDB/Vjl/3drMnoPdzG8oo6oqSm2Fl1ZSNY3KyphfkO9N9p1WAGbVlRKJhZgzo9RPzQz6nuRmJbKmQzCgUV0Vo6amhLJYkK6ESSQW8t83rw6U5YSjKqkoj9KZyKIAZbkuLlXPMnNG2bCtwQCRWIjSWJDmjjSx0sioP5tM1qasLE1TR9qrh4R0ysrC1NeVFtRDBmN2vSdnMqe+jFgsTGmZQl1d6ZDnZC2H88+cS1k0WHBcTWSZ3VBGRS+HkbUc5uaWde1tSXLBWV7qcdMLu6irjPCu42rREllKyyMjjuYmk8Pl97eYGNGnfNJJJ7Ft2zaOO+64MT/hn/70J84++2yqqryd1WvWrOEnP/kJra2t/mNaWlqora2lsrKSRCKB4zhomuYfB29hV2trKzNmzMC2bRKJBOXl5SO2o60tccjDZDU1JbS0dB/SuRPFodrUO83TFU/T3NKNrqnsO9AFgGM5xONpFMdByTn3ZDKNoQqSGYtNf9nNMQ2lfOz9x/DIn3bx2s52Tj26ing8RUBX6OjO0tqepCUWoK0rTUBXcXPptIzpYGgKqUSGbO7uv6PLs2Gggbdd+z2bFOHS1ZkmGTUQ1tB5+WTCc4qJtEV5LEg8nqKlRUdXFVJZm4PN3ZTnhhA7urPYjkt5SZD2jgSJlEU4aHAwnSWgqyQzNp0dyRHdaduOJ5Wfylq0tSVoKRk+iupNd8qkoyPFzsY4J82rAKCzM01XZ4rkMKk9gNKgRmnEoLYsSHNbN9Wl4WG/H47r0tWVxu3znnanLOKxAHa2J23luoJ4V5rj51Twt7eaaW3vZmdjN0++sJtZtVGOqo3SnbRoPNDlz/X0xcrNLI002hsvDqff38lEVZUhb8JHdCt/2mmncfHFF7N06VI+8IEP+P8dCgsWLOD5558nlUohhOCZZ57hjDPOIBgM8tJLLwGwYcMGlixZgmEYLF68mI0bNxYcB1i6dCkbNmwAYOPGjSxevBjDkBO3o8WyRcGf88FgXnI9aKh+aigvQ55fZvKHvzdi2g4XnD4bTVP58JJ5rH7PUbzvtAbv8UGddKanZpBMe2muytKgvw8lEtTRddVvQR5qp0lHdwZVydUDlOGHB6GnddjJ7U3Xc9FrNKSTzWlK5SPgfO6/NBLEsl0iIYPaijCm6Q5YNxgKXVOJBPUCxeLRkDZtOrszZC2H2bUxf/nVcDWiPJVlIT656kRqysO4Tq/Pbgg01ZPn758REP0aHVTVa5E+dk45Wcth98EET7y4B4CDbemc7L6n7zYYXcmsvydGMv0ZUWTy/e9/n7vuuos5c+aM+Qnf+9738sYbb7BmzRoMw+Dkk0/m2muv5YILLuBrX/sayWSSE088kauvvhqAdevWcdNNN3HfffdRX1/Pt7/9bQCuv/56brrpJlauXElJSQl33XXXmG07Euk9cW45boHiLHiFVN+ZBHv0uZo70ry0rYXTF9RSk0s1KYrCu46t9n9eOKSTNm1sx9t0mMrYlEUDRII61aUhWroyhIM6Ac1rtw0amrdtcZCIsbPbpCQSQOQk2EdyYe1ddDZ0xZ+LiYYNXOFFZrYjMHSFjm4vJVYaC2DaLrUVQaJhHVX12nSrSkIDPsdglEZ7NMAKxS+HJ5m2acoJWs6qjWHZzog6yfL0XhgmFAo2Mw5FXn4mP1vjydoP/F4HDZ2j6kMoCjzyp510pywWHVPFy++00dKZprosREd3lrqKyIBOOJGySA3Q8SXrLNOTEX07y8rKWLFixbg96bXXXsu1115bcGzBggU89NBD/R7b0NDAAw880O94eXk5999//7jZdKTS25nYtlsw3Q49rbSAr0mVsRz+9OoBgobGklNnDvqzoyHdXx/rCkEyY3vT4YZKdXnYdyZG7q63Zy6jf8HaFYJ4yqQs5hXfYyOoG0AfZ6L1RCb587OWF3UYukpnLiVWGg2QSZtEggaaqlJVGmJ3U4JZNaPL/edrD5kRLBTrjRCCVNamsS1FJKRTWeIJTI5mAZWuqd4umdxO+OE6ufL03toI3g3GYE4sFFBBV5lZFWV/a5KT5lXynpPrefmdNhrbUtRVRrCyzoDLs0Ruut91RYHCsO24NLYmmVM3vesHRyIj+oa9733v45vf/CZ///vfef311/3/JNOf3suoTLtHqyqdtdFUBU1Ve93NexeErbs62L4/zrmn1A+ZPslfhNKmTSbrTcKHg96Cp3zLaiSo+6mocCA3xzLAXIbjCOIpy1tOZQsiIxQR7H0R1XVPTh8g0kuGPt+A4Ke5ogEQnnMDT5MrHNSHndPoS769ODtEtDUQ+XUA+1oSzK7xOsscMfzQYW801dPrMm2HWNgY8Z1+0NAKbLVt13+v+hIK6NiOy4K55YSDGh88fRaVpZ6a84Gc7I2Gpw7dF2/JGigovhYYeF13+aVskunFiL6deXXgJ5980j+mKApPP/30xFglmTTsXrMllt2ju+XtvlBRFPyJ77z0+uY3mymPBTj9hNohf3a41/rY/MBiJKgTDugcVV/KjgPdVJWF/At8KKiT6TXk2BvLcUikLcqiAQQ9F/rh6O1MArrmp5ryrbPJTI/cS2fC9CXkg4EeeftQQKemIjSiGk1vCtSJc2+s7bjsbe5GwRvWrCgJ9btrz1ouqbRFezzbkzYUw0vH9EZRPFFH03SpLR95LdEw1AJn7goG7cYKGBqYLucsnMEZJ9T60UV9dYTGnPJyMKjSHs9SUxYucGiZXK1K0xSSGcsv0scTJlY+kpOprmnFiJzJM888M9F2SKaI3tPZ3tBiLjIxbQy9R0oF8Nt1hYAPvHvWsDWLSK/1uHmdqHBQJxTQqC4L8qW172ZPY6d/0YiENLoS2QEnxju6TVxX5KIGMezMRB6jYCOhV2AG/DpPIm36BfKupEksbJA1bb/ekae+Mjqi5+tN3vlmsj2CkqblEM/N23Qls2iqOoAzcTjY4c2XzKrJdc8ow0/79yWgq2Syzqhac72tjYXv/2ARWSigQW51b29Zm5lVUV58oymnNKySyppkrUI7kmnPcetaz3CjyKUygVwBXzqT6cSIvp3JZJKvf/3r/NM//ROdnZ3ceuutvkaWZHpTUIC3XV9eJGM6no5Vr1/ofM2koTrKiUdVDPuz8ymwtOn4ulyRoJa7qOTniISfRosEDbKW4xf/e5PfrVEaMdA0dcQ1AF3v40xy3qSqzOso6071DC7Gc87EdsWQC7BGSknO+WZ6RSZZy0HFazYIBfUBJUdSaYvmzh5BS9t2CQX0EXdy5ckPaQYDIz/PX7RFrqtrCMetayrBoF4Q3QLMrI7guMKX7ldRSPRZB9ydsnPORCGd9SR6MqaD7QoUdeAZM0lxM6Jv2Te+8Q1KSkpoa2sjGAySSCS49dZbJ9o2ySRQUIDv1c2VyXprWHtfSAxd44Onz2bVe+YWpCyE8OTk+6rEKnnHlLX9O85QwNv1EcwtcQIKWo+zlkPWdvopGLd2pf3HjKarKb/fA8htG1T811ISCRBPmv5++O6U5W9hHGkabSjykZynAdZTi9J1zwZd87TL+pLMWDS1p6kpD+d2yDi+/P1oCOQcVl/drqHo3QU2lLBknvJYELOPJE19lRfFNeY2VoYCGu29JFfyq5F1rSfqzZouyYzlX5AOUWhZMoWM6Fu2detWbrjhBnRdJxwOc9ddd7F169aJtk0yCVh9IpOemknuzrFPBHDmiXVU9LmwJdM2laUh0pmeekc6Y2MYXgSRzjq90lze3XI0pOcuQkqBM7Edgeu4vmRLnnxkEg3po+pqgp7oJKCr5K+rqqJQFg3QlTRze+29i1ksbBAw1HHZX27oam4HjIvIhXyprO1HGJqqYjuioEZkO5722YG2JA05QUvH6VmbPNrnLx3leV7LtUJ30iSRtoZ9r0siBn12qlEeCxAOav7uGV33RDXzem8Z0ym4GVHw0qpdiayXyhNiTKuOJVPDiJxJX5kSx3FGJV0iKV7s3NBi0NByzqRnxa4xQDpJ19SCX3TbdlFVhZnVUWbVxUikbNJZG0VVqKuIEA5opE3bT3PlO4vCQR3T8lSDlT4FfoHoJx7Y3p0hoKvoqjbqqCFfNwkYhfLx5bEgnQkT1/Um5L1BRX1AKZdDQVW9JVUZy9MnEzm9rb7bDntHdKbl0pUySWedHnXk3G730VISCfTTzBoOXVM5dlY582aWMaMy4u1wGQIvlVl44VcUhfqqqL/+GLzoZPfBOJbtkMnaBU3Shq7SlciSynibGhlESklS3IzII5x++un8x3/8B5lMhj/+8Y98/vOfH9M+E0nxkE9zhQJe2in/K5wxvTRX3zt0Q1cKUhDJrE1DdRRdU6ksCTGjKuyp3M4o9YrtubpAd8oioKu+aGIwoHnKv72cVb4m4zieVEvvi2xnwqQkGkBRh19O1Zf8xTvYq2YCUFkaJJG2sF2XllzkEwnqBXs7xkqPDL3rv56CNl1R6Ewypk1TTtxxZnUUx3VzMv2jt8nIrVoeLd5aXoMZldFBpVB6nkMjYOj95pVmVkdo7sj4r82zQ2H3wQRdSRPHdfnpb7bS1J7KiUL2SvcJxrTqWDI1jOi38ktf+hKRSISSkhLuvvtujj/+eP71X/91om2TTAL5NFcoqGE5PfLvWcsh2GtgMU/vyCSVsSmNBArk2mvLIxw3u5xgQMvtItdJZx2S6dyiLbUn5eR1AfXakZG7+/bqJYpfnE5lLDq6s5RHA5Cbfh8NfmQSKIxMqnKquPGk6afRwjnp+PEiHNS9qX7H9brlRN+7+MINl8mMTXNnGl1TqK0IYVqu38ZcrJRGDEzL+x6l0t6NQ32Vt1a5KbdxEyAc8qLUeMpk58Fu9rUkeXVHG6qqYNmuX0sSiEPWzZNMHSOKnQ3D4HOf+xyf+9znJtoeySST78QJGTrdKSunIuzdRRuGRt/aq6GpCNdrcRV4d88F+e/cfAN4jicc8Np9U1mbYED3nYeuqRiGhu726hbLr+41HapKFToTWSIhnd1N3STSlldDGOW8BfTUTAxdo3dQUFWWW1ObNH3HFQ3pGLpWIE0/FsJBna6k6Wl0WQ7/2NFOVVmQ4+d43XD5bqY8yYxFU0eKGZURNFXFsu2CdbnFSDRi0NqV8RQUFO9zrFfCKAps29fFrNoeccBY2ItidjZ6a7/z/y+JGmiqwpY3m9E1VU7AT0OGdCZXXXXVkJOz//3f/z3uBkkml3x6IhzUsB1vu2J+7iKo99+k6O33dsmagqMbyofUfFJVhUhI93bJByxCgZ6CvqJ4/yZ6dTPlayZ5ufh40vTFH1MZm5Kogd4nVTUS8g6sr1BjTYVXT4inTL/LKxYOoGsq/We2Dw1PUNJree1OmTzz9/24ruDcU+p537tmomuq78hsxyWbdTjYnu4ZVuyzLrcYCQc0BJ4EzFEzSrwdNo7DvPpSXtvRxnnvmum/797NhsqOxjiqonCwPU0ybRENG2RMmyf/upc5dTHOHUKmR1KcDHmLd+WVV7J27Vpqa2uJRCJcddVVXHPNNVRUVDB37tzJslEygVi59txwUMe2vfRC/k45GOh/4dZUBVVVmTujZERKtCW5FEginYtMekUVJZFAQZrLn8vIdfsIIWiP9yyXioWMQ9qNkX/Ovumr6rIwqqLQnbLoSpm52kSPGOR4kG93th2HxtYkriuoLQ/zx1cP8Mvfb4dcmksIgWk5tHV7dYaG6qgnsqgoox5WnGwMXSNoeGrQpdEgkaCB7QhOObqKzoTJnuZEweOb2tMkMzbvPt5zmDsOeNHJm7s7vZuZQfTZJMXNkL+Z+XW5P/nJT/j5z3/ud3C9733v42Mf+9jEWyeZcOxcX2cooOEKgeX0DNj17X4C74I8r75k2MJsnvwGxUROfr53vaO2Ikxrr77SvAZU/k49EtJRFIXdTV49IxY2DukuPe+wIsHCcwO6RmnUIJ7s1WmGOuo02lDEwgaOW7g18eJz5/HO/i6e+dt+tu+PM7M6iu14Q3v5GsPM6qinqxUZua7WVDK7NuYX+4OGFwEumFOOoav8Y3sbc3ulrbY3entp3nNyPf/Y0c6Oxjgnz6/itZ3tgNfR5g6ggiApbkb0W9PR0UE229OqmUwm6erqmjCjJJNHfp9Jvvidybqk82kuQ6dvB7imqiN2JOCt2M0TCWgFkY6hFzqXoOG1/e7L3clqmhcZ5S/20ZBxSMXxfGqtb1Rj6AqluVmT7lyqRVXxtcLGg/xq4WTGpq0rg6J4e9nzumZNOdkUy3b94nt+54tlCd8ZFzuRkOF/lnkVaENXWTCnnNd3dhRMye9ojFNbEaY0GmBefSk7GuMk0xY7cxFK1nL8Vc+S6cOIcgarVq3iox/9KBdccAFCCDZt2sRHP/rRibZNMgnYjoui9FxoTccGJ9dKO0BkMlrKSnoGHMNBvV93WG8UReH0BbX88dUD7G1OMDtXuO3KRw4R49BaZHPzMn0jDk1TKY0G2NEYJ6Cr1FVExmVYsTexXtFWW1eGypIQuq6i4w33NXek/fbgRNqkqT1FfZW3/0Mc4nzJVKMqCtGQgWU7nHJ0Ff/Y0c7b+7o44agKLNthT1PCd6bzZ5aydXcHz73S6LWU15ewvzXpL1STTB9GdAt2/fXXc/311xOPx+nu7uamm27iU5/6FAC7du2aSPskE4xluwX6TRnTJZOvmRj9C/CjpXdkEg4N7UwAzjqxlmhI5+mX9iGEN+i3ryVJJKT709mjJaBrGJrarxaiKgpVJSFSGa+dNRbWR6VjNRL8poKsQ2tXmtqKnp31tRURWjrTaJpCIm2Rydo0d2RoqIkihEBV+td5pguxsI5lu8yrLyUWNnhleytCCHYfTOC4gqNnevvo5+f+v/nNFmrLw8yqjWFarp9+lUwfRnzbc/7553P++ef3O37DDTfw61//elyNkkwetuOiqaofmWRN2x9KHEmBfTh6z0hEQ8awnVjhoME5C2fw2y372La3izd2tfPOvi6WLpoJglHPmAAsXTQz13ra/9yqci9yclxBNGwQ0MY7MvGcSVcqS2fC5JSjq/x/q60I8/a+TkCQSFs0dWRwhaCh2rugxkLGtJVhDwd1XNfr6Dv1mCr+/I+D/OixrYRy80f51t+KkiCVJUHau7OcNL8SPff9SGVlZDLdGPNtmJQ9mN7kI5OQPzDoFhTAx0pZLzmO4dJc4NU3TplfSUVJkF/+fjv/2NHOeac18N6TZ6COYgd6b45uKOOEuRUDdmnVlUcK7BvvSCDvTFo6vSaCgsik3FML6EqYWLbrF98bqqPesGJ0etRLBsKTRfH+/L5FM1lx1hwc12XXwW6OmlHiN0UIIZjf4EUnC+dV+lI5qSF2x0uKkzFfLaZDp4lkcGzHRdMUwvmaidWj2Dse+fqAoRHMSYrEwvqw3xdNVVBUlQ+8u4GH/7CT5WfO4YwTarFsh9AhtsjmF3wN5IhmVPY4k2hIH7VUy3DkVYjzOlXVZWESaU9QMu9YWroyBAyNpo4UZdEAsYhBd2p4kcViJpDbHeO6Ak1TWbyglncfX8O+lqSf+kylbRzhzdwcPbOMipIgB9o8ZzKQNL+kuJmSBvZnnnmGNWvWsHz5cr7xjW8A8Pzzz7N69Wo++MEPcvfdd/uP3bp1K5dccgnLli3jq1/9KrbtfckaGxtZu3Yty5cv57rrrpP7VQ4RKyfUGMrVCrx9It7K3uA4FKNVRfE7xUbSBWZoKkIITjyqki+vXcQZuUKt7YhDmjHJ26AoyoCOrKI06EcssXDAT7OMF/mp/qaONJqqUBI2cB1vpqSqLIiqKjR3pImEdA62p/x6CUzfegnkh1KNAlVqRVGYXRvzIy4n9zpjYYPj55QDPa+59w4YyfRg0p3J3r17WbduHffeey+PPfYYb7zxBn/4wx/4yle+wr333svGjRt57bXX+MMf/gDAjTfeyC233MKTTz6JEIL169cDcNttt3HFFVewadMmFi5cyL333jvZL+WwwHYEutojJGjagnRuZe94De9Fc8Xz8AicgdZL+6t3Z5XjiEO/uCqgaUo/aRjwivNlOScXDWvj2hYMniMLBTSE8FqCHSEoiwVJmw6aqlJTFqK5I4XrCjoTppfisl1KwtO3XpKnJGz023GTx3VFr1UEPY/Rcq/ZNF2ZQp9mTLoz+e1vf8uKFSuYMWMGhmFw9913Ew6HmTt3LrNnz0bXdVavXs2mTZvYv38/mUyGRYsWAbBmzRo2bdqEZVls3rzZH6rMH5eMnnwBPi+LYuUiE0PXRrVUaSiiIYNQQBvRdkRVVfoqmgPeoUNpC4bcgiyFAYv/eq492IvE9EPqFhuOfB2gtiKMcL1oKJqT4K+tCNPckWZfbvdHQ00Uy3Ipmcb1kjyRkIGbk97vS8b0Fn5VlAQxzZ7OLX9ZlmXLBVnTjDEnZY866qhRPX737t0YhsEnP/lJWlpaOO+88zj22GOpqanxH1NbW0tTUxPNzc0Fx2tqamhqaqKjo4NYLIau6wXHJaMnXzMJ+pGJ66/sHa/I5D2nzGD7/rg/zDYUXoF+gOcVYkyT6ZqmDninr6oK82aW+it9x8uB9iYc1OlMmP7e+aCuMaMqyvbGLmorwt4U+P4uf01vOuuMaptksRIJ6ZTFgiTTVr9mDsfxGgwCuurfO7iu8B2vTHNNP0b0jf2v//qvAY9//OMfL6hvjATHcdiyZQsPPPAAkUiEf/7nfyYcDvd7XF6baTTHR0NVVWz4Bw1BTU3xqZoekk25FNfMeq+jRtM1bFcQCurU1ZaOernSQLxv8VwqylqYUVvqK/UOZnM4bdGetCjrtc3RtB20gE79jLIR737vS0fapramxO9a682HlhxNIm0TMFTfnvH8fEtjQQ60pZg3q5yysjD19WWoCmRc6ExZwH5e29nBzOoY1VUlJFIms2aW94ukivE7B0PbFSsN8/qONsqiAf93VAiBZujMaShH01TiWQchvBuZE3MLwRRVpbIySiQ0PovKRmPzVFGMNo2GETmTbdu2+X82TZOXXnrpkJdjVVdXc/bZZ1NZWQnABz7wATZt2oTWq7+/ubmZ2tpa6urqaG1t9Y+3tLRQW1tLZWUliUQCx3HQNM0/Phra2hKHvDOhpqaElpbuQzp3ojhUm9JpGxB0daZQFUgks3QnTTRVpaM9iZMde4tmIm2RTpl0dqRwe6kED2RzOmvT0Zkim8li6BpZy8GyXI6qL6Wz49CbLLq707QbyoAT7ulUlua2FBWxIC0t3eP++Qbyy7lUyKRN2ts8uRgDgU7PbvgZFWFaWrsxdI22tkJxxGL8zsHI7NKFS+PBuK+9ljUdby98e+7zdBya2lK4QEW4DE1V6OrO0NLSPSHOpBjfy2K0qS+qqgx5Ez6i27w77rjD/+8///M/efjhh2lrazskg8477zz+9Kc/EY/HcRyHP/7xjyxfvpydO3eye/duHMfh8ccfZ8mSJTQ0NBAMBnnppZcA2LBhA0uWLMEwDBYvXszGjRsLjktGT75moireVkXLdsjmCvCjlXofDF1TRpw2CwU0ZtfFEEKhK2niuoKjG8r8eY1Dpbo0POiMSijgTWsPJac/FsqiAQKGSjSsE+7VRBAyNEoiAf95G2q84ns0PP1TXL2pKQ/jCNdXf85aToEyQjRkYDsuIUMjHPRayTOmI7ctTjMO6VtbWVnJ/v37D+kJTz31VD71qU9xxRVXYFkW73nPe7j88suZP38+n//858lmsyxdupTly5cDcNddd/G1r32NZDLJiSeeyNVXXw3AunXruOmmm7jvvvuor6/n29/+9iHZc6RjOy7hkI6ieLMBvWsm49Ulq6neHpORdCcpikJlSYiKWJCM6XhLtMZh9mOoAUx9EIn68eKC0+cwu7YEx/EkZfIYObma2ooQe5uTzKyOIgREghOT2pkqAoZGQ3WMg21JbDe3hKzXzUEooBEO6VSXhVBy3W+mrJlMO0ZdMxFC8Nprr1FVVTXEGUPzkY98hI985CMFx84++2weffTRfo9dsGABDz30UL/jDQ0NPPDAA4dsg8TDdrwWTUWBgKF6C5pMh4CujdtAqq5563mHm37vjaIokza0Z2gqAV09pOn6kVBREmRGZQRBoXJxXhBxRmWEtniW6rIQyYxNsMj3lxwKlaUhymNBEhlPg6x3ulFRFGbXlPi6aOGgTtZyETI0mVaMumYCUF9fL3fAHyZ4aS5voC9gaGRNF8txxzXNpSjesN5EXazHiq57r30i2oIBPyITiH5RVjSkc9aJdZxzcj3gdbMV6/s0VlRVoTQSGHCnfe/IMRzQ6U6ZONKZTCtG5EzuuOMOALq6utA0jVhsbJ1QkuLBdlx0TUXBk5xP5jSRxkN+vjczqqLj9rPGG031dtWP98Biz8/PraxF6edMPDVkjZKoQdZ0iI5AcuZwJxzUaYtnsOWgybRiRL89O3bs4JJLLuGcc87hjDPO4Morr6SxsXGibZNMAj1pLm89bCrj5bSDhtZvMdbhTF1lhMA463LlUVUFx3UHdNDeIKZ3B27Zri9ZfyQTDno1E8eWkcl0YkS/PTfffDOXXnopL7/8Mi+//LKvkyWZ/vSkubwLW15gb6CVvYczkdDErcfVVAV1kBpQIDcs6boCwfRchjXeRII6puX62l2S6cGInEk6neayyy7DMAwCgQBXXXVVwfyHZHoihMBxewrwQUPz2zGDgfErwB/pqKqCmtOh6kuBIKIQBMZ50+N0JBLSsRwX05ILsqYTI3Ims2fP5m9/+5v/923btjFr1qwJM0oyOdiO5zk0zWtR7T1nEZrGirXFSDCgDaotVhL26iW6Pj5t0NOdfHSWzkpnMp0YUUzd1NTEVVddxfHHH4+u62zdupXq6mpWr14NwGOPPTahRkomBjsnD967AJ9HOpPxJRLSB235DQV1spZDRS8JmSOZfN0oKRdkTStG5Eyuuuoq7r//fm666Sa2b9/O9u3b+dKXvlQgwiiZfuR3TeQL8L33n0dl7n5caagevAMyaKgYmlowyHckk28TTo2DlI9k8hjRFePXv/41l19+OWeccQannnoq2WyWDRs28KMf/Wii7ZNMIHZuj0S+JTZo9HwdIvLCNmkYukY0bMhoMEe+tpSRaa5pxYgStB0dHb6MSTAY5JprrqGlpWVCDZNMPH6aKzcH0TvNFQ3JC9tkUl8VPeRNkocbeXHHtOnIBVnTiBE5E8dxCvaFtLa2yg/5MMDKFeB13XMm+TtjTVUI6PLCNpnEwsa4KQ5Md/JpLtOU+lzTiRFdMa655houvvhizj33XBRF4fnnn5dyKocBTq+aCfQIHY7nyl6JZLTku7kyprdt8TBVlznsGJEz+chHPsLChQt54YUX0DSNT37ykxx33HETbZtkgrF6dXMB/oyDoWuHrT6UpPjJb5nMWDLNNZ0YcS5jwYIFLFiwYCJtkUwy+QK8ruYl2PNOZWRy8RLJRKBrKrqmkDVdmeaaRsjbzyMYu0/NJF+Az+9Dl0imilDAm72RWo/TB+lMjmAGS3N5goRTZpZEIhdkTUOkMzmCyae5jD6bBg1dlbpckiklvyBLOpPpg3QmRzD5OZO8HlQ+zTWei7EkkkMhHNQxLUduW5xGSGdyBJNPcwV8Z5L7v35kyc9Lio9wUMO0HbltcRoxpc7km9/8JjfddBMAW7du5ZJLLvF3pdi2t1ejsbGRtWvXsnz5cq677jqSySQA8Xica6+9lgsvvJC1a9fKifxDIF+AD/jzJRqGruYG6KbSMsmRTji300RuW5w+TNkl4y9/+Qu//vWv/b/feOON3HLLLTz55JMIIVi/fj0At912G1dccQWbNm1i4cKF3HvvvQB85zvfYfHixTzxxBNceuml3H777VPyOqYzdp8CvK6pfPFjizjlmGoZmUimlEhQx7QduhLmVJsiGSFT4kw6Ozu5++67+exnPwvA/v37yWQyLFq0CIA1a9awadMmLMti8+bNLFu2rOA4wLPPPutL4K9atYrnnnsOy5Iqo6Mh70xCvTS56irChORiLMkUEwnq2I4gkbb9VdKS4mZKnMmtt97KDTfcQGlpKQDNzc0FcvY1NTU0NTXR0dFBLBZDz+lE5Y/3PUfXdWKxGO3t7ZP8SqY3fjdXL2eiqIrf3SWRTBV5fS7XdejozkyxNZKRMOlqfr/85S+pr6/n7LPP5uGHHwYYUDJBUZRBjw+GOopEf1XV4PslRkJNTcmYzp8IRmtTIGigKFBfV0p1eRiAUMrEVdVJe33F+D5CcdpVjDbBxNhVl/uZZWURHFWjvCKCMY4rjYvxvSxGm0bDpDuTjRs30tLSwkUXXURXVxepVApFUQp2yre0tFBbW0tlZSWJRALHcdA0zT8OUFtbS2trKzNmzMC2bRKJBOXl5SO2o60tgXuInSI1NSW0tHQf0rkTxaHY1NWdQVMVOjqSCMtLJaQyFonuDC0tE//VKMb3EYrTrmK0CSbOLjf3fWxpSxALB9i+W6GqNDQuP7sY38titKkvqqoMeRM+6fmM//qv/+Lxxx/nkUce4X/9r//F+9//fu644w6CwSAvvfQSABs2bGDJkiUYhsHixYvZuHFjwXGApUuXsmHDBsBzUIsXL8Yw5EKn0WDbLppaOKBo6Bol4cAUWiWR9KzuzWQdwkGN5o5U0Q0wpjIWLV3pqTajaCia5Phdd93FHXfcwYUXXkg6nfaXca1bt47169ezYsUKtmzZwhe+8AUArr/+el5++WVWrlzJgw8+yK233jqF1k9PLMfNreztOWboKmVyF7lkisnL0KdNG11TsW236DYvprI2HfHsVJtRNEzpBqQ1a9awZs0awFMlfuihh/o9pqGhgQceeKDf8fLycu6///4Jt/FwxrJdNE2RbcCSoqOmPISuKbyzP86JR1UCChnT9gvzxUAqY5PJ2jiuF+Ef6ch34AjGzkUmEkmxEQkZLJxXyWs72khnbQxdIVlkLcLJjIVAkDV7BitdV7CnqfuI3MMinckRjBeZqFIhWFKUnHliHbYj+PvbrRi6SjJTPHNktuNi2wJNVcmYPU4ubdrEk6Y/w3UkIZ3JEYzjCnRVkQOKkqJkVk2M2bUxNm9tRlEULMstmou0abmgeLuAkukeJ5dIWaRMB9MuDjsnE+lMjmAsW6a5JMWLpimcdlw1XUmTt/Z2IgDTKo4ivGU7ILwdQN29nElnwiSgKUVj52QinckRjO3k01zSoUiKD11VmVdfSlk0wItvNKEqkDEn9yI9WDtyMmOh6wqqqmA7LpbtYFoOWdshYGiks8VV35kMpDM5gsm3BiN9iaQI0XUVRYHTjq9hT1OCjGmPqgifSFscbE8e0nMLIWhsTbKzMT5gai2VtX2BVEVRyFouqYwFQmDoinQmkiML2xa51uCptkQi6Y+qKAgB8+s9mZEDbSmS6ZGrCHd2ZznQmhr1hV0IwYG2FC1daTKmw84DcS+tlcMVglTG9jvNVAVSWYt4yiJgqOiaSsZ0jriOLulMjmB6WoOlN5EUH/nNn3WVEQxdZV9LAtMeWRHeFYJ4MkswoHKgLTmqC/vBds+RlEYMomEd2xbsaOz2HYpluTR3pnns+d1sfrMZQ1fpTlp0JU2Chqe47QqKpllgspDO5AjGcyYqsmQiKUaCAY2yaBDLdpldG2NPUwKFkRW3M1kH2xVEQgbdKYtEemRtxamMRXOH50jyXY7hkIbjuDS2phBCYNoO+5q99Nnug90Yukoqa4MQPZ2RwmtwOZKQzuQIJh+ZyAK8pFipLg9h2S5z6mI0daTJmPaIivCJjOlPpUdCGvtbkzgj2NrY1J4maKj92uUjYZ3OZJaupEnatNnfkgBgb3MS1xU4jkDpky8+0tqDpTM5grEdF12TBXhJ8RIJ6oQCOg3VEQCaOlIjKsJ3dpvedxtPvNR2XLbvjw9ZP0llbOJpk1BwYMmWaEhnf0uSeMJkf2uSkoiB7bjsb00SCWlEgjp/f7uVf+xoQ9MU0kU2sT/RSGdyBGM7Ak1TpS+RFC2KolBbEaaiJIymKjS2JelOmUNGGabl0NSZ5tvrX+GVd7zVFrGwgRCCt/d1cqA1MWDLb1NHiqA++CVRz/2u7DwQJ2M6nLNwBgC7DnYTMDQsx2XTi3t49u+NGLripb6OIKQzOYKxfdVg6U4kxUtpJEDIUJlZHWFvk5dW2tecLHAIGdP2C96pjMWbu9qxbJcn/7rXl2EJBjRiYYN9zQn2NicKHFIqY9OdGjwqyRMJ6zR3eLLzC+ZWUFcRZvdBbw/J6zu95+zozpLOOmRM+4jq6JLO5AhFCIHtCD8VIJEUK6qqUFkWor4qyoG2FAFDpTORpbk9he24NLYm2bank7f2dNLWlaE9nmXr7g5qykOYlsvvtuwr+FkVJSHiCZOdjd3EUyaNrQl2HegikItKLNvltR3tPPrnXew8EO9nz77WJBUlQcqiAY6aUcLe5iS24/L3ba1+B9reZm/5nu1IZyI5zHFyWyaldLZkOlAaCTKzOoorBPtakpRGDZo60ry1t5P2eIaSqEEoqLK/NcHWPR10JkzOWTiDsxfW8co7bexuKtxiWBI1sGyX3Qe76Ux4EUkoqPPnfxzg2794hYef28Gr29v4f09t44XXD/oRhhCeKvDcOm/j4FH1JdiOy8tvt7K/Ncl7T5mBoavsbU4ASsF8yuGOvJIcoeTbFqU2l2Q6EApqzK6NoSiwp6kbRVEoiRgEDZVYro1XU1VKowF2Hohj6ConzK3g3FPqKYsGePzPu+lMFC6yCoc0SiIGkZCOqiq8ubuDp1/az+y6GFctO44vXXYqx88u56nN+/j1czvJmg7NHWnSWYe5M7xByjl13v9/99I+VFXhXcdWM6smyp6c8zqS2oOlMzlCyeeXNU1+BSTFj6oo1FdGqK+K8LdtrSQzFqqq+JImeWzb5fWdHZwwt4KAoREwNFa/5yi6Uyb3P/I6f9/WMmAdoytp8uifd1FfFeGj5x3NvPpSQgGdS887mvNOa+D1Xe3c/8jrvPB6E4DvTMJBnRmVYUzLZcGcciIhg9m1Xhuz7bh0j2Jif7ojryRHKPlcroxMJNOFkmiA8941i3TW5tE/7RrQKWzb10nWcjjl6Cr/2PyZpXz2opOYWRXlsed3c++vXmVfbk4EvIVWD/9hB64ruGTp/AIHpSgK555SzzUXLkBVFV7Z3kZZNEB5LIhlO6Qztu9Y3nVcNeBFK0JAazxNW1eG7tSR4VCKZwemZFKxcpGJLp2JZJqQjwLOXzyLJ/+6l79ubeakeZW8ubuDPc0JUhmb5o40JRGDo2aUYNkOtu2ls8pLgly17Di2vNXCc68c4Ke/eZNjGkoRAhrbPP2ui8+dR2VpaMDnnl0b4zMfOpHnXjlAVZn3mHTWQQFOX1BLKKAzv74UgFk1URTFG2g8akYpe5oTHDerHGOItuPDgSlxJt///vd54oknAFi6dCn/+q//yvPPP88dd9xBNpvlwgsv5IYbbgBg69atfO1rXyORSLB48WJuu+02dF2nsbGRG2+8kba2NubNm8ddd91FNBqdipczLbFtmeaSTC9URaGiJMjCeZXsPBDnqc17eWrzXoSA0ohBLBJgRmWYU4+pRlUVEmkHXVMxLU8WXlEUTl9Qy5LTZvPkX3ay5c0WoiGdBXPKmT+zlJPmVQ75/AFD4/zFswAvmtFUhaChExQuSxfNLHhcfVWEPU3dGHoDlu2yvyXB3Bklh3Ub/qQ7k+eff54//elP/PrXv0ZRFD71qU/x+OOPc9ddd/HAAw9QX1/PZz7zGf7whz+wdOlSbrzxRr7xjW+waNEivvKVr7B+/XquuOIKbrvtNq644gpWrlzJ//k//4d7772XG2+8cbJfzrQlXzPR9cP3yy05/CiNBmnryvCh9xzF48/vpqYizIlHVVBbHi64UGcth2jIYEZVhHf2d6FrKmouCg8FdJacOpMlp84c7GmGJW3aVJWGCAY09jUnCORagvPMri1hy5vN2I5LJKTTlTTp6M4OGvkcDkz6bWlNTQ033XQTgUAAwzA4+uij2bVrF3PnzmX27Nnous7q1avZtGkT+/fvJ5PJsGjRIgDWrFnDpk2bsCyLzZs3s2zZsoLjkuGJJ7Psbe72di9AvwKmRFLMRII6mqYSMDQ++v5jOO9dDdRVRPrd8WdNh7rKCNGQQX1lhER6fKfRHRfKYkFvsr7XcdcVxBMmc2qjOK63EwUgFvakWLKTsIFxpKKW482kX0mOPfZY3zns2rWLjRs3oigKNTU1/mNqa2tpamqiubm54HhNTQ1NTU10dHQQi8XQdb3guGRwhBC0dKbZecAb1Np5wGtdlDUTyXRCVRVmVkVIDeEcTMshEtSJhrzrQ3V5mGhIJ5Ea2UVWCOH/NxCW7RLUNUIBDUPXiIZ0X8k4kfZ2msyoiqBrCk+8sIdUxkJTVXRdYX9LYkKn4i3bobE1OSXy91NWgH/77bf5zGc+w5e//GV0XWfnzp0F/64oyoBv+lDHR0NVVWx0BvehpqZkTOdPBIPZlM7aNLUlSdouc2aWe/nkrPdlq6mOTelrKcb3EYrTrmK0CSbfrurqGK6mYZou4VDhJcxxvT0mx82poCwW9I9XVcXY09RNW2caxxVUlA9cX42nTE9KXlVwXYHrCsI5sUn/Mckss+tKqKnwxCfRdfYcjBMK6IQjQRpqY2zf18UnVy/kp4+9zs9+9w7/vOYUKsqjdCSyKIbec24vxuN97OzOYgQzVFREh5WGGW+mxJm89NJL/K//9b/4yle+wsqVK/nrX/9Ka2ur/+/Nzc3U1tZSV1dXcLylpYXa2loqKytJJBI4joOmaf7x0dDW5skdHAo1NSW0tHT3O541HRIZyy8UTiYD2ZTK2BxsT5JIe3dGkZBGV9y7g+qKe/pCdtYe8LVMBoO9j1NNMdpVjDbB1NkV1hT2tXUTCxtkLQfLFiiApimURQNkU1la+sx4xHSFbEAlkbbo6kr5x1UVArpGOmNTFgvSUBP107+pjMW+liRNpoOmguuCosCMsiAtLd7vUtZy6OxKgYCjZ5VhpkyS3WlqSgN87APH8POn3+ae9S/ziZUL0DSFf2xr5rhZ5QQDPXWW/Pto2Q4H2lLMqo0d0mqIvc3dtLanOdgcIBoyDuGdHRxVVYa8CZ90Z3LgwAE+97nPcffdd3P22WcDcOqpp7Jz5052797NrFmzePzxx7nkkktoaGggGAzy0ksv8e53v5sNGzawZMkSDMNg8eLFbNy4kdWrV/vHpwJvo5tJc0earGkDCq4QGLpKLDy+HyZAd8qksTVFSUQnFg4QCekD1j2ylrduVNcUSqOBfv+el1ORBXjJdCQc1KktD9PUkaaiJEhlaYhQQBuyBqgoCtVlYcorojQ3xxF4KatEyqQraVFfHaG6rLCQHwkZHNNQRmcii+26BHSNgK5i6D2OIGhoRILeJH3+Al5ZFqK1K8P8maV89P3H8OBv3+aZv+1n+Zlz0DWXfS0J5s0sLXAYQgj2tyTp6M5SVxEpcDYjwRWCrqRJQFenRBNs0p3JT37yE7LZLHfeead/7LLLLuPOO+/k85//PNlslqVLl7J8+XIA7rrrLr72ta+RTCY58cQTufrqqwFYt24dN910E/fddx/19fV8+9vfnhT7XVfQlcjSlchi2i6tXWks2yUc1CjJXbQt22V3UzfHNpT16/IYC5btsKcpga4rdCVNWruyaJo3GVzZ647BcV32NHWjqfhfSMt22PxmC3957SBZy/X3vgf08bNPIplMaisjVJWFRz2/Yeiq/3sZNDwl4RlVgz9eVZVhu7Bm18YKbszKokFfXfiYhjLOOKGWv25tZsHcCo6aUUI8adHelaG6POyf0xbPEE+Z6LpK1nZG7UwyWQfhCjRNwZqEQn9fFHEkaST34lDTXMmMRXO3SSKeQVEhZGjoA3yZUxmboKExr77Ub0kcC0IIdh2Mk8k6BXlix3FJZhxqqqMojkMsHKCzO0NnwiQWMXBdwd+2eYNaibTF/Jml1FWEc1vgBB8779iCL/RkIlM3I6cYbYLitWsoJsvmt/d1AgJD1zAthx88+gYAn/nQieial26bV1+KrqmUlUfY8lojsbDhtx3PqBzd3FxzZ4pf/2EHKLDq7KNoqBlbXbgvRZfmOhzQVIWS6NAprEhI99JfnalRfyl6k5eK70xkiSdM9rUmqa0IU5NzAJqmUhpVCWgaBzuStHSkAYVoWGPb3k5+t2UfrV0Z5tTFuOR985lb11PkS6SsAR2hRCIZO9VlIfY0Jbzfz5xG2H9veovfvbSPFWfNJRjQ2JWTuC/LbYb89XM7iIYNLsgNR46Gfc1JNr/VQixscMHiI6ib63BECEEibRENGaiqQixs0NyRpjQSIDJAMcwVAgQFkYsQAtNySWQsOrszpLJeuOq6LpvfbOb5nNDcCXMreO8p9dRXeV0huq76+VohBL97aR9/ea2JqtIgH3v/0Rw3u7xfx5sQo++Ck0gkI6MsGqSm3KGlM00srHPUjBLOOqmOF15vorY8zOIFtf7+k1gkwM+fepM393QSNDTOPmkGrhAjLsJbtsOWN5u9OZekSWIIgUkhxIT83ktnMg7saermH9vbeWd/F11Jk7qKMB88Y3ZOeVRjT3OCY2eVFewOyZoOe5q7yWRtVE3F0FRsx/UK40IACsGASiysoygKf3ylkedfb+K046qJhAw2b21m6+4OZlSGWTi/inNOaQC8L8ozf9vPX15rYvHxNSw7c/YQO0uEXNkrkUwQqqowszpKOOhNyYeCGue/exatXRmeeHEPFSVBjm4ow3UFP//tW7y5p5P5M0vZ0RinrSuDaTkFLclDEU9ZvLqjDUNXsWyXtnhmQGckhGBPc4I5tbFxdyjSmYyBeNLkt1v28frOdgK6yryZpZx2XDV/29bKA09uY8Hcci5+7zxs26WpPUVlaQhdU0llLK+QrimURAN+P3sooBVEKa7rLQJ6bUcbm99s4ZSjq1h59lwUReGck+p4+Z02XtvZzu+27ON3W/ZRVxGmojTIm7s7effxNVx41pyCL4xlO2TMXPjrhSXjUs+RSCSDU1ESImhobN/fRSSkcMnS+fzXxjd56NkdVJWFaO5IYTuC971rJqceXcV3H/oH+1oSmJZLqH8jZj9sx+WPrzSSyti8710zefbvjXR0Z3EcF7VPg41pu2RNB+92dXyRzmQUCCH4y2sH2bqnk47uDPtbvH3US06t5z0nz/DbBc86aQYvvNHEs3/fzy9/v52Pvf9o2uJZ2uJZEF56Kz+d25nIkkhZZCwHxxHYjktLZ5qD7Sn2NSdJZW1URWHRsdWsyjkSgFBQ56yT6jjrpDra4xn2tKR4eVszb+3p5LTjqlnRy5FYtkMq600FN1RH/TScpkpnIpFMBpGQwey6EnYfjFMSCXD5B47h4ed2oqoK7z6+lpPmV9NQFUJRvDmZxrYkqaw1YFt/b2zH2xa55c1mKkuCnHFCre9MbEdg9LnCezM5E1NPkc5klPx2yz7SWZuyWICF8yp57yn1/QYUDV3l3FPqiYV0Hnt+N48+v5sPnzvPv7jvbU6w6cU9vLW3c8DnUBSveHfMrDKOaSjjmIZSf5pVCIErREHqqrI0xNFzqlh0dCWW7Sml5p8rlbZB8doTx3uISSKRjJzyWBCzOsqB1hSl0QAfX7HA/7eK8igdnUkc12VmtbepsTtlDtm847qCfc0Jtu/vorEtxQdPn03Q8ORdPGfS32mkM7Y/YzbeSGcyChRF4dZrFtOWtHCt4YXj3nVcDcmMzTN/209bVwZDV8nkVn+GgxpnL6yjujRELGIQCuhouc1x5bHAgPMp6ayNZbvomort2Oia2i81lo+OHNclmbIpjQZoqIkd9rsUJJLpQE1ZGMvyahp9ow7LdklnbGbXxti6u4MDbWmOzskf9cUVgn0tCXYdjPPY87soiRgsOraKZNqmvCRIR3d2wAgkmRlfwcveSGcyzgjh1T9UVUFRFN5z8gwA3t7XhaJ4HV7vOraadx1bPaKBRiEEGdPBtF3Ko0Hq6sMEDY101qYzkfVyoy4YAZOs5aCpCpbtYjuChtoYlSVB2bElkRQJiqJQXx3NTd5bxCIGQgiSGYus6TC/oQzD0Hhq816vbmL3L8K7QtDYkuCd/V38+rkdKIrC1cuOR1MVhAp1FWG27u7op1DsCkEqazFRlwPpTMaJjGljWl53lGGoOJbrD0Wes3AG7z2lfsDz8s7Ha+ISiFxNxckV5RWgvCRIVWmYSK9hxUjIIBIymFEZJW3aGMEAVtbCtF2Chsb8+tioJ2glEsnEoyoKs2pj7DoQpythesOAlTEqZ5URCujMm1FCJKizvyVBtlcRPn9j2RbP8PquDh5/fpfnSJYfT2VpkHjS4uiGMmZWRfnbtlY6u7PUV/WkySzLxRUCZYJ6OKUzGSNZ0yFjOpRGAsysChVoZbmuoDWe5mBbCkNX0TUFITxdLNv2HI2qKbn0loKuaeiqgqYpBAxP3joc0IaMYFRVIRoyqKmJEVCOSDEDiWTaoWsqc2eUkjZtIkGd+hll/lR+KKgzqzbK/tYkB1oTtMd1FCCVtXEcwRu7vQ7O0miAy88/luqyEN1Ji+qyELGwwcxqz4E0daQ54aie58zaDv/Y3o4QYtitkof0msb9Jx4B2I5LMmkBgnDQ4JhZsQGL26qqUFseoSQcoKkj5UUaikJUVymJBogEBxZplEgkhz+GrmLoA3drHTe7nG17u0hkbFRVwXYF+1q6eW1HB6/tbGdefQkfed/RBA2NeMKkLBakrtIbYG6o8ZxJc2e6YEAxnjD546uNHNNQNiGvRzqTURIKaCyYW0lnR9Lb+Karw9YkwkGdo2aUTpKFEolkurPwqEoef343//eJNwkYGqmMTdZyCBoa5yycwXmnzcR1Bd0pT+24ppfa8YzKKIoC7fEMjivQNe/437a1YFouJ8ytmBCbpTMZJZqqUhYLYg4hVyCRSCRjYf6sMs46sZZU1kHkVlocO7uco2d6wpDJtI2iwNENZf1WXRi6SnksSEfcaw/WNRVXCF5+p5XSqMHs2vEVgMwjnYlEIpEUGbqqsuKso+hOm4RzM2auK8haDom0TUVJkIbq6KBp8tqKMG1dGX+vSUtHmt1N3bzn5BkT1t0pnYlEIpEUIWWxAO3xDLZjgad+RFkswOzaENGQPqRTmFEZYfv+OLbtAAZ/ef0gQsApR1dPmL3SmUgkEkkRUhIJsGBuBQJAeOmrkcof1VdFsB2Xpo40KAp/ee0g9VURr/MrZU2IvdKZSCQSSZFyqJtaZ+bmS/62rYWs5dDSlWHFWXMOaSHgSJHORCKRSA4zZuT2HP12yz4AaspDnDSvgnjKYmZ1ZMR7UkaDdCYSiURymFFVGmL5mbPJWi4L5pRTVRqkO2UxsypKbXlkQp5zWk/MPfbYY6xYsYILLriAn/3sZ1NtjkQikRQFiqJw4ZlzOXl+FaGATiJtU1cZ8dd9TwTTNjJpamri7rvv5uGHHyYQCHDZZZdx5plncswxx0y1aRKJRDLlRENGbsOrp0Y+0Wob0zYyef755znrrLMoLy8nEomwbNkyNm3aNNVmSSQSSVGQ1+0LBSZHtmnaRibNzc3U1NT4f6+treXVV18d8flVVWObAq2pKRnT+RNBMdo0HMVqczHaVYw2QfHaNRTFaHMx2jQapq0zEaJ/i9toJjvb2hKH3CZXU1PiK3wWC8Vo03AUq83FaFcx2gTFa9dQFKPNxWhTX1RVGfImfNqmuerq6mhtbfX/3tzcTG1t7RRaJJFIJEcu09aZnHPOOfzlL3+hvb2ddDrNU089xZIlS6baLIlEIjkimbZprrq6Om644QauvvpqLMviIx/5CKeccspUmyWRSCRHJNPWmQCsXr2a1atXT7UZEolEcsQzrZ3JWBipYNpEnT8RFKNNw1GsNhejXcVoExSvXUNRjDYXo029Gc4+RQzUFiWRSCQSySiYtgV4iUQikRQP0plIJBKJZMxIZyKRSCSSMSOdiUQikUjGjHQmEolEIhkz0plIJBKJZMxIZyKRSCSSMSOdiUQikUjGjHQmEolEIhkzh5WcSiKR4LLLLuP+++9n1qxZPPzww/z4xz9G0zTOPPNMbrrpJrq6uvjEJz7hn9Pd3U1HRwd///vficfjfOlLX2Lv3r1UVlbyne98p2ABV57GxkZuvPFG2tramDdvHnfddRfRaNT/94ceeogtW7Zw5513DmjT/fffT3NzM4ZhcNppp/G1r32N/+//+//885uamojH47zxxhsTYtNQfPe738W2bX7/+99z//33c+DAAT796U/jOA6KojBr1iweffTRCX0ft2/fzi233EIymSQUCvFv//ZvnHDCCf3ey5/+9Kd897vfxXEc6urqePjhh7Ft27ers7OTeDwOMGF2zZ49e9DvXDAY5Nxzz+XKK6/kE5/4BKlUin379qFpGrZt8+EPf5ibb755TDa98847fO1rXyOVSlFWVsadd95JQ0PDlL9Xg9lVzN+7PAcPHuRDH/oQDz/8MOXl5f0+3x/+8Ic0NzejaRrHHnss//Zv/8aNN97on9/a2kp7eztbt26dEJtmzZo16PvY9/e8sbGRlStXMmfOHACqq6v5yU9+Muj5Y0IcJrz88sti1apV4qSTThJ79+4V27dvF+eee65oamoSQgixbt068dOf/rTgHMdxxJVXXikeffRRIYQQt912m/jBD34ghBDi17/+tbj++usHfK5rr71WPP7440IIIb7//e+Lb33rW0IIITKZjPiP//gPsWjRIvHlL395UJv+6Z/+STz++ONi3bp14hOf+ETB+d/61rfEggULxBVXXDEhNg1GPB4XN998s1i4cKE4++yzfZu/9a1vidNOO21S38fLLrtMPPPMM0IIIZ5//nmxevXqAd/LhQsXigcffFAIIcQll1wirrrqqoLXfOqpp4pzzjlnwuw6//zzB/x8d+zYIW6++WZxwgkniI9//OP+z/3JT34i7rvvvnF9r6688krxhz/8QQghxIMPPij+5V/+pSjeq4HsGohi+t7lf+YnPvEJsWjRIvHUU08N+PnedNNN4gc/+IFYt26duOGGG/zncRxH/PjHPxYnnniiWL58+YTYtHfv3gHPH+z3fNOmTeKWW24Z8Jzx5rBJc61fv55169b5C7LeeustFi1a5P/9vPPO43e/+13BOb/61a8Ih8O+8vCzzz7r/3nVqlU899xzWJZVcI5lWWzevJlly5YBsGbNGn/3/ObNm3Fd179LGcimU089lVdffZVly5Zx3nnnEY/HC85/8803Ofroo5k9e/aE2DQYTz/9NEcddRTz589n6dKlvs0vvfQSgUCAa6+9ls9+9rOceuqpE/4+Xnrppf5umuOPP54DBw70ey/feOMNHMfh0ksvBWDt2rX8/e9/L3jN559/PpqmTahdA33nXnnlFY466iiWLVvGzp07/Z/9j3/8gw0bNvD666/zm9/8hgMHDozZpv/6r/9iyZIluK5LY2MjpaWlRfFeDWTXQBTT9w7gxz/+Meeccw4VFRX85je/GfDzffHFF1m9ejXnnXceBw8e9J9n+/btPP300xx33HFUV1dPiE2DMdjv+T/+8Q+2bdvGmjVruPrqq3nrrbcG/Rlj5bBxJrfffjuLFy/2/75gwQJeeeUVDhw4gOM4bNq0qWAzo+M43HfffXzxi1/0j/XeK6/rOrFYjPb29oLn6ejoIBaLoetehrCmpoampiYA3vve9/Kv//qvhEKhQW16+eWXCYfDKIrCpk2b6Orq8s8/++yz2blzJytWrJgwmwbj4osv5tprr+X8889n5syZ/vH6+npc1+W+++7j3HPP5Vvf+taEv49r1qxB0zQAvve973H++ef3ey9nzJiBEIKWlhYcx+HFF1/ENE3/NX/xi1/kT3/6EyeddNKE2bV69eoBv3Nnnnkmn/zkJ9m5cyfJZNL/92g0Sjwe53/+539YunQpN9xww5ht0nWdeDzOkiVL+J//+R8++tGPFsV7NZBdA1FM37vXXnuNF198kY9//OMAfOlLXxrw821qaqKyspJNmzbR1tbmP8/8+fM5ePAga9eunTCbBmOw3/NgMMjFF1/Mww8/zCc/+Uk+97nP+Z/9eHPYOJO+zJs3jy9+8Ytcd911rF27luOPPx7DMPx//+Mf/8i8efM4/vjjh/w5qlr4Fokx7J6fN28e1157LZ2dnQU25c/P2zRjxoxJs2k47r77br761a9y3XXX8dhjj5FMJguef6LeRyEE3/zmN3nllVf4yle+0u+xs2fPJhaL+Z/vcccdV3D+H//4R6qrqykrK5s0u/p+56qqqnznA3D++edzwgkncPzxx3P55ZfzzjvvDPh8o7WptLSUP/3pT3z729/muuuuw3GcgsdO1Xs1nF1DMdnfu3Q6zde//nX+9//+3/3OyZP/fG3b5uqrry64pqiq6tuUzypMhk3D8fnPf57LLrsMgKVLlxKJRNixY8ch/azhOKwK8L3JZrOccsopbNiwAYCnnnqq4EP+3e9+VxABANTW1tLa2sqMGTOwbZtEIkF5eTkXXXSR/5iHHnqIRCKB4zhomkZLS8uwu+ebmpr49Kc/ja7r3H333QSDQX72s5/x9NNPU1dXRyaTmXSbep//yCOPDPgYIQT33HMPK1as8N/HU0891S/mTZTNtm3z5S9/maamJv77v/+bkpIS3+b8e/nLX/4Sy7L41a9+haZp/OIXvyAYDBbYdcopp+C67oTb1fvzveeee/z36gtf+IJ/gXZdl3vuucf/xR4vmzZu3MiFF16IoigsWbKETCZDV1cXH//4x6f0vRrKrjzF9L3bsmULra2tXHfddYAXUVx77bV8//vfp7W11f98161bR0NDA9/97nd59dVXmTVrFn/9618pLy+fVJvuuOMOmpubAfjhD39IXV3dgO/lAw88wKpVq/wUmRDCj4DGm8PWmaRSKf7pn/6J3/zmNwQCAR544IGCUPvll1/m05/+dME5S5cuZcOGDXz2s59l48aNLF68GMMw+n3pFy9ezMaNG1m9ejUbNmwYdvd8XV0dP/rRj4hGo1x88cUsWrSIRx99lIcffpjKykr//LxNL7300oTbNNgvcm8UReG3v/0t/+///T+efvppNm3aRCAQYNWqVf5jJuJ9/OY3v0kikeCnP/0pgUCgwOb3v//9/OhHP8KyLFzX5eGHH+aiiy7iBz/4AaeddlqBXR/60IcK7sImyq6+n2/+O/fKK6/4qSNVVXn77bdJpVIAbNiwwb9AjsWmn/70p+i6zgc/+EFeeOEFKioqqKysnPL3aii7hmMqvnfnnnsuzzzzjP+Y97///fzwhz9k1qxZVFdXF3y+5557Lr/61a944YUXOOaYY/znydt08ODBCbfpRz/60bDvI3i1lEwmw6c//Wn++te/4rou8+fPH9G5o2ZSyvyTyHnnned3PKxfv16sWLFCfPCDHxTf+973Ch53yimniEwmU3Cso6NDfOYznxErVqwQH/vYxwbtnNi3b5+48sorxYUXXig+8YlPiM7OzoJ//9WvflXQUdHXpgsuuECcfPLJ4swzzyw4P29T7/MnyqbB+N73vie+973v+TZv27ZNnH/++WLhwoXi5JNPFrfffnvB48f7fWxraxMnnHCCuOCCC8SHPvQh/7+B3ssf/vCH4tRTTxUnnXSS+MAHPlDwmk855RTx85//vOA1T5Rdg33n1q5dW/C9O/nkk8Wll14qVqxYIa688krR2Ng45s/37bffFpdddpn40Ic+JNauXSu2bds25e/VcHYNxFR/7/rS+73r+/kuW7ZMnHrqqeLMM88seJ68TS+88IK48sorJ9Smwej7e37w4EFxzTXXiJUrV4o1a9aIrVu3Dnn+WJCbFiUSiUQyZg7bArxEIpFIJg/pTCQSiUQyZqQzkUgkEsmYkc5EIpFIJGNGOhOJRCKRjBnpTCSSEfCJT3yC9vZ2Pv3pT/POO+9M6HPt3buXz3/+8xP6HBLJeHPYDi1KJOPJn//8Z4ARD4uNhcbGxgKBSIlkOiDnTCSSYbj55pt5+OGHOe6443jnnXdYv349qVSKb3/729TW1vL2228TDof5/Oc/zwMPPMDOnTv54Ac/6Gt3PfPMM9x3331YlkUoFOLLX/4y73rXu9i+fTtf/epXMU0TIQQf+chHuOyyy1i+fDlNTU2cfvrp/OQnP+H+++/nd7/7HdlslnQ6zZe//GUuuOAC7rnnHvbs2cPevXtpbm7mlFNO4T3veQ8bNmxg37593HjjjaxatYp77rmHt99+m9bWVtra2liwYAG33347sVhsit9ZyWHFhI1DSiSHEccdd5xoa2sT5513nnj11VfFCy+8IE444QTx+uuvCyGE+OQnPyk+9rGPiWw2K9ra2sRJJ50kDh48KHbu3ClWrVol2tvbhRBCbNu2TbznPe8RyWRS3Hzzzf6ui+bmZvGFL3xBOI4jXnjhBbFy5UohhDcZfdVVV4l0Oi2EEOLxxx8Xq1atEkIIf2I8Ho+LdDotTj/9dHHHHXcIIYT47W9/Kz74wQ/6j1uyZIloaWkRjuOIf/mXfxF33nnn5L15kiMCmeaSSA6RWbNmceKJJwIwZ84cSkpKCAQCVFZWEo1G6erqYvPmzTQ3N3PNNdf45ymKwp49e7jgggv48pe/zKuvvsrZZ5/N1772tX7qsA0NDXzzm9/kscceY/fu3bzyyisFsvbnnHOOLzhZW1vLueee69vT2dnpP2758uX+jo2PfOQj/Pu//ztf/vKXJ+JtkRyhyAK8RHKI9BZ7BAZUY3Vdl7PPPptHHnnE/2/9+vUce+yxnHfeeTz55JNceOGFbN26ldWrV7Nnz56C819//XUuu+wyEokE73nPe/jUpz41ahuAAil813UPWdJcIhkM+Y2SSEZAfm/7aDnrrLP485//zPbt2wH4wx/+wIc+9CGy2Sxf/OIX2bhxIytXrmTdunXEYjEOHDiApmn+Nr7NmzezcOFCPv7xj3PGGWfw9NNPj2ovSJ6nn36a7u5uXNdl/fr1nHfeeaP+GRLJUMg0l0QyAi644AKuuOKKghTTSDj22GP5+te/zr/8y7/4uyTuu+8+IpEI//zP/8xXv/pVfvGLX6BpGueffz5nnHEG8XgcTdP4yEc+wv33389TTz3FihUrMAyDs88+m66uLhKJxKjsqK6u5tOf/jQdHR2cfvrpfPaznx3V+RLJcMhuLonkMOeee+6ho6ODW2+9dapNkRzGyDSXRCKRSMaMjEwkEolEMmZkZCKRSCSSMSOdiUQikUjGjHQmEolEIhkz0plIJBKJZMxIZyKRSCSSMSOdiUQikUjGzP8PYdH+cwpIPh4AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"cpu_demand\", data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 129, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 129, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEUCAYAAAAlXv26AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABz7klEQVR4nO2deZxcZZX3f3evvar37nRCVrKYkLDviygghgCKjKCgKA6IjjujKOLgOC+jMDq+AvMCKjjiHiUiMIhsg4qyK2EJJCF70vtW6627Pc/7x3Pv7aruqu7qTlenuvv5fgYn3V1V99R27rln+R2BUkrB4XA4nFmLeKgN4HA4HE514Y6ew+FwZjnc0XM4HM4shzt6DofDmeVwR8/hcDizHO7oORwOZ5ZT044+k8lgw4YN2L9//5i327lzJz70oQ/hggsuwMc+9jEkk8lpspDD4XBqn5p19Js3b8YHPvAB7N69e8zbUUrxiU98AldddRUeeOABrFq1Ct///venx0gOh8OZAciH2oBybNy4ETfeeCO+9KUv+b+7//778eMf/xiEEKxevRo33ngjtm/fjlAohNNPPx0AcM011yCVSh0qszkcDqfmEGp9MvYd73gH7r33Xui6jhtvvBE/+tGPoGkavvOd7yAYDGLRokX47W9/i/r6emzZsgXLly/H1772NSQSiUNtOofD4dQENZu6Gclzzz2HPXv24P3vfz8uvPBCPPHEE9i5cyds28bzzz+Pyy+/HA8++CAWLFiAb33rW4faXA6Hw6kZajZ1MxLHcfDud78bN9xwAwAgm83CcRy8/vrrWLhwIY444ggAwIYNG/CZz3zmUJrK4XA4NcWMiehPOOEEPPbYY+jv7welFF//+tfx4x//GEcddRQGBgbw5ptvAgCefPJJrF69+hBby+FwOLXDjInoV65ciU996lO44oorQAjBqlWrcPXVV0PTNPzXf/0XbrjhBui6jtbWVtxyyy2H2lwOh8OpGWq+GMvhcDicg2PGpG44HA6HMzm4o+dwOJxZDnf0HA6HM8up2WLs4GAWhEy8fNDQEEF/f6YKFh0ctWrXWNSizbVoE1CbdtWiTZVQi3bXok0jEUUBdXXhkn+rWUdPCJ2Uo/fuW4vUql1jUYs216JNQG3aVYs2VUIt2l2LNlUKT91wOBzOLIc7eg6Hw5nlcEfP4XA4sxzu6DkcDmeWwx09h8PhzHK4o+dwOJxZDnf0nGnFdggIl1ficKYV7ug500rPoI7BtHGozeBw5hTc0XOmlbzpoHsgN6OHTzicmQZ39JxpxbBsmBZBKsujeg5nuuCOnjNtUEphOxThoIyuQZ3n6jmcaYI7es60YTsUoBSKLMK0HGR061CbxOHMCbij50wbDiGAIAAAAqqEnoHcIbaIw5kbcEfPmTa8iB4AVEWCbjrgmyw5nOrDHT1n2nAIBSAM/4JScD/P4VQf7ug504ZlORAKPnGCIICCe3oOp9pwR8+ZNgzLgSQOR/QUPKLncKYD7ug504ZpkyJHL0Dgjp7DmQa4o+dMG+aoiH74fzkcTvXgjp4zLVBKYdkEYoGjByi4EgKHU32q7uhvvvlmfPnLX672YTg1jkMoKCgEodDRCzyg53Cmgao6+meeeQa//e1vq3kIzgzBcUa0VrpwGQQOp/pUzdEPDQ3hu9/9Lq655ppqHYIzg3AIOdQmcDhzFrlaD/wv//Iv+PznP4/Ozs5J3b+hITLpYzc1RSd932pSq3aNxVTZPJQ2EM9aiIc1/3eiYqChIYJwUDkkNk01tWhXLdpUCbVody3aVClVcfS//vWv0dbWhpNOOgmbNm2a1GP092cmpVne1BRFb296UsesJrVq11hMpc39qTxSSR3Esv3fpXMWevvSyAUqd/S1+jrWol21aFMl1KLdtWjTSERRKBsgV8XRP/zww+jt7cWFF16IZDKJXC6Hf//3f8f1119fjcNxZgCm6YzouGHwFD2HU32q4uh/9KMf+f/etGkTnn/+ee7k5ziG7aCzP4s/be7EB965DAFNBii4qBmHMw3wPnrOtGBaBLu70tjXk8Fzb/SwX/LuSg5nWqhaMdbjoosuwkUXXVTtw3BqHMt2kMyaAIDntnTjxLe1sLwN9/QcTtXhET2n6jiEgFDWeRMNKcibDl54sweAwFM3HM40wB09p+qwYSmKwbSBZe1xLG2P4dnXu2E5Dg/oOZxpgDt6TtWxHQrTIsjmbdRFNZy+bh5yho1XdvRPqoWWw+FMDO7oOVWHUIqUm5+vi2pY0BzBvIYQdnSkuAQChzMNcEfPqToOoUhmDADM0QNAOKjAtgl39BzONMAdPafqEEIxVBDRA4AsiUzRkkvgcDhVhzt6TtWxHQeprAlNkRBQJQCALAmwHR7RczjTAXf0nKpj2xTJrIn6mObr0fsR/Sx29BndmtXPjzNz4I6eU3Vshzn6RGRYudKL6GerH6SUonswB9uZpU+QM6Pgjp5TdUzbQTJj+vl5AJAkEbZDZ217pUMoTMvhOvycmoA7ek7VSWYNOIQWOXpZEuE4BGSWjkw5DoVlER7Rc2oC7uhnCbWcC+5L5gFghKMXQCjg2LVr98FgEwLTJnBm6RULZ2bBHf0sIKNb6E/lD7UZZRko6ejZR8+0Z2dqw3YobIfAspxDbQqHwx39bMAhFFaNOkxCWQ+9IACx8PAmKc/RW/bsdISW5UCWBBjc0XNqAO7oZwGEUNi16ujJcMeNJA5/3GSJtVkaVbQ7nTMPWUrLsBwoijRrr1g4Mwvu6GcBjlO7uWDiyh8kImrR772I3nGq4wgJoejoy8K0Do2jNSwCVRZhjojoc3mbR/mcaYc7+lmATUjNtvE5bkTv5ecNy4HjED+ir1bEa9oOsnkL2bxVlccfD8OyocoiLJsUXVUMZQzkzfKOPqOb0A277N85nMnAHf0swHEoqhQYHzS6YUE3nGFHbzrI5O2CHH2x4amsOSURr2UTgAKDaeOgH2uiUEphOxSiKICCFl1tZfPWmFcxqayJvd2ZWTtfwDk0cEc/C3AIrdmIPpllEXUkyAqxgiAgqMrwRmJHFmOzeWtUumMyGKYDVZGQy9uwp/ksaDsUoNSVexDcxSvsBJA3HdhjvFeGRZDRTQzUcBcVZ+bBHf0swCEUtRoApnPM0YcCMhzCUjbzGsPwAnlrxEAREzo7+OPmDBuyzNJD050KsR0CuJo+APyTsGUTWA4Zc3bAsBzEwio6+3M8l8+ZMrijnwU4hMBxalMgLJVlqZOgJsO2KUIBGeGAjGiIRfgj7Z4qWYScYUORRMiy4C89mS4KUzVeGgcAc/IOhV3m+VHK2mQVWYQkAV39uWmxlzP74Y5+FuClCmrQzyOtuxG9JsNyCMKaAkEQ0FYfAsAkjAvNJoQcdCeOQwgsm0CSRGiKhGTGmFY5ZNshyOkWkhkDoiDAcoYjelEo32lUKJcQCihIZQ0YYxRuOZxK4Y5+FuAQCgioSW33TG7Y0VMKqK4efTTM2i1HRvQOwZg57EooLPCKogCb0Gl1mIbp4KnNHdj4vzsgigJM99h63oYii2Wfn+0QFJ2tBQE549B0DXFmF9zRz3AopXAcVvirxdRNRrcgCIDmOnhVZh85xeu6GSFVTNznczCYdvFjSoKIjD59DtO0CQZTeQykDEiSAMMtOOumwxx9mefHUj7DuX1VFjGUmd60E2d2wh39DIfS4bpfLRZks3kLQU1mHSiUQpWZw1dch2+PcMqE0IOO6E3TQcEQLlRV8GsF04Fp2UhmLTYzYBN/aEs3WW99ueE22yGFNVyoioisbk2ooyqXt3m6hzMK7uhnOIRS/3K/Fnuvs3kbIU2G4xAoigRRZJ7Mc/TM6RWmbiicg/RTXiHWQ5ZE5Axn2toskwWzAFnDhmWzYxPC6gakzGYtY8QJShCYwqduVP6CdA/mDtmQGKd24Y5+hkPc/DwoarIYm3MjelaIlf3f+xG9M1xEpm5B+WBnAnKG7U/eAnDTWpiWdkVCaNGQVka3QCjYNKz7PAWUrqcYllOkBwQAksSmZSvBdgjSWbNmBe44hw7u6Gc4zGEwT1+Lxdic4SAUYK2VwcCwo5dEEYKAonWC3tXJwej2FHbcFCKKmJZI13aIPzsAeHMEFHnT9q9bKIBS5zLm6IWi32mKVHGePpe3YVrE7/LhcDy4o5/hEALmOWqwGEsIhW7YfseNpkhFf5clkTl6eJOjAAQclKMvF81qioj0NPTTe/txPVgPv4Bs3ipy4qVOyqblQJKKHb0sMb2cSq5GhjIGVFXkET1nFNzRz3AIpayAV4N99A4hyJssoocAvxDrIbt7YwtTN4IglM1hV4InkuY4pEi62cvTV1sqwiEEqawJRWY9/F50b5iOr+8DOnoozCEEhMCVTRgBpdDHuRrxjhvSZO7oOaPgjn6GQyhFzh3xr7XUTS5vgxCKoCoBlPp5eQ9FEuAQWpC6Gf7bZJ+LaToQBOD+p3fjJ49u83/Pun4wpnLkVGDZBKmciURERSysIJ0zIYBJPXiSDBCEUc/PdmhhZ2URiiIilRvb0euGA0opJEmYEq0gzuyCO/oZTipj4AcPbsGuznTNadKnciyFEdBkaKrsd9x4yLLodsIMi365/yiZw64E3XQgCcDuzhT29WTQM6j7fxMENrRUTUzbQTpnIRHREA2pSOUsiCK70vAKrQJGd0g5DoFu2Hjq7weQN4ttlCVx3BNUKmtAkkRWeC7x+Jy5DXf0M5yBtOFqvhs1p2DpTcWqsohgQceNhyyJcJxhQbZ0zsJrO/sPasrXsgl000HWdeiv7Oj3/6YqYlH+vBrkDQeprIl4REU0xCJ6SRRBCgqklI6eebAdip0HkvjT5k787LHtRb3wkjh2lE4o6/QJqMOpsVr7LHAOLdzRz3A8LRnTIgc9UTrVePlpTZX8idhCRsoBvLS1B4+9uB/pnDXpiNSwbPQMsSg+GlLw6s5+/7EUWUTOsKvqBJNZE3nT8SP6jG5BloTiDVsCHaV3Y9oOku7r1dmXw88e3+YXYL320HJzAIbpgFAUXTGVm77lzE24o5/hZL1in+1U7XLddtMKE8U7CWmKNCo/DzAZBBbRM7t1N4rNG86kCsueUmT3AFN9PPOodqRzFnZ1pQAU9NNXKU9PKUW/qyOfiKiIhRRQyvr6JUnEgb4sbrvvVRjm6KKwYbpXAmEV73v7EhzozeK+p3YM30BA2RP5qHbKg2xR5cw+KnL0119//ajfffrTn55yYzgTx0tRmBapmqPPGTaGMhOXEMh4OXpVGu44KUDxcvSu2V4eOm85k0rdOIT14Xf262iMB7BmcT0CqoRX3hpO30iC4BevpxrboX5dIh7RfOE2r5C6bd8QBtMGhjIm7BGa9IbFumYSUQ2rFtbhzKPa8daBFLoHh6WKy12JmJYzqo7Lc/ScQkYnTgu48cYb0d3djZdeegkDAwP+723bxs6dO6tuHGd8vCEgw3LK6pwfLITQSXVypF1Bs4AijeoPB9yIvqCV0ou086Y9KUfvLfzo6s/isNYoZFnE6kX1eGVnPwzLYVcWioB01kJjPDjhx6/k+Cl3uCkRVuFlUtJZE2gMo7MvC4A9v5HvlWk7SGZNLGuPAQCOXt6EP77cgZfe7MX6kxYW6dqPRDecUZPA071Vi1PbjOnoL774Ymzfvh1bt27Fu971Lv/3kiThqKOOGvfBv/e97+EPf/gDBEHAxRdfjI9+9KMHbzGniJwb0RuWA1Kl3LNpT04nJqNbCKgSBEGAWKI/XHYFvjz3ZVjuczEd0EmctBxCkdMtpHIW2hqY3v3aZQ14aVsv3twziHXLGqHITMmSEDqqC+hgsRyCVM6CLIkIBWT/eaVzFiil6Ohj0XnedIpy9IRS6HkbGd1Cwt2tGwrIWLO4Hq/s6Mc7j50PURBg2qVPtoZlF00Ci2L523LmJmM6+iOOOAJHHHEETj75ZLS2tgIAMpkMUqkU5s2bN+YDP//883j22WfxwAMPwLZtrF+/HmeccQaWLFkyddZzhlM3plO1BeGWRSZV3MvqTOcGgjBqtB9gqRunYKOUJ95lWM6kcsy2Q/1CrLfYZH5TGPGwii27maP35JwNyynZCXQwWDZBKmsgEVHd3bhMxC2VM5HMmH7KSB8xuOU4FEk35VMX0fzfH7OyCZt39OPVHf1YvbgeVomrKkqZ1n4oIKMvmcdDf92NC09dBMtSpvS5cWY2FeXoX331Vfzbv/0bMpkMLrjgAlx44YX48Y9/POZ9jj/+eNx7772QZRn9/f1wHAehUGhKjJ5r2A4pW0DUjeHUTbW6SUybTCot5ClXgqJ06kYW2USol7pxHZlpjb1AuxyW7fiOvtV19IIgYOXCOuzsSPmPL1YpT2+YNtK65XfY9KcMRIIK0jkLHf1Z/3Z50y46cbKUD6uBeBE9ALQ3htHWEMKLW3shCiyPPxKHsBOlIAh4dUc/9nZn0Duk84ieU0RFIc1dd92Fm266CY8++iiOPPJIfOMb38AVV1yBK664Ysz7KYqCW2+9Fffccw/OPfdctLS0VGxYQ0Ok4tuOpKkpOun7VpPJ2pXMGNANG00N4VF/85ZrmzZFNBqc8ufe1BRFd9qEpNhobIyUHtEvg2E5qIsFkKgLoaU5Nuq+sYgGQoG6+gia6kN+bzkVRMTjobLPpdzvdYdiKGuiMR5AW0vc//3xq9vw3JZudA7kcdSKZgSDNmRVmvLXKhQJIJ2zsGx+HWLRIGwISEQ16KaDgTTTuglqMhwKhKMB//jJjAHD9eGL2usQCw+3Yp5+1Hz86vFtSOcJEnWBUTbn8hbiiTziYQ17ujMAACpICEdYDaJWvwvjUYt216JNlVKRo6eUYsWKFfjBD36A008/HZFIpGItks985jO46qqrcM0112Djxo245JJLKrpff39mUp0DTU1R9PamJ3y/anMwdqWyJhPFGhHlEkr9oSTdsDAwmENPT2pCzngsPJv7+tJwHIruntQoGd2xSOcsNCWCyKR19PVlRv3dtpjSZF9vGpLj+F06qYyBnt4MtBJPo/B17B3KoTEe9J9vd08a+3syaG8MY3BoOIKOByWEAzJefKMLi1rCoJSid8BGVJNK1g4mQ1NTFHv2DyKXtxFQBPQNZGDaFEFFRF8yD8ty0FIXhEMohtIGBgaG36u+pI6evizT/jFNDFrDcgeLW8KQRAF/39aNgCygJ6IUvb+pnIlkUkcmrWNvN3tduvsy6KsPgi5tKPm61zq1+B2uRZtGIopC2QC5om+tKIp4+OGH8fTTT+OUU07BH//4x3Hvs2PHDrzxxhsAgGAwiHPOOQdbt26dgNkcj3Ij7ZRSf1zedigIIVMubEa81IAoTOjESwjrvQ9qpXvoAUCVBTgOAaHsBOZ19hiWM+6CcNshGEgZRQJeySzLhXuFWA9RFLDysDps35+EZZOq9NMTQjHg6tAnIhpsh0JVRIQDClI5E539OcxrDCMUkJEzWDcSKeg28vRxRp6kVUVyJ2wtUIzuj/daK3d2DDshVvw9OBVQzuyiIkf/5S9/GRs3bsQXvvAFNDU14Y477sANN9ww5n3279+PG264AaZpwjRNPPHEEzjmmGOmxOg5R5kBGEJYz7nnSI0J9J9TStFT0KNdDocQ7OpKw7KdCenPZPI2HEIRVOWyjl6RRVDAd+peDrrUQNFImNSB7atVAsCBXha9tjaMrgWtXJiAZRPs7GDDU0zgbOry9JZDkMqySDweUeE4FJosIhxUfJnhtsYwQpoMPW+7ejfsvnnTRjJroq4gP19INKS6U8bCqKEpr7VyR0cSQU1Cc12Q9fILfGiKM0xFqZunnnoK//3f/+3//Mtf/nLc+5xxxhnYvHkz3vOe90CSJJxzzjk477zzJm3oXIaidHSWN204DkV9QkPvUB6mRSpOqZk2wVDWRFMiOGaqpy+Zx/1/3oWzjpmPVQvrK7Z5wJ0Q1VQJaolhKQBQXNli03LrDNZwH/14HUSWQ2CYBHnDRiSowCEEfUl2zJa60Y5+UWsUAVXCm3sGseKwBBRFQCZnoS4aqPg5jYVtEyTdgmpdRAMhFAFNRiQ43P0yrzGErv4scgZbQlI4EZzMGFjYUjoHHA0p6HKnfW1CoGFY08awbIiigB0HUljcFnM7f5gG/nhXRZy5Q0UR/VNPPTWpB//MZz6Dhx9+GA8++CCfpD0IyumzZ1yJgYTbksci+soe03KXVo93BeA5bN2wJxQh9rtON6BKvkMfiad/Y9rMDi86H9lnXgrTdCDLArJu15G38EORRYQDo+MXSRKxfEECW/cNwSEEqiwhpVtTtqzFsglb/CGLrv6+gIA67OgVWURTPIigJkM3HBCHTTLbDoGet2FYBImoWvKxvYiekmI9I0op8oaDgVQeGd3C0vY4YmHVn8Tlejccj4oi+vnz5+PKK6/E0UcfjXB4uPODD0BND4SWXhPoOXrvkt+0Kte7yZs2bDcdUybgBgAk3UnPicoSeNG1pkiQy6VuFM/RO340LwrM0Y83oKUbNoKqjKzO0i+OQ5HKmqiLaGWvUJYviOOVHf3o6s+hvSkC4ursSFNQkDUtNtlaHxs+fkCVEAmyr1hrfQiiKCAUYI4/71592Q67sgKKe+gLiYVY+sd0aFHbJFvaQrGzk+Xnl86LIZMzoRs2bJvwiJ7jU5GjTyQSAIADBw5U0xZOGWgZffbhiJ5FgnmzcjGwbN4qewIpxHNChln5ScR2iC8HHNTkksNSAJNAAFg07BVGoyEVSXfBNaG0bFeMbjpQFZE5NYfAJsR3tB6ZnIVQYFgHf34T60g40MccPQR2tTTWia5SdNNGMmOipT4EQihkSYAii4gEVQgC0N7EAqSQO6TlXSER6q0bLO6hLyQaUt37WEVSFN7JcMeBJJoTQcTCqn/brGHBsAlKX0tx5hoVOfpvfvOb1baDMwaElha0yupe8a8goq/Q0+fyNkRh/E4aT8zMsCqXQTAtB3l3ICmglnf0qrtD1rIJ8q4Di4WZozdsltoQSwxaEXeyNRKUAcq02i03ol46L+Y/piAIyOiW35ceDSmIBBVWtF3VzB5rioLerG4hmTWxcmEdbIcgoMoQBQEBTcL7z1zKTixg0gYAOzEQCji24zv6uoiGTM6CQykiQdlvZY2G2VVAzk3xeFgOAaHA3u4Mjl3Z7L9+3m0ty5mSkxhn5lORo7/mmmtK/v7OO++cUmM4paGUlsy9Z1z5g6DK3kbDrqwYazsEtk0hSeKo2+dNG4os+k7Gc0Ks5bHCk4hh+0qUAVUq23uvKl5E7yDvyh94jpBdnZSR5bXZViovRWJYDgYzrNXSi4oNy0FjPIBU1vIFzQRBQHtTGAdccTFUcEVTKb0DOhxCUR9jrZXRIDuJaYqMJfNifp3Ci+jzhg3HnXhOZU0ENRmaKsG0HDTXBdE3qENVJGiqhKib58/qdlE7qem2ZTqEorWeDUjFQuy2Gd2CaTtFy0g4c5eKHH2hoJllWXjyySexYsWKqhnFKYaS0n303oCRlwf2FlCMh+coQYVRt+9P5RENqYi5KQB/ufUEIvpMbtjJiKJQVjzMWxZu2hS62+oYcXPYhmmXjbYte1jaWJYF5PI2et2VgV5hmhAgFlIRC2l468AQVJmt2WtvDGPr3iGmr19id+tkcAhBzxDriqmLso4bzXWwiizC1G0o7jet8ETmEHYlk8qZqCtYTNIYCyIR1rC3O4NszvbTMZk8S914S9S9bh0AaIix7iFPGjmjWzBNB+COnoMKHf173/veop8vuugiXH755VUxiDMaCrZA2/uCe2TzrLVOlUXmUOzyUXAhw3leOuoEYpgEsmgXOPrhHH0l+jOUUmTyli+0BUrL5+jlgojedfQxN02RN8unoUzL8V8HRRKRzQ9vlaqLau7rxNJGoiiguS6I3kEd0bDq58oP9Gb9fPrBYjsUg+n88PExnJbSVBGpbGGnDPznZzts4C2ZMdHaEPLfX5bfl7G0PYZ9PRk34peQKRiakkR29TWYZu9PQ5w5ek2RoCnstmYJbRzO3GRSGTxCCHp6eqbaFk4ZCAVA6ahCay4/LAOsKRJzxhVE3bm8K2tbYjer7RBfERMYVsfMmw5se/zHZr38LH0T1OQxI/phR0+g571irBvRj1FvyJu2L5ImSQIM08ZAyptKVWFaBNGg4h+3KRGEJIlwHIJ5rl7Qgb4sUGJ362SwbDalK4qCf4L0Cs2aLBVdmQgiK9LqBhv28rp1vGnagCb5JzFZErGwNYrGeMCdsGVDU5ZNcKAvAz3PFsKEA3KREmcsrCCtW7BJ5XMVnNnNpHL027Ztw/HHH18Vgzijoa5mO6EUIooj+qDq5YIlvx99PHKGBUUWQKzRCypsh/jpAcchyBUuNqkgR593dfF7BvWi3HQp/K4bZ1jKwUtTGGN0EOmGjVzeQt50WGupIGAwYyAUkKEqEtJZC811w4tFJFFEa30IB/qyiIYUNCUCONCXxdHLG6ekBVE3bAym8qiLaP7JRZa9ExE7oQLuFRmAoCohbzp+ft4hFI3xAByH+M/fQxTYFQlTwWTRe0df1i0yK+hL5tEYHx76MkyH3TZr+ntmx3oPOHODCefoBUHABz7wAZx66qlVM4pTDAHrlx4d0du+3ruqiKxgao/tjCml0E0H4YAMyx49gOOt4zNtgpzBujwCrmOqRNo3p1vQDQfZvI3W+lDJFYIeXkRvO8Qv3nqFx3yZdk6m7+Pgf57ZC92w8fELV4NSimTG9PvQqUAR0or12BMRDd0DOdgOQXtTBFv3DkEAJiWHPJKMbmEwbaAuprntmoJfgC5MW1k2gSqLCGgydMMGIQT97pVIYyIA2xnO7RciiSIiQebUJUGAbtiIuzn9vmQeb1tY59/WtByEAwp6h/IQBCBnOIhzRz/nqSh18973vhdvf/vbkUgk0NDQgKOOOgqSxD8800UqY8Kho6djddNmXRWUdbdUonVj2QTU1S8fuXJu+L6sZXEwNZx3BtiJZbxUgGEVasIHy+rcAMMSCLY97OhVRYIqi2Wfi+2wQaCOviy6B3VkdAuSJLD0R1SDQwhkUfQ7ejxEUUBrQwi5vIP2xjB0w0Yqa417YhwPQikyuomBVB71UQ22Q4rSKIUnOpZSUt3pWNZe6S0Tb4oHQcGuzEYiigIiIQUZ3YKmiYiEvHZLC7ph+/l5T5c+EpSR0S2IgoAht3bAmdtULIFw7rnn4u6778Ydd9yB9evX44UXXqi2bRwwRcbv3fcq9nSmMdIl6QbbkuQVHtmWqdJRsBcdmzbbqwqwtEBhROv9UxTZSaR/hKOv5ERi2gRdAzkmmRoPQJHLT50WR/Q2JFGALInQ3JNWqXqDaREMZS1fLmF3ZxqaLCHjqj8aJkGizHRsPKxBkQW/FbFrMHfQe3Yti7D+djeNZDsUgQJnXbhwhQKIhBQEVQk5d3p1MG0gFlL8SL7cFVDc76YZvqrypo+91I3pbs2KBN30l+UgnbOqtpCGM3OoKHXzve99Dz/96U9x+OGHAwBef/11fO1rX8OmTZuqahyHtVASQpEpocuSN2wEVBmyLCKkychbpKSjH8oYONCbZQ5BEPwsvyAAToFSr3cyUGQRWd1Gxo2yfUdvji+ZYFoOOvuzaK0PQhTEinL0NqFwTKbCKYoCAopUtm/ftB30DAyrbu7qTGF+U5gtMIlqcBzqR7wjEUUBTYkgLItAkUV09ecOOkfv9fADo1srAXYylSUBjlsYDbmFU92wYTkEA6k8GhNuPYGi7BWQl5ZKZ03f6Y9y9DZBc13I19dJZU3IlEI3HESC7HG7B3MIarJfNObMDSqK6AVB8J08AKxevZpX86cJrwBqO8Va87ZDYNosf65IbHORWWKdIKEUXQM5qIqI/lQePYM5P60hjkjd2A6LThVZRC5v+Z0s9a7CozGOlo5D2Immsy+H9sYwKMpHqMBwwdJxUzeKLA5H9GWkinXDRk9ShyQKWDY/jt1dad/RJiIaILBOl3IEVBmCKKC1PoSugdxB5+izuuUvf6mPaiCU+q2VHpoiwbTYtKwsMeliw2JF7/6kgaZEwJ8CLvd6edIOaX14KUlfMg9ZEvx8PcC6lrwW1WTagCQJRUNvnX1ZpN2fOXOHMR390NAQhoaGsGbNGtx9993IZrPQdR0/+9nPcOKJJ06XjXMay3XEllPcKufp3ARUCaosIRSQ4BA6qnc67erGqIqEcFBBLKz6jkgQi6UVnt3SjXv+5w1/8KqvYAgIGLu3HWDCYgNpA6ZNMK+RtTGW66FnfxPd9BGTNFAk5uiDmuwqWI4+lmE66B3U0VIXxNJ5MQymDezuSg/bSek4dQERAEU8rCKbtyue9i1HJm8hlTUhgGnVCMLoqFyRWTHbi7S9grNlE1gOQVMiCIcQf8K5FI1xFvV7nTcAUwhtiAUgCKzlMqDKUGTJz9knswY0VcJQxmD7B9yUWmH6hzM3GDN1c+KJJ7rbeNiX4T/+4z/8nwVBwHXXXTctRs5lvN512y6WQfC+8KoiQZZEv8vEa4cEWG6+e1BHUBsd4T7+4n4EVAlHLmv0f9czpMNyWI69IR7EYNpgW5L8yVt7bEdPKLoHmLxAewWOHmBRvdd1o8giJInJBHT2l86fm6aD7kEdb1tUh8VtTNdm8/Y+CAIQDsogBGX79gHPCQtuJ5F9UDl6hzC7vUIwe67CKEevqSJMq8DRu2mTfe6ilMY467gJB8qfoOIRFaIg+MtNABbRe9u0DFc6AQDiIQ2KJGIobUASRdjEwUDKwECG1QMyOSYEN9bVFmd2Maajf/PNN8d9gIceeggbNmyYMoM4xXiplZGFSU+eV1NEKAXOWDeGk+7ZPNOc8S7lPUzLwXNbutHWEMK6ZY2+SqQnqdA9qKMxEUQqYyIcUHzd+HyZYq+HQyi6BnRoCosqM7o1ptMFWJ7edjVfFFmELDI9d6NEGgoAugdyyJsO2hrCaEoEEA7ISOUsJCIqKEHJk1ohoiAgqElQFckfAhs5cVwppkUASjGYNtAQD7KtUsroPbSaLEFxJ1YBIB5h78ded5l3U4LdNzBGRK/ITPLYO8Hbrv79EUvYMhhKqC8fEQhIiIQUDLkS0xIEdA/mEFCGh7FMy+GOfg5x0O/03XffPRV2cMrg5eiZUuHo1I2myFAkcVjn3BUDo5SieyAHTR39Fu/oSMEhlPXF0+EirPeYXQM5KLKIVNZEJMjyypoisscey9G7VwPzGkO+Qxk3opfE4dSNW4wNB7z1e6Md/b4elqZpbWDHWORG9WyytLi1sRxhjZ28KIUvhzwZ2JCXgIG04fbBlz6+JLFiuVcb8fR49nZn/KnWQtmEUkiigHBQ8bWHBtIGKGVXA5RSf9EJwD4TkaDi6+AENJGdhNwisSAwmWfO3OGgHT0vylYXP6K3i4uxmfxwjl6ShrcqGRabKM3mbWTzVsm+7K17hwCwvngIw++h5+i7B3JQJBEZ3UIooECRJQRUFmVbY3Sp5PI2+oZ0zGsMu48pjOvoFVn0VRxlSYQsCf7VSWEaCmCpkgM9GQgC0OJ2qixuY+v36qIaHEIRUMZ39KGA7Auq5ccQTxuPjG7DtG3k8jaa60KwHOKLlo18joUtn17Hi2E5aEoMT/AqY0TYkigUTcd6G7wa4kGYdrHkgywJiIdV9Azm2ACXJPpSx549GZ0XZOcSB+3oJ3PJy6mc4WJs8cBU1o3sAhpLFXgRvWHZbq48V9LJE0Kxff8QgOFUjOfoPF2b3qE8m6A1bIQ1ttw7oJVvefTY050GoSw/bzsUwQLdlnLIkgjboTBsx5dH9nLZIwe0CKHo7M+iORH0t1Z5efq6qAYB5dsTC1FkCZqb4pnIQvWRZHQT3a5q5sK2GACh5GuuKRJa64f32Ba2NjYlhuUL5DFmDiSJOfpUjrXZ9iXZcRtiGmybIBQsHtKa3xxGLm/7u2YL8dpneZA2d+BJuhrHL8Y6xa2N3q5UTZEgSYIf0ZsWQUa32BRliXH6vT0Z6IaDRa0sEs4blu/ocnkbmiKyE8WgDt2wEQowRx/S5HGlivf1sJzzvMYwLNvxc8Zj4UX0puVAlUU/RQEMn7Q8LJugsy+LVrcA6RCCuqiGS9+5DMesaAJKdLyUQlVEBNw0St6wJ6Vg6Wn6H+jLQhIFLGiO+s9n3OOrEjT3+I2JoL+RqpxuPzAsg2DZBI+9uB/PbulGXVSDqkijrmRkSfAXje/oSI16LFFknU6F2vac2Q139DWOH9HbxZLCWd12lSsxwjkS9AzmENCG5QWeeGk/uvpZZLd17xAkUcARSxsAALrJctSUspz9YSMcRFBjNYCgJsMwyZh95wOpPCRRQDSkwHaof5UxFooswnIIbIdCkSUIolCgSV+sxTOQNpDN22hzJX37kwYch2D5ggQrgopiRQVGubCmYU1O4dG0CCCwk1tbQwiyOwFbiaOXpeF8elM84G+kGg+vzfW5Ld04rCWKi9++FABGXclIoohoSEV7UwQ7DyRLPpZA2dUMZ25Q0WTsWPDLv+pi2cNdN07Ba53Luzo3bh485OfobeQtx5+efHZLN/7yahdeeLMHl75jGbbtG8Litqi/Z1bPs4jWExGb3xTGrs6U7yCCmlQU0Ttj+IahDJva9HR0SqUxRiJLYoHODeurD/vCZjZsQqC5m0/3druF2PoQTJsgElSgm2zq03bG7kMfiTeAVOmylpFYNuvY6ezL4fhVzWU7bkohiezEmcyaaEoEYVrEH0obi7ctqkfetLHisDrf6QMoeSWjKSIOX5DAn/5+wN+wVYgosuGzkWqZnNnJQUf0559//lTYwSnDcERPiiN6t9AqicypyiLrjDEtgpAbzadzJv78SicWt0URD6v46aPbMZg2sOKwuoLdpRYIHS7EhgIKmhNB7Oth/fBBjUkshAPKmKkbhxCkciZiEdU/+StKJflyps0OAKrEHKU3UGSYxdLIPW4+3JMxiIVV/zWxHepLNldCg+tY86Y9ZstoObJ5C/2pPBxCsaA5AsuurOPHg3XbSAgFZBAKBEsUcUvdZ92yxiInz9QyR1/JBFQZS9rjIJT6A2U7O1L4z19txlDGcAuyfHBqrjDmp+tDH/rQmMW0e++9Fx/72Mem3CjOMF6O3hqxDzabt6GpUlEkpykSnAL98Sf/dgCEUJx30kIENRm/eHw7OvtzWL4g7t9HN1gkn3K7OQSwKcwON9UTDiiQRAFBV9bYLHO57zgU6RzTgbdsgnBAqSi6VeWCiF4VIYoYTkPZBFbB8QbThpv2kJDO2UhENFfPnaV4AuP00BcSCbHnlR9H1qEcOWO40Dm/OQLLIeP28Bdy9IomLE5F/e9XJVc/iiKOsrVcS6emSpjfFIEii9h5IIUlbTH8zzN7kNEt7OlKY+3SBmR0y5+h4MxuxnT03rrAxx57DJlMBu973/sgSRJ+97vfIRaLTYuBc51CCYTCYDpv2ohHtKKWvIAqIe86xgN9WWx+qx8nr2lFvbtP9MPnrkA6ZyEaUv1hJCY1QPy2vXBQ8UfoARbhS+JwsTdn2CUHjEzLQVa3EAurbEl3REMlyJLot42qMkvdKAprszQtpyiPnMyYiIbc1JD7fOujGvrTeQCjNWbGQlPcZdzu858IxBUKO9CbRV1UQySogIIioI5fk/BYu6QBqZzpShdIFdUW1BHbqgDm6GPh0ekXVZYg2hQLW6PY0ZFE8FUJg2kDoiCgoy+LdcsaQSmFYTplr0SSWRPRUGUnbE5tM6aj9xaO3H333fjlL38J0e0KePvb345LLrmk+tZxiiJ6hww7PcN0/F2xHgFXI4ZSikef34dwQMZpa9v8v8uS6F/2S6IITZH8PHjaHa0PqhKa3FF6TZHclkc2xOQdl1AKacSXfyBtgILJ6RKCitMYaoH9qiz5+X3Wt188NJXMmYi7g1Ga6xxjYRW9QzogCGP2oY9EUUQEFHZinKiwmeUWcPf3ZrC0ffjqaKLHJ4TCpI6/2Hs8Ss0kOA5K1iZkSQQosHReDG/tT+LpV7qwZkk90lkTHX3sSkQUBGTzVsn3ihCKzr4MlJZYydkAzsyiok/m4OAgDMPwf85ms0gmS1fzOVNLYQucaQ1ftuddEbDCPHjQVX18Y88g9vVkcObR7SVbLD1CARm64fhpF4CdLJri7GQQDSnucm/R/7KXGzDy+rrjYZUpSFYYXReeqFSFpW4A+Fo0pj18ckvnTMTCbCes12sf0GTfAcoVdLx4iIKAgCb5S7ongmE7SGZMZPM2FjRH2C/HkBguhbdLljjwT6LjIResJSyk1HFlSQAE5ui925xz7ALMawyja4DJM6uK6E/PjiRv2sjoNnSzOI9vO6SivcSc2qKiU/WGDRvw/ve/H2effTYopXjkkUfw/ve/v9q2cYAiJ+TlxwllKpWqLPoTngAQ1BQc6MvhiZcOoDkRLBIsK0VIk6G7EX1GZwqMQddx1kU1REKqv9zbb3m0SksG9A2xSc2oe59KnV6xox/uWvEULK0CLZqMbiEeUWE7w45eFAQkogF/o9JECGlMJ6eU4zJMByndRKOrDllI3rTR2c+K1QuaI7AdgmhYHlfXpxDJjf5ZyqeykyI7oY147cvMDsgy2y7cEA9g9eJ6rFiQQCSkYF5jGA6h6BnKo7U+WFbgLJe3IYpM4rjwiqN3SEdQkytOzXFqg4oc/Wc/+1msWbMGzzzzDADgy1/+Ms4444yqGsZhFEoOGBaLroyCtXuFziUUkP3umQ+effi4jicUkFkx02E655rKhLdUWcTbj5qH+njIL+wWKViWKF5626iCAQlhTa54YrpwMYmmDk/SRkMKOvtzAKWwHQrLtmFaBLGwNuqKIRFRx5VaKEVQk9E7lC/K0Vu2g55BHf2pPAgF4iF1VO4/q7NCrKZIaEoEYLg7eCeCJLJlJKygXqGjlwSAFmysohSiUFrzXxQEKLIEM2/ifWcs8X/vyUd39GXZPAKoK6Fc/BjJrIFwUEY2P1ywpa6A20SuXDi1QcXvWFNTE5YtW4YvfelLvBA7jdh2oaNn//bbERUJcqGjd3OtS+fFsKwgd1wOlrphEV3GHcBSZBGhoIwlbTEcvqDO/1L7ve0WKal3M5A22AAXFcpueCpFodMozDU3xAJIZU1QsHSBtwQlFmZyvYU7YUMBBS0FEgOVEg4obnvl8O96h3QMpAxE3a6ckc/VGyzr7M+hvSnsasHTkgXRsZBEAQSY0KYnSRQhFGgTsUGr8jIT8Yjqr1z0SERUBDXJvyKRCxaTeDiEIGc4vvCbF1jkTQe6cfAa/pzppyJHf9999+ErX/kKfvjDHyKdTuOTn/wkNm7cWG3bOCiWJ/a+cJ7yoKaIRVG7t/ji7OMWFD2GZRP/5FBIQGG7Sx03deNtQAqorLebkOElHlF3D6llO0hlRgtiDaUNxMOsh76SKU+PwmJsYQqjMR6AQyhbDkJYJAmwKDwcUKZEY4nJIRe3jGZ0G8GA6zwpRskEMEllGz1DOtqbWHRMhcqLzx6SxIrHkQkOLCmyWDw7MMZx4xEN1ojl54IgoK0hjI4+5ug1RUIyaxS17rKCPvzXIO/m6bN5C4SgpHw0p7apyNH/9Kc/xa9+9StEIhE0NDRg06ZN+PGPf1xt2zgYkbqxvZbI4Yi+UB/llDVt+Mi5K/0FFB66YYOMcFp6ni3jth2KvGH7kgos788e0yHE7yRhU6vsMZJZY1T6Jpk12Uo7ofL8PDA8VCVLQlEKw1N1TGZMOA7x1wUGNZkViacAbzDLOwk6hMAo0GmXJAF6vvgEadoEPYM6KIWv0slaPSeauhER0qSiReKVoMiSP+A1noY9OwmMjr7nNYbQM5iHZRNIkgjbJkWbybJ5yy+KK7LgpwOHMgaCmsQj+hlIRd9IURQRiUT8n9va2iBJE/uAciaH7RB4QbsXeeYNL6KXinLTmioiES0ukumGjXhYw2HNEV8N0rYJSxu4BbWcK2nspW5UWQIohVMQ0Xstj0zxEkXdGA5hffjxMHs8T/elErxisrddysNbeJ3WTRi240f0kZAyYadajuF0FBuaGrmGUZaEUbrtedNB9wDrMGpvCPtSDBMpxHq0NoSLUlCVoBYMTdES+2mLbytBU+RRxeZ5DWEQd1+BR84YloROZky/BqIqEtI5C5btQM/bUBXxoLZycQ4NFX3KEokE3njjDf9y+YEHHkA8Pn4OmHPw2AULI7wOFO9LGVCLi7EjC5KUMoXClvogoiEVjfEAMjnm1Bc0hf38cM60Xe0cGZLE0kGaKsMmpKhl0Wt5lCQULZjO5CwYFvHz2mOpMI7EO5HIEtsu5eFF9OmsCcMkSGYMNqGrShM6kYyF7+gNtiLRtNnGKA+mw1Mc0Wd1C92DOcTDKiIhBZb7vCd1/EmkoFRFhGE5SOeYdIU6ztVTPKKMEi/zC7Lu9LOqSOhP5n2Ji7zBZjGy7oYwxyEYyrB6iSAIPHUzA6koNLr++uvx2c9+Fnv37sWpp54KTdPw//7f/6u2bRywiF5TJOiG4y4IH47oR+qjjIwqs7qNpkTQj4Bb6kNI6xbi4QBiYQ0Jd6VdOmfBIRShwPAVQlCTkbNJ0cnDG8gKqGzKsqWebXnqc5dgRIITj7a91JAiiayrxEVTJXdNoAnTsjGUMdyWSmHKVuBFfMVPFtHreavIBubkqN9+SClFRrfcLVrMWRI6XASfDhpjQUSDKtsjMM4idACIBFX0uhpBALsqDKoSIkFlOE+vSsjlLWzbN4hERAMFxSPP78PuzhQ+/b4jQAF/f7AosN0InJlFRZ/QJUuW4He/+x12794Nx3GwePFi5HKjFxpwph5vChQAbLeHPefmlEdOLHp95NRNuwiCgMb4cL5elkQsnRfze7jjburG62gJqsPDR+GADCNnFTn6kCYhm7chiSIs4iBvMlXEXZ1M0pg5+ok5Yc9RaapUFNFLooBYWEUyy64WklkTkaACQRx/a1WlRH2VTDbtm81box2nW5eQJRGWzbT+hzImjl3Z7Pb3Y8yhtKlGFIUJFX49hVPAW2ZOAFC0NYSwvzfjzyiEAgoIoUhmTKiyiG37hpDLs6JzLKQia9hIhNltJiMCxzm0VPStvOiiiyBJEpYuXYrly5dDURR84AMfqLZtHLDUjbdUwrRZN4SXTgiOWJsnCMwJEkqR1W20N4VHOS5FHh5KqouwPPiAm/8OBYaHflR3mXVhGsYbYgLYwul0zsT+3owv7hUKSNAmGtG79nn7Yj1Eka3DS2aY5nwqayESlKHK4pRtNYsUOHqHUGw/kMLt972GLbsHim7nFcTzluPr+s9rDMN2mFLoRFJV0w3T3pdg2Q6yORtNCTYAdvj8OAZShl9vANhrHgkpGMyYbM0kgJ0HUu5shYADfVn8xy9fxkAqz+XJZxhjfkKvuOIKHH300di6dSuOPvpo/78jjzyyohz97bffjvPOOw/nnXcebrnllikzei7BInr2NrFLZgo970AShZKRpCyJyGRt1Ee1cacXIyEZggAMesNOBXICqixCK4jwAbcdsUBpsmtAx1DGgGk7EAQgHFDHzRmPxHf0kuhfaQCsKyUeVpHMmq6MsolwUJnS6DkS8obAHBimgz1dKaR1C795aiee+vsBP2I33eecy1voHspBEIB5DSGYFpkReu6xsIZkxkI8oqGljs0brFpYB1EUsHlH/6jb73B3EURDCnZ0pPyF7X/f1gfTIkhmTHA/P7MYM/z6r//6LwwNDeH666/HN7/5zeE7yTKamprGfOC//vWvePrpp/Hb3/4WgiDgH//xH/HYY4/h7LPPnhrL5wi2mzaQJQGWzZZk5EzbX7s3EkUWYSsi2twc8lgokoSgJhf1qHuPqcgiGhMqxIJNI54mPXVzw4SwQnEyYyIWUstOaY5pgx/Rs21ZhdTHAqCUba4yLCZ9XKmGTiUEVAWyJCBv2tBNB/1JA3VRDYe1RPCnzZ3QDRtnHt3ut19mdAs9Azqa4kGoCtPJqWSL1qEmHGAtqfMaQxBFASFNAqEUy+fH8drOfpx97Pyiq6mdHSm01AWxqDWKl7b1sqE9AXhjzyAANqFNKIVYSniHU5OM+a2MRCKYP38+7r33XsTjcbS3t6O3txfPPPMMnLFWDYFN0n75y1+GqqpQFAVLly5FR0fHlBo/F7AdCklkwzWm23VjmDYUufTavGhQwcLWaEUOV5IEBFXZn54s7AIRBGHUtGkoIMMh1L+9F10ns6Y/GTphR+8XY4VRWjXNbudNp6+NL09Za6WH1zJqmDb6kjpa60O44JRFeNuiOry2axCSyFosHUKQy7OJ2HmN7HURJiDedigJBRQsmRcvkLNQYNoEa5c2IJu3i/bKmpaDvT0ZLGmPYUl7DLZDsbcngx0HUn7azrDIpDT8OYeOir41t956K/bs2YNrr70Wn/zkJ7Fs2TK88MILuOmmm8re5/DDD/f/vXv3bjz88MP45S9/WbFhDQ2R8W9Uhqam6KTvW00mY5dDKIJBBaoqA4KAhoYIHCowlcmmKJpGOOOJHINSimhY9XVq5s+Lj7p/4c/rVrTgob/uwdb9KZxx1Hz/9xndxqK2KBJ1YbS1xiaUQ7cFV2IhrKK5Keq3PALAiqWNALaiz73iaGoIQ1UkJCYhd1COcFCBTQBJVZDMmDhhTRvq6yJYsbAeW3YPQgsw/ftINAgqisgZNpYdVo9oJIhgSMO8tuEUZi1+7krZpAZVmHQIxzXG8NAze7B1XxLHr5kHAHh9Zz8IoThyeQsWtsWw8ckdONCfw5ArcZE3HYiSiPqGyISngQ/W7kNNLdpUKRW9U0899RR+9rOf4Ve/+hXOO+88fPWrX8X73ve+ig6wfft2fPzjH8d1112HRYsWVWxYf39mUlFDU1MUvb3pCd+v2kzWLsshcCyHDe/oFnp700hm2BLuVDJXlFqZDJo7sKMpEvSsUWTjSJvb4gEsaI7gD8/uwfL2KJNKIBRDGQOaHIeeNdDXl5nQ8dOuEwchGBjIIFcQsWtgU6d73K4eEApZEqb0/dVkERndxLZdfaAAYgEZg0NZRAIs+t2+ZwDNdUHs2kfx1h5WpE2EZHT3pZGIaL4ttfi5K2eTYTpIJnMgIRVvW1SHl7f3oquHSVq/sr2H7S0Iychl85jfHMEr23uRzFpYt6wBf9vWi2Qqj56edNV06mfSa1lLiKJQNkCu+Do7GAzir3/9K0488UQAgGmO1jsZyUsvvYSPfOQjuPbaa/He97630kNxXAilbCeoxGQJLIeAuCJTiruN6WDx5Ie9qdixkEQBp65tg27YeOa1blBK8ZfXukAIZWJZkyiUBjWJab4ER28yCrhyB/5UbECesh56D6/A3OPKLHtLV1rc/98zqAMUSGctdA/qkCUBLfVBOIROSLytllAV1rlEKcXaJQ2wHYqnXj6AvGFjx4EkFrVG/EG5pfNi6E8ZsB2CI5bUQ1MkNnfAq7Ezioq+NXV1dfj617+O1157DSeffDK+/e1vo7m5ecz7dHZ24p/+6Z/w7W9/G+edd96UGDvX8JQrZUmApki+Vk3edNxtTAd/DC+3HgrI425IEkWgtT6E1Yvq8OyWbvz+2b34378dwJol9VixsA7aBHamegRUGddesg4rD6sbNfDlbZAC2ElGU6QJLRepBCZs5mDInbytdyUkQgEFkaCC7kEdgiDAsAm6BnJoawhDEkVQYMI6NbWCIAgIawosm6C9KYwVhyXw3JYefPfXr6A/ZWDJvOF01BJ3cUk8rGJBcwQBVSq7k4BTu1T0rbn55pvR3NyMu+66C8FgEIIg4Oabbx7zPnfffTcMw8C3vvUtXHjhhbjwwgvxi1/8YkqMnit4GiWSKDKNEZt9wfKmA2WEcuVk8cb3K4noPTXDM49uh+NQvLi1F8euaMJ7T1sMgQrQlMldytfHApCl0VcossR66T07RXHqpmI9mFSxg6G04csueD3izXVB9AzqTNzMXQbe3hSGQwhkSZjQjtpaIxxiRXhBEHDJO5bhqvNXYfXietRFNaxcmADACrOt9UHUxzQcvaIJgiD4ET338zOLir6ZjY2NuOiii7B161Y4joMPfvCDaGwce3vRDTfcgBtuuGFKjJyreKPmfkTvSiAYpgOtYPDpYPAcaVAbPy0iimw9XX0sgHefeBhM28GJb2thJwBhYmJmhQiiAAjCqCsUQRBQ74qbRYJKVRxrJMgi266BHJa2x5HJWYDArnSa64J44Y0eKJKIXN6C7VC0N4bZApRg7ffPj0VIU0Dp8LBUW0MYF5wy3JJLKUVatxEOSPin967xf6+pEkxr4gvVOYeWisKjp556Cpdeein+9V//Ff39/Vi/fj0ef/zxats25/FSN5LEFnnbru6K5RCmRT8Fjt5LjQTV8VfhiYIAAcwJHLOiCSetbi3osKGTjrZFQYAkCSW7dZpcCYdwUJnwMFYleAXFbN6dGhUFJt3rELTUsVz8UMZguXoA7U1h2DZBdIKLRmoNVRZLCBgPY5gOokEFlk39he0A/IieC5vNLCr65tx+++3YuHEjYrEYmpub8fOf/xy33nprtW2b8xSmbjRVgu0QXx+drRE8+GN4EX1AkyrSkJFlsXR+lmLcHH85RAFFOjeFeEXRkCZXpWc9UtDO6aVuWutD0PMOmt0p0p4hHQf6sggHZLZcRZi5+XkPxR24K9fZZtqESUVTFMkdsAEzh+vdzDAq+mYSQoqKr6tWrZoyvRFOeTyNFUUSEPAcvTulqSnlV8hNhIZ4AKLAcuGV5PxlSQAdEcw5rpzxZGsGgiCUTfu0NDBnGw7IVREPK3T0DbEAgpqEuogGRRGRiKgQBKB7QMeB3gzmN0VAKSCNWGU4ExEEAY2xgK9pM+rvYHWRiDtcBXhyHDJMi/DlIzOMij6twWAQHR0dvmN58cUXoWl8C3y18b5MrL2SpW6yrhb9yMXgk6UuGsBH16/EsvnxiiJ6SRwd0Y+36Wg8BKH81UBTPIizj52PVYvqprwQCww7ek2RENQkhDSWwmqrD8GyCRpiAezpTqM/ZaC9KQzTdhANqbMi0KmPsfrHyKjesgkCmgxFllAXVYdXWOZtREMKTMuBdZDzG5zppaJv57XXXosrr7wSvb29uOSSS7B7927cdttt1bZtzuNF9LIkQBBYNJvOektHpiZHD7CURd50KkvdSCKyecsvjNoOWyQ9Lzr5E78gCJDl0sdWZBFrltQDtHzUfzB4jr6pLgDHGd79GgurUGQRTYkA3tgzBIDl5y2LIpKYmf3zI2F6RgH0DeWLZgJMky2rAVhtRIB7MhAE1EU0UAC6wXP0M4mKHP3RRx+NjRs34u9//zsIIVi3bh3q6+urbducxy/Gulo3AJDKsUG1icoBj4Uqs+JjJVFqYzwA2yZIZUwIggBRFLCoNern+ieDLAkIaqWdp9d26VBapG45VXiOvjkRBAX8E5ggCIgEVTS4US/AVvAZllPV0f/ppiEWRN9QHoRQ/wrRocSXolBkCaGAglTWQkt90F8/mM1bZR+TU3tU9Il9/fXXAcBvqezs7ERnZydWr15dPcs4fjFWlkS/xz3lRvSTmUIth6ZIfh52PIKajMXzYtANG1mdSd9OZBl4KQLq2GJlAVVCVrerEtFHQyrmN4WxfH4CglC8mi8ckFEfZY6+KRGAqogwLWdGCJlVysionhAKeYQEdl1UQzpnoi6q+V1KusEd/UyiIkf/6U9/2v+3ZVno7e3FmjVr8Jvf/KZqhnGKUzcB94uXynlLQqYufaCp0qi9ouMR1ORpi2w9hclqLPgQRQEfetcKAGw5eGEdIKDKaIizlNT8pghshyAYGL8NdabREAsiq9tIZ004hKIxHixKC0ZDCtoawgioMkLulZee5zn6mURF39Qnn3yy6OeXX36ZO/lpwHaLsYos+RFWJuetEZy6qFKWxZqe8gxoMlSjdHfIVKBIIjJ5CyGteFm3poqIRzQceXgj1i1rgOkWZ2cbiixiaXuciZ3lDEQC6oi/S75kdSTIXEbe3UswG4rSc4FJhUhHHnmkn87hVA8vR6/Ior9OMK2zHH2oTE57MoQ0uaYdmCqLVV3ALUmivzS76PeiiIAqY/0Jh+GwliiIwyQTZiuaKqE5ERpTldL7m7dnlzMzmFCOHmDDE6+99hry+XzVjOIwvNSNphRG9JarszJ1aQy2wWrKHm7KCQcUqFU0UJEFWBYt6eAiQRnJrMnE1GbIopFq4qUMDdMBIUAV6uOcKjDhHL0gCKivr8fXv/71atnEcfGLsQWplWzegipXNsU6WxDL7MedKmRJhCQJ/gamQsJBFf3JPIjibvqqggzDTMKry3Cp4pnFpHL0nOnBS92osgDNdTCGRRAPy1UpTM5VFLerqdRVkiqLoIIAyyYIB+U5n5NW3QnovOnwdYIziIoc/e233z7m3z/1qU9NiTGcYoZTNzLUgohWVcSq9JTPVbwrhlKTt5oiQQCbFm10tXDmMoIgIOBLFXNHP1OoyNHv27cPzz77LN71rndBURQ8/vjjaG5uxvLly6tt35xmuOtGLMoNq7KEOZS5qTqSKPjdJCMRRQEhTUIyY03p7MJMJuBKFfOAfuZQkaPv7OzEpk2b0NDQAAC45pprcM011+BrX/taVY2b61i2A9HVgVFlkUkEg0X0s62X+1AS0GS0yOUXjoeDCjK6XdMtqNNJQJPcYiz39DOFiq7/e3t7fScPAOFwGKlUqmpGcRiWQyGJIgSR6YF7hcCp2hfLYYhC6UKsRyigIBxUqiKqNhMJaTIvxs4wKoroV6xYga985Su48MILQSnFr3/9axxzzDHVtm3OY9sEkiT4Tl2RRZg2mbJ9sZzKCGkyWuvLR/xzjaAmoz9lcE36GURFIcr/+T//B+FwGDfddBO+853vYMmSJbj++uurbducx3YIJFHw8/Fenl6dou1SnMqQJbFIt36uE9RkmJbjt/9yap+KIvpIJIJ//ud/xu7du7F8+XKYpglVndmr1GYCls0cvdfS5+WItSnSoudwJkNIY4vFCXf0M4aKIvqXX34ZZ511Fq655hr09PTgjDPOwN/+9rdq2zbnsR3C2ihdn+71eWvK1CwG53AmQ1CTYdkEps1TNzOFihz9Lbfcgv/+7/9GIpFAa2srbrnlFtx0003Vtm3OM5y6KY7op2q7FIczGbhU8cyjIkefz+exbNky/+czzjgDDl8lVnVY102BmqLr6AO8n5tzCPEE5vjykZlDRY5elmUkk0k/V7xz586qGsVhsK4bcVQx9mD2s3I4B4u3fSpn8GBvplCRx/j4xz+Oyy+/HH19ffjCF76Av/zlL/jGN75RbdvmPF7qxjvBeo4+pPGInnPoCLupm1y+ejsCOFNLRY7+tttuw+23346nn34alFJ88pOfLErlcKqD5Tp6D01lF2CzaWcpZ+bhSRXrJnf0M4WKPEYgEICmabjsssuqbQ+nANuhCGrDHTZeRB+cxcsvOLWPV4w1DDYdyzvAap+KHL2u63jnO9+J1tZWhELDE4IPPvhg1QzjsNSNLAp+e6VXhA3z1A3nEOJdUeYtG4RQiFVY2s6ZWipy9F/96lerbQenBLZNIBZMxh63sgU9g3mEyigtcjjTgdd1o5sOTMvhGkAzgIo8xvHHH19tOzglYANTAryQfl5jGKcc0QpF4hE959DBVk8KsG2C/lTez9lzahd+Kq5hbIdCFoUiATNJFOfUGkFObRJQZdg2xVDahGVzKYRahzv6Goa1VxYLmMmSAIE7es4hJqhKMCwbEIBUzjzU5nDGgTv6Gmak1g3Ateg5tUFAk5G3CIKahJ7BHF8rWONwR1+jEEJBKIr06AFA5o6eUwMENRmGyQqxlk2RrcHhqYxu8S1YLtzR1yjeYvCR+fimeBBB3l7JOcR4W6YAQJEFDKTyh9ii0XT0ZZEzau8EdCjgjr5G8ZY6yCMcvSKLviQCh3OoiEdUpHMW8qYNTZGQzpk1lb6xHYKcYSHHFTYBcEdfs9gO+9JIvEeZU4OcsKoFtkPw8vZ+iKIAQlFT3Tem5YA4QDpbXCjOmzb0ORjlV92LZDIZbNiwAfv376/2oWYVtvulkfnUIacGWdwWw7zGEJ5/o5vlwSn8VE4tYFgOFFlEznDgkOETUN9QHkNp4xBadmioqqPfvHkzPvCBD2D37t3VPMysxC6To+dwagFRBI46vAlDGRPb9w9BFGtL5CyXtyHLAigFDJOdgAihGMwY0M3aOSFNF1V19Bs3bsSNN96I5ubmah5mVuIVYxWeuuHUIIIgYFl7HPGwiue29ECRRWT12nH02bzttiLDL8jmDBuEkJo6IU0XVRVNOZh1gw0NkUnft6kpOun7VpOJ2JXMs6gjFgse0udTi69lLdoE1KZd1bKJEIqOoTxOO6odDz29CxYVENIUNDZGpqRZoBK7DcuBWqI5wSEUgb4c4hGNtYCqIpqaotC7UmhqiMImBIm6MBR5YkFULb6/lVKz6lj9/ZlJ9cA2NUXR25uugkUHx0Tt6u3LAACMvHnInk8tvpa1aBNQm3ZV26ZUUsfK+TE8Ign489/345Q1rTjQqfpy2uOR0S0ENQmSWOxwx7ObUor+VB4dvVnMb46gPhYo+rtu2EimdBDbBqUU3X02IoqInXsHEdRkZPM2OjqTvtxyJdTi+zsSURTKBsg8L1CjeKkbrgzIqVVkWYSmSljQHMGerjQAAWaFBVlCKHZ3pdE/wf572yHY253Bgb4sggEZB/qyfg7ew7IJTMtBKmtCEARQSjGUNuAQ5gxBqf/9mitwL1Kj+MVY7ug5NUpQlWFZBAtbo+ge1GFYdsX575xhw3YIugdyE+rW6RrIIpU1EQ+rUGQRiiRgX08GpKCHP2dYeHZLF77/wBamly8IGMwYfgebJAnQ51h/PfciNYrXXqnw9kpOjdKYCMC0CRa2sNx196CObK4yR5/KmlBlAZIooKs/V9F98qaNgZSBSGg45RLQZOQMG31Duv+7rG5hT1cGOcNGR38WiiwinbP8xT2yJEKfY4vNpyVH/+STT07HYWYVfteNzOUOOLVJSJMR0mTIEnPYHX1ZLGyNglI6ZkGWUoqhjAFNZfn5ZNZARg8gEhxb1753UGfqrSMeOxKU0dmfg6ZIiIVV9Kfy6EuylNDuzjTaG8OgVGJpG3iOfm513vCIvkbxUjeqwt8iTm0iCAKa60OwHYr2pjD2dKdBKYU5zoRs3nRgE+oXYQOqhN2dKfQl9aLhpkJ0w8ZgxvDXGBYiigIiQRl7u9MYypjY18MaGRRZxJ6uNARBQECTQQjFqzv7QUHhEOJ/x+YC3IvUKJ4EgjbBFjAOZzqJBBUosoj5TRF09udgmmRUcXQkWd3Cm7sHsemPO0EIhapICAYkdPRlsXXvEIbSowu0PUM6ZKm8zpMkiQhoEg70ZbC/JwtVFrF2SQP29mT8k8druwbw2z/twhu7B4Eak2yoNtyL1Cjeh1CpsFWNwzkUiIKAlroQWuqCoBToS+nY15tBfoyibH8qj+e2dOO1XQN4aVsvALY5LRZWoSoitu9LIplhMgWEUvQM5TCUNsZVbVVkCYos4kBfFgtaIlgyLwbLJujoYzWAv7nH2tOdAQV39JwawLusrLQnmcM5VMTCKuY1hiEIwIG+HGRRwM6OFAzTAaUUhuUgl7dg2Q4My8GurrSfhnnypQPI5IY7YGRJRDSkYHd3GgOpPPZ2pdHVl0M0pPjRvEMItu0bwhu7B0cpZto2QV8yj0WtUSxsZT3lu7vS6Evmsbc7A0EA9nan3c6buZOnr9mBqbmOzSUQODMEWRJRHwugtT6Evd1pvOPoduRNG28dGIIgCHAcAkBg/ycI2LJ7AIos4vJzluOe/3kDf3hhH953xpKix4sEZOztzkCVRcQiKgDAsh089XIHXnmr31908rZFdbjglEVQ3YBoTzcbalrYGkUooKC5LojdnSnkDRuiIODYlU14/o0eWDaZU46ee5EaZTh1w98iTu2TiLCo/kBvFrZNEFBlBFQJAVVCNKwiGlYQDSlQJAHb9w1h1cI6tDWEcOraNry+awBv7hksejxJEpGIqggFh2PR3z+3D8+81o0FLRFc8o5leOcx7XhjzyDu/p83hrtsutJQZRFtDSEAwKLWKPb1ZLF5Rz+WL4hj1aI6AEBXfw65OaR5w71IjWI7FKIojBoP53BqkVBAwfzmCBxC8crOfgDMWYsj1Fe370/CsAjWLWsAAJyyphXNdUFs/N8d+MNze2HZpQu5r+7sx8vb+3Dq2la8/8xlWHFYAqcc0YbLzl6OjG7j+w9swQtv9mB3ZxoLmiOQRBGEUCxqjbIlJHkbRy1vQntDGJLIhqwcm86ZzhvuRWoU2yGQRGHUF4XDqUU0RcKK+Qksao3iD8/vQ2/BAFMhm3f0IR5WsaiVDVnJsogr16/EcSub8dwbPbjrgS14dUdfUe59IJXH/zyzBwuaI3j7ke1Fj7dkXgzXXPg2LGyN4PfP7kVfMu/38g+kDbTWs8g+FlaxdF4MsixiXmMYe7tZC2bPoF5Tm7GqBc/R1yi2QyCLArif58wU6mMazjl+AX76h23Y9Med+Nh5q2A5BNv3JzGQykM3bOzsSOGUI9oAAENpA6GADFWR8O4TD8PKhQn8zzN7cM+Dr2NeYxhvW1SHnkEduzpTEAUBF52+uGTgEw2p+OBZh+Nv2/rw3JZurFpYB8NyEA0qMC0Hx69qRltDyL/vYS0RPPNaN1RFQN8Qc/RtjWGIs3hFJ3f0NYrtkJKXvhxOrRIJKghrMi48bRF+8fhbuOuBLRhMG74OTUCV0BgP4OjDG6EbDqJhFZmc5e9BXtwWwyffswbbO9L4/TO78fiL+xEOyJjXGMbJR7QiHtHKHlsQBByzognHrGgCwCQW2hsjONCfxbknHFZ028NaovjLq13o6M9hUWsU/SnWytneNHlp9FqHO/oaxbIpJFGAAO7oOTODgCZDFAUsnRfHaWvbsGX3AE5c3YK3LapDa/1wRE0pRTpnYXFDDANKHoNpA5EQkz8QRQEnrmnDsrYIdNNBOCBPWN+eEPbdqYtq6EvmYdsEsjt4aNkELXVBAMDe7gwWt8UQDcnoS+YRj2jjyjDMVLijrzF0tw3McggkScAsvprkzDJEQUAsrCGjmzjz6HaceXR7ydvphoN4RENQk9FSH8JQ1mSpyoJWYkkSEQlOroSomzbqogGIooC6qIqeIR0R19Hn8jYEAWitD2Kv24opCAKCmoTO/iyWtsermsJxCAGl0y8/zouxNQKlFH1JHdv3D2HbvkEkMwYrxnJPz5lBNMQ0mDYpW+CkrhZ8c4JF1bIkor0xjKxuV9QBQwiFbthIZU2ks1bJCVzHoUi4aZ5IUAF1FxgZpuNLNhzWEsWergy27hsCAKiKBD1vYyhT3cXhg2kDAxPU4J8KuKOvAWyH4EBfBh29WUSCCqJhFY6ro80zN5yZRCigoKUuiHSutN57VreRcKN5j3hYxcKWCEyLIJ2z4JTZLGdaDjI5C5GggoWtMSxpj4FSFA0+2Q6BKou+XEJAkwFBACFsQrelPoSGeADHrWhGa0MQv35yh9/DH3ZVMMc64VBKi7TvJ0oyY4yrBVQN5pSjdwhBRrcmtaKwUrLuqHclEEIxkMpj274hDKVNRMOKn8ckhEKWRN51w5lxNCdCCGky8iMccCpjIhxUMM8dZvIQBAGJaADLFyTQnAhCNyykcxZSWQuZnIVc3kYmZ4FQYNn8BBY0RxEPqwgHFCydF4cgCEjnTKTd2zYlQn5eXxQEJMIq0rqFkCYjHJARC6lQFDaZ29YYwq+f2oGtewchSSIooegZLN0aCgAH+jLoS5b/+1hYNkFat5CfwKKVqWJO5Oh1w8ZgOo+BlAHLoWhOBKpSYTctB7s6UhAEAQuaw4iFy3cJGJaD3Z0pmDZBSJNGbZKyCeuj5yE9Z6YhigLmN0fw1v6kH9mLooDDWiKIR7SyxVVZEtFSH0JjYwQdnUnkLYetBTQdCCLQGA+Oym2rioSl82LoT+UR0hQEVMmXQ/CIRTT0DulonRdmksWqjIAqQxAoLj9nOX7yh6343dO78Yn3hBEJKugd0hELq6MKswPpPLoH86iLaGhOTPx1YfU3seJ1i1PJrHb0humgZzCHgYwBRRIQCsgQBKAvmUcoIKMuGhj/QSZAZ38OogiosohdnWk0xE00xIKjNLQtm2B3J9PujrrdBpmchadf7cSOA0nIkoiBtIH5TWFejOXMSAKqjGXz4yCEre5jy0kqSyAIggBVGe2wy6HIElrrw2X/HnKLvuGCZeAN8QA6+rKIhhS897QluOuB1/HQX/fg0ncuQ1CTsK8njcPnJ/wTi27YONCbQTysIGdY4y5XKUUyY0CVBVjuRO50FmRnnaPvHdKx48AQ8haBQyhkEYgVKN8BbCPN/t4MAqpccpHBZEjnTAxlDMRdAaZYWEEya2Ig5S5LkCXohg1ZYssQCCEIBmToho2/vNqF59/ogUMIlrlV/1BAxorD6rij58xYAmptuBdFFkddwcdCCjooBaUUDfEA3nF0Ox59YT9e2dGPdcsakc5a6BnUURfVYFoOulMGVFmCLImg1IZpkwkpyxJC0TWQwx9f7sBJa1rhOBTTuTyuNt6JKWQwlYdpEwRVqeywkSSJUGUJe7szWNoeO6gzKyHs7Ly/NwNBoNjdmcLC1igEQUA4wKJ103LY9puhHAQMt3M9+3o3/vxKB3TDwRFL6nHGkfNQHxu+yki7W+w5HM7UosgSYhENqYyBaFjF8ata8MaeITzy3D4sao0iFlbRm9TRl8pDoBRNjVEMZQw88dJ+nL6uHablTMjR500bm3f0Y8ueQbQ2hHDk4Y3QMH2eftY5egBlNWIopaCU5Qs1VUImZ6GrP4v2psiEHKplOxhMG+hL5uE4BIIgIG86uO+PO9A9qKMpEcBpa9vwtkX1EEV2GRoPayAWK07l8hbu/p830DWgY8m8GM46dr6vyVFkLx+X4nCqxvymMA6ASTFEwwouPHURvv/gFvzyibfwkfUrEQ+r/m3TORM/fXQbsnkbbQ1hLD8sjmhILf/gIxjKGHhlBxN7G0ob/ga56WJWOvpCCKF4c+8gtu9LYkdHCqbt4NQj2nDi21oQDsroSxkIBRXUj8jXG6aDwUwelkVgORSEEoiCAEqBnGFBgIBgQIIkyjBMB798cjv6knmcceQ8vL5rAJv+tAuPvbgfqxfXY83ieiTizJHn8jZ+8odt6Evm8Q9nLsXKwxIlTzKUUoiufjeHw5l6JFHEguYIVFlEl5umufjtS/GLx7fjvj/uxKXvWAZRFDCUMXDvH7aBUjfl05dFNmcDicqP9fybPcjoFkRRwGDGgFWmIEsIa9+c6vz9rHb0e7rSeOS5vege1BHUJCxpi8G0CZ782wG8tLUXG05ZiEWtURzozUCWRKiyCFEQMJDOo3dQhygK7p5K5nAdyqbaIkGW87cdgrcOJPGnlzvQ1a/jH96xFCsWJHD6ujZs3TuEzW/14/k3evDs691IRHdi2bwY9vdm0JfM45J3LsOy9nhJuy2bIG840FS+XYrDqSaCIKC1IQwKoHcoj2Xtcbz7hMPw8LN78ZNHt8G2CboHdciSgA+9awX+trUXr+3qR1o3Ky7I9g7pePHNHiQiKlobQuge0GGUcfRDGaYN1BgPTunznFWOPqNb+N+X9uNAN1tVtrszjXhYxcVvX4KVh9X56ZxdnSn8/tm9+OXjb7Fe2oYQ9nSmAEHw37yw21qVM2xk8xZsmxV3s3kbXf05dA7ksKcrDcsmUGQRF52xGCsWJACwD8/KhXVYubAOumFj694h7OpKY/OOfhBCcck7ip28ZTswzOFpQlWRMa8pjNgELg05HM7kaakPwbQIMjkLx65sRka38NLWXjTGAzhmRSNOP2oBgoqAha0RvLStF72DekUF2d4hHS9v70NHXw5nHzcfuuFg694h5PKll55k81ZVArxZ5eg3v9WHh57ehYAqIRHRcPq6NpxyRCuUEeXtxW0xfOTdK/Dfv9+KXz7xFq44dwVa3SEOhxC8vmsQz77eje7BHEoNwQkC0BgPYO2SBhy+II7FbTEorpYGIdS/AgCAoCbjyMMbceZxC9Hbl4ZDqf/h8NaZaQpT6NNUyb+y4CkbDmf6EAUB7U1h7OxIQTdsvP2odrz9qGGtnrpEGINDWX9z1f7e7LgF2b6kjo6+LF7b1Q9FFnHUskZs2TMASoGeoRwOdwPDQtI57ujH5ZQj2nD4ogYMDWXH7cENBRRcds5y/OjhN/HTx7bhsOYICKXoGtCRyppoTgRxyhFtiAYVhIMyFFmEJIoIqBKaEkHfsXtQSpHJ2RBFwCHUz+EX5tpkWfRfcD1vg4JJpsbCKte04XAOMbIkYmFrFDsPJJE37aL2UEop0lkLQU1BPKziQF8WummXLcimsgYO9Gbx6s5+vLqTqXiqioSwxm7fO5QHobToe+8tT6/Kc6vKox5CRjrgkRRG3PGwisvPPhwP/nUPBtIGREFAcyKA8046DMva4xVF1Q4hyJsOHIeiORFEYyIISimSWRN9Qzpyhg0RAkJhx08LZXM2VFXEotboqKsNDodz6NAUCUvmxbGjIwnDcqBIInTThiAbaIgH0JQIYn5TGDs7U8jkrJITsnnTxt7uDF7Z0Yf//XsHVi+ux1nHzEdGt7C0PQaA5eIdh0As+P4bFvF3RU81s87Rl8KyHeQNAipQSG7nDKUUFEAiouGj61eWvS+lFA5hbZle26ZlExiWA0ooJElEQzTgy656NMaDaIgFYFgOMroFQRKQ0W1QShEJKjisJTrtUqUcDmd8NFXCknkx7OxIwXEc1EUCWLaoAbkMU508fH4Cr+8exP4epmfvBYSUMuG0XZ0pPPN6F555vRtrFtfjPacthmk7CGoylsyLI6hJTI7FplAKPHDOKC0ENxXMakfvOATZvANVETG/OYygJvs5NduhyOYtHOjLwrAIgpoE2yGwHQLH63EVAAGsD14UwSJ3wnLszXVBRAIKW7ZQJvIv1NVoaooirkkwLQcBVeabozicGiagyli+IAHRlQoPBxXf0R+xpB73P70Le7ozWNidhiiKAKVI6xZ0w8Ifnt+Ptw4kceThjdhw0kImzWwRLJwfgygKaIgFMJA2YJPi6H0gaeDpVzpRH9PQnBg9V3MwzDpHLwhssYFhOZDcAktdRBvlWBVZQCKiIRxQ0NWfRVq3ENRkxMIqgpoCVRbdvLxQlMKZjMaFhyyJPIrncGYI5b6r85sjiAQVdA/kkLccUGojl2f7cJ9/owf9qTzOPWEBjlvZDNsh0A0HC1ui/hV/a30IW3YPwi5I0xBK8dK2Hry2awDvPLb0wpaDei5T/oiHmPnNUahgAwey2xc/FoosYkFLtOLH590wHM7cRpElLGyN4s09g+h56A0AQH8qD0rZgvTLzl6OJfNiMEymvrm0Pe7LoQBAW0MYL27tRSpj+pInpuXg9V2DaIwHSk7JHyyzztEHNRmhwOzc+8jhcGqDs49ph+0wKXFCKFYuTOBtC+vRUh+EQyhSGRPBAFPwHCnu1tbIHHnnQBaL5rHi7N7uDLoGcjhjXVtVgslZ5+g5HA6n2qxaVA9BEBAt0MMxLQcZ3YYkCljgau+Xyii0NzJJ5a6B4QUmf32tE4IArFpYVxV7uaPncDicCaLIEiIhZXhlIqUIaDIWNIURDatjau+31LGIvi+p+0KLf9vWh6XzYv5E/lTDHT2Hw+FMgoWtMTgOgdc8U+lEq6pISERUDKTyyOgWXt7ei4xu4cjDF1TNVu7oORwOZxKIglA08DQRWupD6E/m8afNHXhuSzcCqoTlCxLQ8w7EKoiT814/DofDmWZa60PoS+Zx3x934kBfFqeubYNlEQQ0CfFI+V3Tk4VH9BwOhzPNvPOY+dANG4vboljUGmNrT2URi1pj48q4TIaqRvQPPvgg1q9fj7PPPhs/+9nPqnkoDofDmTHMb4rg7GMXoK0h4ssdV8vJA1WM6Lu7u/Hd734XmzZtgqqquPTSS3HCCSdg2bJl1Tokh8PhzBjmNzHFXFURx+zSmQqq9uh//etfceKJJyKRSCAUCuFd73oXHnnkkWodjsPhcGYUmiohqMlVd/JAFSP6np4eNDU1+T83NzfjlVdeqfj+DQ2RSR+7qalySYPppFbtGotatLkWbQJq065atKkSatHuWrSpUqrm6GmJ1UwTGe3t78+AkIlvSm9qiqK3Nz3h+1WbWrVrLGrR5lq0CahNu2rRpkqoRbtr0aaRiKJQNkCu2jVDS0sL+vr6/J97enrQ3NxcrcNxOBwOpwxVc/Qnn3wynnnmGQwMDEDXdTz66KM4/fTTq3U4DofD4ZShaqmblpYWfP7zn8eHP/xhWJaFiy++GGvXrq3W4TgcDodThqoOTJ1//vk4//zzq3kIDofD4YxDzU7GHsyqvVpd01erdo1FLdpcizYBtWlXLdpUCbVody3aVMhY9gm0VHsMh8PhcGYNXNSMw+FwZjnc0XM4HM4shzt6DofDmeVwR8/hcDizHO7oORwOZ5bDHT2Hw+HMcrij53A4nFkOd/QcDoczy+GOnsPhcGY50yaBkMlkcOmll+LOO+/E/PnzsWnTJvzwhz+EJEk44YQT8OUvfxnJZBJXXnmlf590Oo3BwUH8/e9/RyqVwj//8z9j3759qK+vx//9v/+3aLGJR0dHB774xS+iv78fixcvxre//W2Ew2H/77/5zW/w4osv4lvf+lZJu+655x5873vfg+M4aGlpwaZNm2Dbtm/X0NAQUqkUAFTVrnJ873vfg23b+N///V/ceeed6OzsxFVXXQXHcSAIAubPn48HHnigqq/ljh078LWvfQ3ZbBaBQABf//rXsWDBglHv75133omenh4oioKjjz4aN9xwAz71qU/5j9/d3Y1UKoUtW7ZUxaZVq1aV/dwNDg5i/vz5+MUvfoFkMolLL70U+/fvh6IoIISAEDIldr311lu44YYbkMvlEI/H8a1vfQvxePyQvlalbGpvb6/pz5xHV1cXLrjgAmzatAnz588f9f7+7Gc/w7e//W1YloW6ujps3LgRqqr6dmWzWfT09ECSpKraVY6R3/OOjg6cd955OOywwwAAjY2NuPvuu8vef9LQaeDll1+mGzZsoKtXr6b79u2jO3bsoKeddhrt7u6mlFJ644030nvuuafoPo7j0Msvv5w+8MADlFJK//Vf/5XeddddlFJKf/vb39LPfvazJY919dVX04ceeohSSuntt99Ob7nlFkoppfl8nv7Hf/wHPfLII+l1111X1q41a9bQn//855RSSt/3vvfRD33oQ0X3X7duHT355JOralcpUqkU/cpXvkLXrFlDTzrpJN/mW265hR599NHT+lpeeuml9Mknn6SUUvrXv/6VnnXWWSXf3yuuuII+9NBD9MYbb6RXXnll0XO+5ZZb6MqVK+kHP/jBqth0/vnnl3x/Tz31VPq5z32Orl27ll500UX+a3X33XfTO++8c8pfq8svv5z+8Y9/pJRS+vOf/5x+9KMfPeSv1UibvvCFL5S8fy195rzHvPLKK+mRRx5J9+3bV/L9XbduHf3Od75DKaX0wx/+ML3gggv8+9599930uOOOo8ccc0xV7SpFue/5I488Qr/2ta+VvM9UMi2pm40bN+LGG2/0F49s3boVRx55pP/zmWeeiccff7zoPvfddx+CwaCvfvnUU0/5/96wYQP+9Kc/wbKsovtYloUXXngB73rXuwAAF110kb+n9oUXXgAhBF/84hfL2rVlyxY4joN/+Id/AABcdtll+Pvf/150/7POOguSJFXVrlI88cQTWLRoEZYsWYIzzjjDt/mll16Cqqq4+uqrcc0112DdunVVfy3/4R/+wd8tsGLFCnR2do56f9etW4dXXnkF73rXu3DmmWcilUoVPec333wTS5cuxYIFC6pmU6nPXUtLC1atWoWPfvSjWLRokf9avfrqq/jLX/6Cd7zjHXjrrbdw7LHHToldP/rRj3D66aeDEIKOjg50dXUd8tdqpE2xWAylqKXPHAD88Ic/xMknn4y6ujoApf0KpRQf+MAHAABXXHEFtm3bBsuysGPHDuzYsQPnnXceRFGsql2lKPc9f/XVV7Ft2zZcdNFF+PCHP4ytW7eWfYyDYVoc/U033eR/cQBg5cqV2Lx5Mzo7O+E4Dh555JGibVSO4+COO+7Atdde6/+ucAetLMuIRCIYGBgoOs7g4CAikQhkmWWkmpqa0N3dDQA49dRT8aUvfQmBQKCsXa2traCUore3F47j4LnnnoNpmv79r732Wjz99NNYvXp1Ve0qxXve8x5cffXVOOusszBv3jz/921tbSCE4I477sBpp52GW265peqv5UUXXQRJkgAAt956K84///xR7+/LL7+MYDAIQRDwyCOPIJlM+vc/6aSTsGvXLqxfv75qNp111lklP3e9vb04//zzIQgCduzY4b9W0WgUl19+OURRxCWXXILPf/7zU2KXLMtIpVI4/fTT8Ytf/ALf+c53DvlrNdKm97///ShFLX3mXnvtNTz33HP46Ec/6t++1Pubz+dh2zYcx8Fjjz0GQRAwMDCAww8/HN/4xjfw6KOP+ifMatlVinLfc03T8J73vAebNm3Cxz72MfzTP/2T73OmkkNSjF28eDGuvfZafOITn8Bll12GFStWQFEU/+9//vOfsXjxYqxYsWLMxxFHbE+nB7mndsGCBYhEIr5dy5cvL7r/n//8ZzQ2NiIej0+rXWPx3e9+F1/96lfxiU98Ag8++CCy2WzR8av1WlJKcfPNN2Pz5s24/vrri263ePFiXH311RgaGip6f737eza1trZOm02eXd7nbtOmTWhoaPA/d9/4xjegqioWL16Mz33uc3jrrbeQTpfeETpRu2KxGJ5++mn853/+Jz7xiU/AcZwimw7FazWWTeMx3Z85XdfxjW98A//2b/826j6FLF68GJIk4VOf+pT/WhYe589//jNaW1sRCoWm1a6x+PSnP41LL70UAHDGGWcgFAph586dk3qssTgkevSGYWDt2rW4//77AWDUWfbxxx8vimAAoLm5GX19fWhtbYVt28hkMkgkErjwwgv92/zmN79BJpOB4ziQJAm9vb3j7qm98MIL0d3djauuugq//vWvYVkW7rvvPkiShF/96lfQNK3IrrVr14IQMi12efzud78reRtKKW677TasX7/efy3XrVvnF3Y8m6f6tbRtG9dddx26u7tx7733IhqNAoD/OsqyjO9+97vQNA0/+9nP8MQTT6ClpQX5fH7abSp8f++//37/c3fbbbdhx44dUFUVhBDcdddd2L9/f5FdsiwftF0PP/ww3v3ud0MQBJx++unI5/N+xH6oXqtyNhVGpbX0mXvxxRfR19eHT3ziEwBYFH711Vfj9ttvxze/+U3/tbzrrrvQ2NiIu+66C62trfj9738PAEgkEr5dJ554Il555ZVpsaunpwcA8P3vfx8tLS0lX8+f/OQn2LBhg5/2oZT6Vw5TySGJ6HO5HK644gpkMhmYpomf/OQnRR+Ml19+ueiSDGBnO++D9fDDD+PYY4+Foij43e9+5/+nKAqOPfZYPPzwwwCA+++/f9w9tb/73e/Q0tKCH/zgB7AsC4QQbNq0CaZp4q677sLRRx9dZNeiRYumzS7vv3IIgoDHHnsMH/zgB5HJZPCb3/wGqqpiw4YNVX0tb775ZmQyGdxzzz2+QwXgv4733nsvPvaxj+HII4/EAw88gJ/85CeIRqP+/afTpsL3t/Bz5zgONm/ejPXr10MURTz22GN4+umnceyxx+L+++/HunXrEAwGD9que+65B4899hgA4Nlnn0VdXR3q6+sP6WtVzqZa/cyddtppePLJJ/3bNTc34/vf/z6WLFmCH/zgB/5rGY1GkU6nsXHjRpimiVtvvRWHH364f9X28ssvj7rKqKZd3u/LOXmA5e5/85vfAACef/55EEKwZMmSsrefNFUv9xZw5pln+lXpjRs30vXr19NzzjmH3nrrrUW3W7t2Lc3n80W/GxwcpB//+Mfp+vXr6SWXXFK2ur1//356+eWX03e/+930yiuvpENDQ0V/v++++0Z1txTa9f3vf5+uW7eOrl69mr7zne8suv/atWvpL3/5y6L7V9OuUtx666301ltv9W3etm0bPeuss+iaNWvoEUccQW+66aai20/1a9nf309XrVpFzz77bHrBBRf4/418HTdu3EjPPvtsesQRR9ATTjih6Dl7NhU+52rZVMqu9evX0+OOO45edtll/m22bdtGV6xYQc8991x6+eWX046OjoO2i1JKt2/fTi+99FJ6wQUX0Msuu4xu27btkL5WY9lUjkP9mRtJ4Ws38ucf//jH9Mgjj6SrV6+mp59+etHt1q5dS//0pz/Ryy+/fFrsKsXI73lXVxf9yEc+Qs877zx60UUX0TfeeGPM+08WvmGKw+FwZjl8MpbD4XBmOdzRczgcziyHO3oOh8OZ5XBHz+FwOLMc7ug5HA5nlsMdPWdWcOWVV2JgYABXXXUV3nrrraoea9++ffj0pz9d1WNwOFPJIZmM5XCmmr/85S8AgB/84AdVP1ZHRwd27dpV9eNwOFMF76PnzHi+8pWvYNOmTVi+fDneeustbNy4EblcDv/5n/+J5uZmbN++HcFgEJ/+9Kfxk5/8BLt27cI555zj6+E8+eSTuOOOO2BZFgKBAK677jocddRR2LFjB7761a/CNE1QSnHxxRfj0ksvxbnnnovu7m4cd9xxuPvuu3HnnXfi8ccfh2EY0HUd1113Hc4++2zcdttt2Lt3L/bt24eenh6sXbsWp5xyCu6//37s378fX/ziF7Fhwwbcdttt2L59O/r6+tDf34+VK1fipptuQiQSOcSvLGfWUJUxLA5nmlm+fDnt7++nZ555Jn3llVfos88+S1etWkVff/11SimlH/vYx+gll1xCDcOg/f39dPXq1bSrq4vu2rWLbtiwgQ4MDFBK2YTsKaecQrPZLP3KV77ia5X39PTQz33uc9RxHPrss8/S8847j1LKJiY/9KEPUV3XKaWUPvTQQ3TDhg2UUupPk6ZSKarrOj3uuOPoN7/5TUoppY899hg955xz/NudfvrptLe3lzqOQ7/whS/Qb33rW9P34nFmPTx1w5m1zJ8/H29729sAAIcddhii0ShUVUV9fT3C4TCSySReeOEF9PT04CMf+Yh/P0EQsHfvXpx99tm47rrr8Morr+Ckk07CDTfcMEqlsL29HTfffDMefPBB7NmzB5s3b0Y2m/X/fvLJJ/vaO83NzTjttNN8e4aGhvzbnXvuuWhsbAQAXHzxxfj3f/93XHfdddV4WThzEF6M5cxaVFUt+rmUKiAhBCeddFKRiNXGjRtx+OGH48wzz8Qf/vAHvPvd78Ybb7yB888/H3v37i26/+uvv45LL70UmUwGp5xyCv7xH/9xwjYA8LX0PZsmK3vL4ZSCf5o4swJJkmDb9oTvd+KJJ+Ivf/kLduzYAQD44x//iAsuuACGYeDaa6/Fww8/jPPOOw833ngjIpEIOjs7IUmSv4XohRdewJo1a/DRj34Uxx9/PJ544okJabt7PPHEE0in0yCEYOPGjTjzzDMn/BgcTjl46oYzKzj77LPxwQ9+sChtUgne5qEvfOELvhb4HXfcgVAohE9+8pP46le/il/96leQJAlnnXUWjj/+eKRSKUiShIsvvhh33nknHn30Uaxfvx6KouCkk05CMplEJpOZkB2NjY246qqrMDg4iOOOOw7XXHPNhO7P4YwF77rhcA4xt912GwYHB/Ev//Ivh9oUziyFp244HA5nlsMjeg6Hw5nl8Iiew+FwZjnc0XM4HM4shzt6DofDmeVwR8/hcDizHO7oORwOZ5bDHT2Hw+HMcv4/3vtnH3kfWvEAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"requested_burst\", data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 130, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 130, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEUCAYAAAAlXv26AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABzyklEQVR4nO29eZgkVZX3/40998zaq6ur951umn0XFAVEugFBRBRcgIHBGXVcXgUZlHmdx1EZHUd0foCKvoJrKy2LgwiCqIgoKDQ7va+1b7nGfu/vjxsRlVmVVZVVXVmVVX0/j/1IZWZknIyMPHHi3HO+R6CUUnA4HA5n3iLOtgEcDofDqS7c0XM4HM48hzt6DofDmedwR8/hcDjzHO7oORwOZ57DHT2Hw+HMc2ra0edyOWzevBkHDx4c93W7d+/G+9//flx00UW49tprkU6nZ8hCDofDqX1q1tFv27YN733ve7F3795xX0cpxYc//GFcd911ePDBB7Fu3Tp8+9vfnhkjORwOZw4gz7YBY7Flyxbceuut+MxnPhM8dv/99+MHP/gBCCFYv349br31VuzYsQORSARnnXUWAOCGG25AJpOZLbM5HA6n5hBqvTP2rW99K+655x7ouo5bb70V3//+96FpGr72ta8hHA5j6dKl+OUvf4n6+nq8+uqrWL16NT73uc8hlUrNtukcDodTE9Rs6mYkf/nLX7Bv3z5cfvnluPjii/H4449j9+7dcBwHf/3rX3HVVVfhoYcewqJFi/DlL395ts3lcDicmqFmUzcjcV0X73jHO3DLLbcAAPL5PFzXxSuvvIIlS5bg6KOPBgBs3rwZH/vYx2bTVA6Hw6kp5kxEf8opp+Cxxx5Df38/KKX4t3/7N/zgBz/Acccdh4GBAbz++usAgCeeeALr16+fZWs5HA6ndpgzEf3atWvxkY98BB/84AdBCMG6detw/fXXQ9M0/M///A9uueUW6LqO1tZW3HbbbbNtLofD4dQMNb8Yy+FwOJzDY86kbjgcDoczNbij53A4nHkOd/QcDoczz6nZxdjBwTwImfzyQUNDDP39uSpYdHjUql3jUYs216JNQG3aVYs2VUIt2l2LNo1EFAXU1UXLPlezjp4QOiVH729bi9SqXeNRizbXok1AbdpVizZVQi3aXYs2VQpP3XA4HM48hzt6DofDmedwR8/hcDjzHO7oORwOZ57DHT2Hw+HMc7ij53A4nHkOd/ScGcV2CAiXV+JwZhTu6DkzSu+QjsGsOdtmcDhHFNzRc2YUw3LRM1jgUT2HM4NwR8+ZUUzbgWm5yOn2bJvC4RwxcEfPmTEopXBcinBIRu+gPtvmcDhHDNzRc2YMl1CAUmiKhLzhoGA4s20Sh3NEwB09Z8ZwXAIIAgBAlgT0Z3hUz+HMBNzRc2YMP6IHgLAmYShngU+y5HCqD3f0nBnDcYeduiAIAKXgbp7DqT7c0XNmDNtxmYP3EQQe0XM4MwB39JwZw7JdSFKRowcF9/McTvXhjp4zY1g2gSgWOfrhlD2Hw6kiVXf0X/nKV3DTTTdVezecOYDlEEhiaeoGPEvP4VSdqjr6P//5z/jlL39ZzV1w5hCW7ZY6ep664XBmhKo5+qGhIXz961/HDTfcUK1dcOYQLiGgFKWLsRC4o+dwZgC5Wm/8+c9/Hp/4xCfQ2dk5pe0bGmJT3ndTU3zK21aTWrVrPKbLZsNykBw0kIxqwWOiYqKhMYawNrnTsFaPYy3aVYs2VUIt2l2LNlVKVRz9z3/+cyxYsACnnXYatm7dOqX36O/PgZDJh3tNTXH09mantM9qUqt2jcd02lwwHKTTOog9LHuQzVvo7VUn5ehr9TjWol21aFMl1KLdtWjTSERRGDNAroqjf/jhh9Hb24uLL74Y6XQahUIB//Ef/4Gbb765GrvjzAEcQso8ylM3HM5MUBVH//3vfz/4761bt+Kvf/0rd/JHOI5DkCtYeGl3P049qoWVWQoA743lcKoPr6PnzAi2Q/D6gSH89rmDeG3fIABAAK+j53BmgqotxvpceumluPTSS6u9G06NY9kuMnkLAPDHbZ04amkdd/IczgzBI3rOjGA6LtI5C6IgoGdIx/YDQ4AAPlKQw5kBuKPnzAiWTTCUM7F2SQp1cQ1/3NYJSghvjOVwZgDu6DlVh1AK2yHI5G3UJzSccXQrOvoL2N+d5+qVHM4MwB09p+q4LkVWt0EoRV1MwzErGhCPKHhhVy8P6DmcGYA7ek7VcVyCdM4EAKTiGiRJRHNdGAXDnVJTHIfDmRzc0XOqjkto4Ojr4kwCQZZEuITwxVgOZwbgjp5TdQilSOdZxU0iogJgjt5x6bx29PP5s3HmFtzRc6oOIRSZvI1UTA0Gj8iiANcloOWUEeYJB3pycMtKP3A4Mwt39Jyq4zgEQ3kTqfiwcqUsiXDI/I3oCaEwLadkIDqHM1twR8+pOrZLkMlZQX4eAGTJi+jnqaN3CYHlULjc0XNqAO7oOVUnp9vQLbfE0Utejn6e+nk4LoXtuDx1w6kJuKPnVJ3+tAEAqIuNiOgJhevOT0foEtYk5vLyUU4NwB09p+r0ZXQAGJWjB1haZz7ieAvNpu3OtikcDnf08wHTdpHT7dk2Y0wGMn4NvRo85jt6y5mfjt5yXMiKAMuen5+PM7fgjn4eYFoucro122aMyWDGRFiTEFKHVbFliZVZ2k71UhvpnDlri72mRaBIIux5eiHjzC24o58HEErhVNFhHg6EUgzlzJL8PFAc0TvlNjtsXELQNVCYtYjadghUWYTlTC51Qyidt5VInNmDO/p5gEsonBrNdRPCumKL8/NAUY7eLnVqhuVMy2exbIK8YUM3ZyelZTkuFFmE40yuhLQvraN7oFBFyzhHItzRzwMcl6BWizschyBTsEtKK4Gi1I1bGvEO5SwY1uEvYNoOYaqZhZl39NSTZRZFARSYVOWNabro7C+gYFTnTodzZMId/TyAuLUb0Q/mTBBCkYyyhdicbkM33OGIfkQO23HJtCha6qaDsCYjU7BmPBXiEgpQCkEQhv8usssapxLHdFwoiohDfTmu7MmZNrijnwfUspSAH1FHQmwhllIKt0jgZqSjd93pUbQsmA5URQSZhRJHxyWA5+RBUdIdO5AxoI9zx2LaBBFNhm66GMgY1TaVc4TAHf08wJ2mKLga5AqsGiisyV5kLWBxSzy4Axnl6MnkUh3loJSiYDrBXYNuzmwapMR+ASXdsXnDGbNJjBAKQihEUUAsLKNroFCzd2qcuQV39PMAl9RuB2ZWH47oHZcirElIRFQ014UBsLuRYgglhy0b4Lis41YUBSiKgExhZktPHZelbgBA8P8Gc+SG6cAZ4/MVO3U/vz8d6xUcDnf08wCXALRG0zdB6kaT4bgEYa+WvqUuAgCjqlIIAdzDLBVlJY0sdaLKErIFe0aPje24eG3/EP70UicEEUFO3nJcOC4Z8/ONVLqUxOE7Ig7ncOCOfh7guAQUqMn0Tc5z9GFNhutShDXm6DVFAuDZXmS265JRUf5kMS3Xc/MsMiaUPTZTWLaLV/b049nXeyGJQtD9azkELh19F+MzMtJXFQnpfO12PHPmDtzRzwNcQiEIqEklyJxuQZFFyJIICkBTmYNXZHbqOS4FLRoRTig97NSNbjqQZSH4W6Azm6e3bIJM3kauYEMUhtchDMsJ5JnL4TgEKDoWsiTCtB3Yk2i6GsgaNS2HwZkduKOf4xDql/LV5ui6nOEEUTwFoHoOPnD0ZDiip5TCJYd/Z5I3HCjS8KmtKiIy+ZlLgRiWi2zBAqEUuuUG3bEF3YEqi2Pm6E3LhSSO/EkK41bpjGQwY45bvsk5MuGOfo5DCPXS0UJNts7nDRsRTQKlFKIw3BHrO3q3SJOeAgClhzWViVAK03YhScMRvaqIyOozl6cfyBhBA1tet2F76xCs5FMa8/OZTqndAKDIQsV5esclyOv2pGUXOPMf7ujnOH7JIsAWMmuNguEULcRKQRORJPmpm+HFWEKo10k69Q9ie9o2/n6K/3sm8vQuIUgX3T1kCw4AdvFxXRLo8JfDsgkksdTRq4pY8n7jYVguTIdUVSiOMzfhjn6OE/hEWprrrhX8DlXboYhoSvC4KAiQRCFYSAbYGoMo4LDG71mOi7KHYYby9I5LS8o5swULgADddEEhQBAEdkErc3dh2e4oRy+JbBJXJU1fed2GIglcMZMzCu7o5zh+jh4Qaq7qhlIK3XQRCSlwXRp0x/rIksicelGOHgKrH59qGsq0XAhlzuqZytO7LkXWq5QRBK+8lLKF2MCF09HrKX71UfGdSPBySiuqp0/nLe+iylM3nFK4o5/jEM85ArU3f9VyXJi2Gzh4Py/vI0tCiXyDb7+AqS8sG7YLWRRwsCeH3R2Z4HFVEZEzqp+ndwhBpmAhFlYQCyssohdYtB1UAgmj02yu66+1jEaRJ75I2Q6BaTPZBx7Rc0bCHf0cx3UJ/r69F4bl1lzVTXENPUChyFLJ8yyiH3ZK/t0JpXTK6w2WzTpiH/nrAWz9/e7g/QVBAJ2Benqm1mkhGVURjyjIFGwAFJZDgoVoYHRl0ViVOACgSCIMa/y0k2E5AGWpoVrtqeDMHtzRz3G6Bw38/oUO7DyUrrkfd8Zz9CFVgiiJoyJ6RWb5Z1oS0QuAMPXUjeWwZqmewQIKpoOdh4aj+pnI01uOi0zeQiqmIh5RkS1YXtOUG0gzA6PvWFyXoqs/j//esg17OjMlz4lFTVdjkS3YJRU7h9uLwJlfcEc/x8l7IwRN2x03KpwN/IheU8RA+qAYWWI15b7L29OVwZbf7YTjTk3OgVIKxyEYyJpBCeOLu/qD52ciT29YDrIFG8mYhkREQbZgQxQFuG5p/n3k57McF10DBWQKNn76+E7s68oGz4kiW38Zz3ln8iY0ZfjnfDglqpz5B3f0cxxfNMyyq6dgSenUHG/WuwipsoSwKo16XpHFkjr6PR0ZHOrLI1ewp5S6cQl7ry5vQtPytgS2HxgKoviZyNP3p024hCIVUxGLqDAsF6IgIBGVQQjFgZ4cgNGpFct2kSnYkCURyaiKH/92B/Z3Z4teQcesRrJsF5ZLg5JVoDab5zizB3f0cxw/arYc97DKEscjbzjoHdInvZ0vaKapEhRl9KnGHP1wHb1fWWJYzpQclUsoIFB09RegyCLOPq4NLqF4de8gABZRV1P3hlKK/jTTkE/GVCQirJw0p9tQZAmv7B3A9x9+HYMZc9REMNMmyOQt1MVVvP/tqxGPKLjv97uLonhhzCjdcgiEouNF6dgXBc6RCXf0c5yCUf2I3iV0Sm31vuaKpkglkgQ+iiTCIcPV//6Co2lPbs5qYKdLASqgc6CAlvow2hqjaEyGStI31dS9cQlFumACAFIxDfEIm6rlr1Uc9KL5nGGN0rsxbZfN1vW2O/fERcgWbOw4kA5eM768cXGDGLiOPaeEqjr6b3zjG7jggguwadMmfP/736/mro5Y8obvHN3DVn0cC9txp9Rt6S9EKpJYklbwGZm68ZuCWEQ/eTtdQkDBIvoF9REIgoCNKxpwoCeHwSxzwIoiBCml6cZxmZgZgKDqBvCbpoCOPpZSGrmeQim7kA7lzGC27qr2JBJRFc+90eu/yhM9G41uOCULsaLIm6Y4pVTN0f/1r3/FM888gwcffBD33Xcf7r33XuzevbtauztiKXb0pEqLsbZNphQh5nUbYU2GIAoQyzQCyZLIUhMjUjfss0wtdZPOWbAcgtaGKADg6OX1AICXdrOoXpUl5Ap2VXSBHG8YeUSToSoSLIsds2zBhktIsHZgmG6JJr3rDSSxbIJUjN0FiKKA41c3YndHBgMZA5IojqlhY9jDFT15w4YkCLB5RM8pomqO/uSTT8Y999wDWZbR398P13URiUSqtbt5jWm5Y0rPFkz2uGm7qNZv23HJmNK645E3HIQ1tgg7UqwL8CJ6QoPo3Y/ofV2YyWJaLnrSzJkuqGfnWjKmYWFTFG/sHwIwrE9v2dN/sFhEbyEZU0EIhaqwktJswULPoBFo3OiWU5Jvd1yCtJfeScW04PHjVjVCEIC/vcF07ceSQdC9sYk7D6XxXz/bhqGcySN6TglVTd0oioLbb78dmzZtwmmnnYaWlpZq7m7eYjou8kZ5R6+bnnO03KrVTluOO6VRhTndCZqlRmq4AMWpG/be/iKpZU9tf5ZD0DtoQBIFNKVCwePrltShs7+AIS99A0onbECaCqblIlNgeXaXEGiq7HXH2ujszwevM8zS78pxaVD2mYoPO/p4RMXaxXV4YWc/KKUwrdHfrz9GUhQFvLFvCJQCg1nu6DmljC5unmY+9rGP4brrrsMNN9yALVu24D3veU9F2zU0xKa8z6am+JS3rSZTtUvJmsjqVtnti6cXxePhaf/sTU1xdKVNyJqL+oZYWYc9FqbjorU+imQygtaWxCgdl0Q8BJdSJOuiaKqPBM6dQkQiOfZn8R93XVKS+x8sOBjIWljQGEVjw/C2J29ow2+fO4j9vQUsW1QPLaRC0pRpP1aRqIZswcbGlVHEYmHE4wJScQ0F00V/hunQhDUZLoBYPBTsX8oYML0If9nCOoS04Z/lW05YhDu2voiDAzrWLq0fZbNhOkglDcQjKvZ4tfdEEBHx7gxq9bcwEbVody3aVClVc/S7du2CZVlYt24dwuEwzjvvPLzxxhsVb9/fn5tSnrapKY7e3uzEL5xhDseudN5CrmAhNMLHUkqR9275DdPFwGAePT2ZssJYU8G3ua8/B0openoyJW38E5Er2JAbgVzWQF9fbtTzjuXAcQj6+7OQXBd5Lz2VyZvo68shXCbd49tEKcXergwWt8SDYR1dvRkc6sli3dI6DA4NR9AygOa6MP7+Rjc2Lq+DSwgGhwqIysK0HqvdBwdgOwQhRUD/QB6RkAJNEdHZV4BpOWitD7PqmqyJvoE8er18fPdAAb0DeYQ1CbpuQtfN4H0b4wpURcTO/YNoSYbQ1Z0uGU6S020MpXX09OcwkGGlnd39eQwOsAvnQP/o417r1OJvuBZtGokoCmMGyFVL3Rw8eBC33HILLMuCZVl4/PHHccIJJ1Rrd/MbSsumMigFDJulIIg3sGO61xiDFIMwOf0UhxAYFkvdqGVq6IGiHL0XzQY5emviCiLbISiYbpBrp5RiMGNCt1y01o9eC1q7OIX93TnkdRuSyPY73emNvkGvhj6qBWMTYyEFWd1G96COtsYoIpoM3XRKvk/L9mUTtFHvKQgCEhHV60kYPZTFspnkwy5P6kGWBGTzlieHzNM3HEZFjv7mm28e9dhHP/rRcbd585vfjDe/+c145zvfiXe961047rjjsGnTpqlZeYRDKMo6epcQGJYbLHiaduWNRoQybZWJsG2C3z1/CEM5a1J580zOBKVM50YeNR6PoXoDwu2i9BPAqm8mWm+wHALdcIJtXELRPcQWYlsbyjn6OgDAGweGgscqkf6tFMclGPLy7MmYClCKkCIhFlZACAUhFG0NUURCCnTTKdGk1y0X6Vx5Rw/AE0ezACqMaoQyLFZaubsjjbq4hpa6CDIFCxRcBoEzzLipm1tvvRXd3d3429/+hoGBgeBxx3EqKpX82Mc+ho997GOHb+URDlNzHO34CoYDSlmlhm4WYE2i0ch2CDIFGy31dNz0RddgAc/v6EMyquKE1U0V2zyQZU5PU6VxI3oAQdmgVVJ1M/7nsGym1qkbNpJRFS6hGMywfTalwqNe31IfRiqm4vV9gzh+dRMkUUBOt5CIqhV/pvFwHIJ0ju2/Lq7BslwW0UeGh620NUawvyeHguEEmvSSIMC0HKTzFtYsSZV973hEZdo3wuimKcMiEECxtzOLjSsaSrqYp7KgzZmfjOvoL7vsMuzYsQNvvPEG3v72twePS5KE4447rurGcRiE0rINRH7JZSqmobO/4NWfV/aetkNgOSRwNmPhOy/TdifVxNSfZs4mpEiQ5TEi+sDRs4jXj0ANy5nQSemmg5AqBdVIrkuCwRuaMlpXRxAErF1Sh7++1gPTcqEpIjIFG22Vf6RxcVyCoZyJSEiGKouwbAJNkRALs59YJCQjEVUR0SRYDoFDiBfVE+R09nnrxonoswUblJJRTVO65aB7QIflEKxYmMDezix2HUoHx4TDASZw9EcffTSOPvponH766WhtbQUA5HI5ZDIZtLVN10+EMxGUls+9B44+zqJSaxKa9IblwHHYhWG89VV/Xikr36zc0/d5mi8hVYIqj3a8AIILgOO4QRpFlgQYljthg5ZusRr9gsk+s0MoMnkTdbHhCJ14ZYc+K9oSeOaVbhzqy2N5WwIFkzUySWOkliaD7bALTV1cg0soNEWCLAmIhZk9bQ2sUzcSYhG+YfkXZYp0flg2oRzxiOoNPSclTVOOS0Bcij2dGYiCgKWtCfRnTFgOgWVPfAw5Rw4VneEvvfQS/v3f/x25XA4XXXQRLr74YvzgBz+otm0cD5ajH/2jLY7ogclpxOiG490pjP/6dI45IWMSTUyOSwLbQqpc4myL8SN60ybBQmwiqoJSwLbHvmgRb0QhqwBii6qO52j9OnRCKAYyZolGT1sj65Y91OetTRzGgJORmJaDdM5EfVyD61JoqghBEJCMadAUCUtaWWlexCud1E12/G13eJh4Kq6V/f58KQXdcEqapmyHAAKwuyOD9uYoNFUKhNTyhg17CvpEnPlJRY7+rrvuwuWXX45HH30Uxx57LH73u9/hwQcfrLZtHA86RurGlz8IHL1TeXolbzoQhYnnzPpOiDUxVeYVDcuF4QmHhVQZ8hiO3p84ZbskaGDyc+bGODIIfvpCEASACrBsNrIwW7CDY2E7BMmYWrLgGtZkNCQ0dPR6jl6YPjnfTMFGVrdRF9fgEIKQwhx6WJVx/YXrcOp61iwYDpU6etfrpgWAVFRFrmAjnbdKFDbjgfN2SpqmHJdd4Dr7C1jqXUji3vHL6TZM3jTF8ajI0VNKsWbNGjz99NM466yzEIvFqqIVwikPJeVLG/38dNL7cZsVpm5cQmA7rNloou8xSN3YBE6FwmaG6UC3XAgCoKnimKkRf1CGbbvQPccWDyvB/sayzXKG9XEEkaVx+jPDOvAAkzhoSIQQj6glapULm2I41JcHpRTCNA5U7xrIg1K2EEtclrIC2IJzWJODYxBE9Ba7kJneRKp4RGGpLEHAoibWmJbOWyCEIuGpYBZMu2Twt+kJoQHDC9DBaw2HR/ScgIocvSiKePjhh/HUU0/hjDPOwO9///tq28Upgng5+pGOz9ei96taLNutqI4+0HmhGK2LPqK00d/HZNQxswUblu0yQTNgzNSNL11suwSGp9njT6IyrbEXli3bDSqFFFlEQXfQ51Wa+KkbKrAIvrU+AtsZvmi0NUaR021kCjYopj6ycCSdXjqoLq4BwvD6gyqLcIv24Tt6wyuxNC0SyCbAsyceUbFiYRKt9WFkCzai3oJutmCXpPFMyw0uxA1JJvngXyhzBXvC8YOcI4eKHP1NN92ELVu24JOf/CSamppwxx134JZbbqm2bRwPCnhDs0sfzxs2ZEmALIrQFLFi1cdhx0dHvb4vowcpIWB4HcCwKsvRE0qRM2yYths4tXKCZsBw6sZySJBiiUf9xcqxewJ0c1iWV5FE5E0HvV6VT11MY4uwAivtZOmaEAoGe/+FjazGvqM3X/ZCNxVcQtA/xBaf6+MhADToINZUCbTosGlepM/0bihM20E6byMVU0EphSgIkCXWsduUiiARVWGaJNDM8ZumCKUomE4gv9yQYBcKWRYRCcnI6jbsKgi3ceYmFUkgPPnkk/h//+//BX//9Kc/rZY9nDJQbzgHoRRi0YCJguEgpMqsC1NhZXuV5NF9R0nI6MVY0yKQRSdIAfiOnnWrTvze/uInU66UIUpiWYliYLiO3i529N5+x0vdFDy1RsBTo3QJBjLM4SVjKmyHIB5Rgv0214UxkDFBKUVLfQSSKOBQXx7tTdFpSd04LsVA1oAii4iGZeR0J7hbkUQBxXsoGA40RYRuO3AIQd5wkC2wRWTXpVAVKbhbEQUB7U0x7Dg4hFhYZrr2VEBetzGQMaCbLgazJpJRNbhoAkAioiCn26yE07t4cI5sKoron3zyySqbwRkPAlq2xDJvOEEuWFWkihqNAOYoFYlVhYwswXNcEkT0LqGsuQcsdVOJZIBpuaCEoHtAR2MqVHaylE9QXumSQIXTX3hk6w2jtyGEwrBcdA3ksdcfoC0IyOQtJCIKZInVsPsXKoDdOTSlQijorFKnpT7CKm+E6Wkqsh2CwYyBurgGSplz99NVI9cnRIEtUBumC8t2MZAxQCnQlAzBIWTUbF1FFrG4JY6IJrNaelAc7M3BIQSJqIL+jIHG5LBSp2E6iIYVtsBLeS09h1FRRN/e3o5rrrkGxx9/PKLRaPD41VdfXTXDOMNQAniKLiWPF0zbkwH2IvoKxwkWTAdhVfJ05ktf77jEa46iyOsWbJcgEpJRMJzA6Y9H3mDVJ6bNNGeUMZqlgOGI3nGGZYP9BiPDcsp+FttlC7G/+esB5HUH//LujRAAphXjS/x6zrSYhmQIvUMGCKFY2BjFtp19oNPkCAumjcGMifqEBsclJftmKSb2OXy1zbCnd2NaBP2eEFljMszKMrXRPQeaJ6XQOaAjFlYgCKziyJ9Ru2hVY/Bay6WIago6+gqgYGWoyhh9DJwjh4ocfSqVAgAcOnSomrZwxmBvVxbRsDwqwtUNN9BV0RQJOcOuSAyMuASiKI+K6KknnkZBYdsEupcOqYtrKBhOUBI4XirAtFk0DwCtdeHxHb00HNH7qRtVkqB5dyflUje+Vn33gM5kD7KsGzWdt7C8LQFCqFftMzIyZlF9f9rEwqYonn29B4NZA811o+USJksmZ2Iwa2DFwgRcQhELD39mWWIloABbiwipEsKq5A0fGU45NSRDMEwHmjL6JylJAqJhNTj+vnZQ1ltw9SN6tjYhIBaWvUojAdnC9Mk8cOYuFTn6L33pS9W2gzMGAxkD9/zmDbz95EU4enlDyXO65aBVY4uLmiqhP2OA0PEjVFaeN5wDLs67E0q9ZwSYjou0J5dbF9NwqDfvVcJQiGMsrrL3ZyPzFFlEKq6Nm7oJcvQuASzm0ERJhKZKrJyzzEXLtNgCpJ9y2duZwdErGpDT2YKm5bgl+fli/Kh+gSd61j2gY+XC1DhHa2IIoegZMuC4FPXxMhG9KEIQ2EXUdgiaUxGENBkDWROEUgykWcpHkUUYpgClzLEVBQHJqD9/1g7myvrdx37FjWm7iIUkxPx1DstB3nLQ1jisZ2TaLkRBGPcCzJl/VOTob7jhhrKP33nnndNqDGc0Ba8G3LRGl04apouwKkGURKahYrtwy5RO53Qb/WkDDclQSZ5dEFDyeuIt+ooi68L0KzrqE35D1sSdt5btorO/EDjTSlI3rkvhui5Ub+E2pEgwbafswnLBcNDnVdhIooA9XVks9pqF6uIabJuipa58BKvIEhqSIQxmdYRUCV0DhYoWmMfDtN2ge7gurgF0tBNVFSm404iEZE+6wYHrUgxkzeEcu4Ax9f79tFS2YAWOvj/tp33Y9o5DkWoIIRYevigoImW6O94dTkdvHvGIgsYywm+c+UtFjr5Y0My2bTzxxBNYs2ZN1YziDOOnVmyXlDh623FhuwQhTYYiiQhpsjc3ttTTU0rR2Z/3aq5NVr4nF0X0Rakbw3TRM6hjUTOrNR/MMkdSF/ciRk/vRkF5XMKGiHcNFHDSumZQOrbjAkpTN45LoMhsxqqmSmxfZRq0LMdFzxCrcFndnsTeziyOWVGkFSOgbPrDJxZW0JfWsaAhgu5BvaLF6/HQTQdDRaqVEEZf3BRZhGk7kEQRIVVCJKTAdgjypo3BrIlVi5LBBXQsAbj6mO/oh0dK9mcMqLKIWFgJto+FlSBVk86ZaIprKJh2cEyH8iYggDv6I4yKHP0ll1xS8vell16Kq666qioGcUrxu1Fth4AWLcYGWjKKBEUWEdFYDn/kXNGsbkM3XSS8W3/HIUENuiACblGt9VMvd+EXT+7EJ969ERQEg0GO3pMlsJxxtWFcl6IvzYZgL2yMQsDYNfQAK42URCHI0SuyBElkjU79ab2k0cjHtgm6BwporY9gWVsCr+wdxE5PrZFFvXRMWWSAOV1BEBALKxjMmoet2Z7X2eKzKLDSzoLhjo7oZRHpnIuGRCjYNwB09hXgEoqmZBiEUCjy2KWo9V7Uni1YwWN9aVZxIwhMBiISkiFLIhqS7KIwlDOxsD6MTN5CXTyEwZzB+g4Mm3UG87LLI4YpJeoIIejp6ZluWzhl8CNud0RE7zt6VRGhyiKinvMoFLX7U0rR1V9ASC1aHPQc3S//sBu/fe5gSUQ7mGWlft2DLDXSnzFYBOpV9pjjCI0BrFSxa4AN/2hrjAKCMGENtywJQaWPIguQRBFhTRpTwdKw2V1Ha30k0Hd5cdcARFFAVJMhiuK4dxH+RSCkyhWpZI4HpRRZ3WaONBEKPs/IkkpVEWFaJNCm90tIu71j1ZgKwSFsUMlYJKMqJFEojei9dBzAup399E4ionkXFwuqJ8fsuAR9QwYiIRmUFnVHc44IppSj3759O04++eSqGMQpZTh1U9rcFDh6VYKiiIiGhlvrfbK6DcNyRlVd6KaDl/cMoK0hOtyIJQjBzNbuQR3NdRGkcyaiITmYBGWY43fe+tUwYU1GKqYipzus6mQcZEn0OkRdyDLTxYmE5LLa+pRS9AwUYDkECxoiqItrSEZVpPMW6uMaCKWBhMJYSKIIVWbDUCrt9h0L2yFwCcVQ1kRDMgzHLb9/VZagKEKwSJv0Fkt7PNmGpmQYrksQGqc6RvFKLH1Hb9lM/sB39JQiOAc0VUQsoiCdMz3hN4reIR2EsrsoStnxHlmZxJm/TDpHLwgC3vve9+JNb3pT1YziDGN7jsgZGdF7P/iQLEGRpGGd86Kom0Xzo3/MOw+lQakX/VMElTT+xaN7oIBjVjYgm7cQCSlsDUCVSt67HK6Xn1/YGBnu7hxD58ZHkUU4LmuCioZkyBIQDSneY6V1+y6hgaZMq6fvvnRBHNt29iMV1+C4FInIxM7LHw4CDNfrT2RnOQxPW2gga2JpWxKOS5CKjtaUZ7XzSuBYk57w2kFvYVRTJWTzZFTtfzGyyFI+GS910++l1RqTIaapLwnBwBVNkRELKxjyFtMFQcBg1gzu7GSJXdR52eWRQ0Wpm0suuQRvectbkEql0NDQgOOOOw6SxKOBmcDPITtFio3AsESxpkqsztqL5iyLVcZkciYMywmi8WK27x8CwCpYIAwLe/mOvmugAEUSmaBWSIaiMEdverXfY5HVLQxkDbQ1RgPnOdFQD1kS4bhsUIYqixDF4bsTX53ThxC2sCyJAppSLJJdtiABAEjFVLguDRrIxiMakoNhKBNdvMajYNjQTRuG5aK5PgLiUoTK7F+WBCQjapDGKpZSHh57SMetUJJElp7zL/D9RaWVtk0Qj6jDQm+SgFRMQ89gAYRQaKoEy3GDc0GRJWR1u/yOOPOSiiUQzj//fNx999244447cMEFF+DZZ5+ttm0cDGuv2y4paZjynaDm6b1HNBbRW44Lx6Xo7C8EufViXJdg56EMBMGbzUqGh2/kdXbx6EsbEAS20BgNK5BFEWFVZrXt4zj6fZ1ZUMry87ZLyu5/JIrMUjeWTVjqRhICvZtieWHAWwPoL6C5LhxcQJa2xiEIw7XkldSHq4oEzYtuTauybuJyZAt2sJ6xbEECEISyfQMhVcaCxuGB5fGiSHq4tFIYd21BklgjVKbAFlL9EtP6eIiljIo6aiVJxKLmKAzLxaG+POtpKJpeJUuC993zPP2RQkWpm2984xv44Q9/iFWrVgEAXnnlFXzuc5/D1q1bq2ocB4H+uD1CsCyv26wDVBHZQqQvHWA7GEgbsF0SDLkoZm93FqbtYmV7EjsPpmGYdhDRFkybpWgsF10DOkyb1emrite2bznjliMe9NIqCxqYNHB9PDTma31kSYTrshF5qixBlkREvTSUbrol1SGOS9DZl8faxalg+0RUxTUXrEVTKgzTJhU5ek0Roan+8ZpaRE8IhWE66OzLQ5ZEpnPflR6z4qe4wkWVxaD7t3iQ+XjNZZLIxhI6LsGDT+3FK3sH0ZQKsUYryy1J+yiSiMXNMQgAdh1KY1FzbLQtlEkkR0K8cepIoKJvWRCEwMkDwPr16/ngkRnC9lM3IxdjDQeaIrESRnHYOVo2QV/GCBy/Zbv41dN7sb+bCYBt35+GLIlYv7QOAHOmTO+eomC4WNLCKll2d2QAsFJHWRYRDkmeJv3YUeBg1vTmpCqgBGXXB0aiyCIbBehSKF7DVLRI76a4/LEvbUA3naAZy7+7WNgU80oTx6/b95ElERHPNtOsfKB6MZbjAoKAA715tHlrEqI4flTu41cWASyiJ4RCloRx1wkEQQjKXF/ZO4Cjl9fjPW9dGTxffJFgF34V7S1x7O7MjPWOo9ZAOPOXcSP6oaEhAMCGDRtw991344orroAoiti6dStOPfXUmbDviMcuTt2Q4hy9HThSSRIQ8atuLDdw/gDwp5e78PftfXhxVz/e9ZYV2H5gCMvbEsFUKt2rpPEd/sKmKHZ3ZrCrg9WmRzSWz45qCpMqHkfBcihrIhH1c8W07PrASBRJxKCvc+PdncSCi5YbNFIBCC5WrfUREEIxmLWQiqnegi7TkamkNlwQBCS9u42pRvSWTWA7Lrr6CzhtQwscl1S0PgCw7yusyRjKWWhKhUbJJozFUUvrAQCr2pPB4jsAgI7O74dUGasWpfC7vx2AYTrB2oF/h6TIAvKGjfrExHddnLnPuGfXqaeeGqjkAcB//ud/Bn8LgoAbb7xxRow8kvGjVschJQMsdE+L3td7FyUBmiKCEBpExOmciT+/3IXVi5LIFWz87PGdAICzjlkQOArdZFOLdC/nHwnJaE6FccibqxoOseHekZAMyyYYK0XvEoKsbiMZZQM0BFEIKlvGQ5bFILJUZAmigKDe3LBIiYxwr1eO2JgMwXJcJCKK12jFKnd8PZhK8Ad1jKWSORG66XglixTtzTHYDpmwtNNHFARENJn9C7EF1krSXCFVxppFqZIFX8cl0MoMYNcUEcvbEnjiOSaKt3ZJHbYfGMLWP+zGP160HomoEiy+c+Y/456Zr7/++oRv8Ktf/QqbN2+eNoM4pQR19A4piTzzhgNVEUtu2f28rx/VPv53pjZ6/imLEdZk/OyJnTjYk8eqRalA8difXZorasRpTIWYXjuASEhhi70hGbbX2FQO16XIFCy01CVhOwRRTakoulZlEYY5HNEXd476Eb3PUNYKJBJyBQct9SF0DxaY6maFFTc+iQhrQKp0zu5I8oYTqHQuaorCJgSJMhLDY3Hi2iZk8uyYU4rgjmw8VFlEZoStjsuO9Ug0VcKCxhhUWcTujgyWtyXw62f2w7IJ9nVlceyqRtiOA9vhMsZHAoe9EnP33XdPhx2cMRgrdWNYLEdfHDVr3kIqABzsyeHl3QM4bX0rUjENmiLhqnNX42PvOhqxsIJwyCsv9JqG/Nb6aEhBQ9HtPOs2HU4NFQxnTPngvM6as2yHBHcVEyFLYiDswMorWT24LLHRiKY1fGFJF6zhMkKBTaOKR1QmaQxUlCryYZU33rrDJJumKKXQTRuH+vJoSGiIhBQIdHL7P2pJPTauaGDHsoyscjlGjiUE2NpNuUV3VWZlt0sXxLGrI4M/vtiJdN6CJAro6M8HryuYYw8QT+fMaRuezpldDtvR80XZ6lIc0ZcIkHkpi+LcLCuBZJUqv3n2AGJhBWcc3Ro8L4pCkBbxBbYMi4208xtxwqoUVIJoqjTcrepFjWPJIPR7zTnJqAqXoGw9eTmKL1SqwlI3giCwun3bhWEXdfp62ur+OaepEuoToaCdf7yqlZFoihTsY7LCZn5H7MHePNr9ihZhcvtXFBEuIbAdVoY6Ub8BMHosIcCG0mhlLjCyJAIUWN6WxGDWxNMvd+GYFQ1Y1BxDRx+TXmAyCWbZfbmEoKM/zxds5wmH7ei5MFJ1KZYVtorUHJk2jAilqJwvrDH9llf3DuJQbx5nH9c2bpQZ0WToJnN0fmt9WJOC2u54hA01kcTSdEq5KpV+r647EVUBgUKrMB1QfKHSiualhjy1xWKRtmzBDmbCxkJMcz4akoNGpMlorIuiEOjdTLae3HIIhrImdNPBouYYW5PA2MqT5VBlKegfSFbYoVr2YlBGLRPwBp4IwPI21lCmKRLOOakdCxoj6B4owHWZdHEmb5WN2k2L3aEVRvQylJszzKl9eBFtjVNcXmh50S2hzEEokhh0eALM0euGgyf+fgjNdWEcs7Jx1PsVEw6xSUQOIcjpNgSwBb+QKqE+riEWViB680+jRVU95X7ofUOsUzMRVSFAKLkAjUexk1K81I3/WQzbDfoICKFe274GyyaBMJgkiqhPhBDWKqu4Kfn8gXha+VRUOm+VvWM1LAed/Swqbm+KwfU6YiczhFuWRVAKkArz80DpWMJi1DIXVVlmY+QbEhqOXl6PzacvQTSkoK0hCtcbliIIAluILxO15w0Hoghk86ULtj1DBWTz1qjXc2qbylevOLOCXTQZxPBSFGZQjiiVVFtEQnLQ2v6+c1dNqN8S0WRk8hZcl2nnaKoEVZGgyBLeclwb6pKRYKHOz7mbY1Sp9Hva9WGNjcqr1OkVLwSq8vB2iYiKjv48CPFGDZouLIcgEVVBKEry0nVxbVxp4vE+f583HcpHNx10DRSCNYt1S+pGLVbqhoPOgQI0hY0nNCw3UA+tFFkUAAhlxx6OhSQOjyUEWJdz8cWxGDZFSoJlWLjkrOXB422NbOZzZ38eCxoikESWEouGSu3P5E3Ewgpyhh3IWVDKRjfKKR4fzjV4jr7GcZziiJ45eF8aQJUlz2EwfMmB5W0JrFyYnPC9I35E77KIPqQyJx8JSVjelsCqxXVBA1A0EE0jgdBaMYMZM6hjn4zTK11jGHZ4DQkNmbwFCupNYmIXEj/NUZyXDmsyGpOTH6QRCSmjpIr7hnTkDdu7YxBglxl+kjNsJt7WFGVzd70L0GSQRJajrzQ/D7C8uz+WEMAo6YORJGMqzBFyxKmYipAqocOrqtJUKRic4uO4BAXD08bxlC4BdjenG+N3R3Nqk8N29BdeeOF02MEZg+JO1OIfHMDkaIudRCquQQBwzontJe9hWE7ZmumIJrORdoR4DVgyFElAWJXhuhTEHW7EiYWZI7Mdt+yt+6DXLOW6NFi4rYTixdjiBdyGVAguocgbLLU05C0ahkMstVRJB+pERD05ZD89BAB500HYu2AJoKMuao5LoBsueod0LPSiY0CoqOGpGEkSvFmwk7tAKLIY3FHZY5RW+iRj2iinLAgC2hqjwYKsLImwbbekbJadXzR4vZ+nz+s2XAKukTMHGffsfP/73z9u3vOee+7BtddeO+1GcYYp7kS1/Yjey6lqI27bz9jQimhIRmv9sIAWpRSWRaB6FSbFkXAkJDM5YNMNInpZElmqQhDgUgrFGzuoKkxiwHYI0jkTCxqjJemZTN5CKq5NevC0n8uXJSHYF8A02gEgk2NToPz5tWFVDqZlHS7+XUrBUwL1q2BCKntclAQYplPijC2boCetB+JthFAIIls8zk1i35JXRlppft5HVSTYjguJBdtQx0n7REJKyVQynwUNEfz55W44DvEWkAUUDDs4N3K6FQyAVxQR2byFhkQIA1kTYU3iEf0cZNyzzB8X+NhjjyGXy+Fd73oXJEnCAw88gEQiMSMGHunYLoUosEU703P6wxG9XDKqzy83LKZguGhMhZGKqdhxKA3VmzBlWm6wrlcwHeS9rlZFEb18N4VLCMLej18QBGgqWyB1CIVpuUGDEiEUmYKNJQvioGXa8cfDX0iUpdK7E3+maaZgw7bdIL0QCSuj8slTxa8k8mUQLLt0+LksiaMUNG3HRXe/P0UrElQATXYhWBAE1Ce0Sd8JqF4nMSGUOfoJhq9HQvKopqi2xigIpegeLGBhUwyaKqIvbSAeUSFLItI5O6iaUmUROYMNsDFsFyFFLOlW5swNxj3L/IEjd999N376059C9H6Ib3nLW/Ce97yn+tZxWIu7IkG33KA71p8iFVJLKz2kEYtyzBlQT+VQQnMqjL4hA7IkQBCFQC5XN1kZnaayQeO+0yWEQpJKc+iG6UASROR1O3D0ecOCabtIeM1MI+0YD/+ioMhiyb78Es+szt57KGdAEgWEtelJ2wBALFxUSUQoLIeg2HJZEkZVpOQNGz1DBSQiCuIRFbmCPeVB2811kYlfNAJNlWBnKECZBES5iptiUlEN3QM6iuelt3micB39zNGrioS87mBPZwYLG2OwbCeY8qUpEghlazAC2AVqPGE7Tm1S0S9mcHAQpjncWJHP55FOp6tmFGcYx6t3Bjy9G0qDnKnf3eozsvoip9torR+unGlKhaHIrBpjRVsSSa9EMVuw4bgUEW24iieiybAdUuK0/Tp9VRUxkB0+H3q90sp4WIGqTK7M0W8yUiSxZGE5rMqsiihvw7QJ0jkL0bAMAcKE4wkrxY/oTcsNul2L75AkkWnoFOekswUHXQN6UL1CPVtnioZECBuW12Pd0nosXZCYuLIqpIAUtdPaDhsiHgnJwbQugFVVOQ7Bvu4sKAT85tkD+O5Dr7IeAQqk8yaTxBYwJbVPzuxS0Rm6efNmXH755Tj33HNBKcUjjzyCyy+/vNq2cTAc0QOA6bDRdX46YeRgD1EQIIDl5X3Vx7pE8cAJEcvbkpC8AdYJbxiFX9ESVuXAsUfCMoZ0Z4Sjl5A3HCiyiEzeClICB3pYdjoWVkoqZyrBj+g1VYJUlKMXRTaVKZ23YNoO0nkLsbAKQUBJ5H84RItTN4QNXil3t+A4FJLKvot03sBg1sRxqxqLOnRnrtxQ8L7jSglpbL2FUgqXUOgmgQCKtoZIoGfkEw7JMC0XIVXE6/uGkNNtL6WjIKc7aEhooBRw3LFlEzi1SUVn6L/8y7/g4x//ODKZDLLZLG666Sb8wz/8Q7Vt44A5mVBJRM+khQGMEvESBAGSJDDZYcPF4pb4qNI9VZGCx+rizNEPZoYrWvznQqoMRZZGRfTF2jN53UZfWsfBXuboI2F53MXBcpSkbopslUQBiaiCdN6C43XuxsIyJE+tczrwm65Mrzt2b1cG33v49UCLHwBAh+f2mraLnkF2UWxrjMJxCSKaVHF55GwgCuw4GpaLvG6jtT4MCAJWLEyid8gIFEF9/HJLv0prd0cGmiIhFVXRPajjW1tfCiqgOHOHis/QpqYmrFy5Ep/5zGcqXoj91re+hU2bNmHTpk247bbbpmzkkYxDhiN6y2WpG8N0WAv/GBonmbyN5rrwhIuWyagKAQjSMBFtWO5WkUSERjVkKTDsYaXJniEDh3rzMG0CQQAiqlLWpvEIHP2I1I0oCkjGNKS9Fv2cbiMWUipuLqqEYDHWcmDaBLs6MugZ1PGjx7bj2dd62IuE4Tpy3XDQPeAtxDZEvA7d2h+wnYyoyOkOWhsiwXrCUUvrIAjAi7v6R72+eBbB7o4MBEGALIt4fnsvhnIWBjIGFzubY1Tk6O+77z589rOfxXe/+11ks1n80z/9E7Zs2TLuNk8//TSeeuop/PKXv8T999+PV155BY899ti0GH0k4ZfASaIwHNFbDlRvvupIZElELKJUtNCnyBJCmjRcuqgNR/CqNxBcKnH0wxG9pkjIGzaiYRnZvIV4WIEoCZPSe2E2DEf0I3P7DQk2fSmdt4Lu03ICXlMl5KWqDMuFYTnoTxtIRlWsak/i13/Zj989fwiyJAaL31ndRs+QjoZECCFNhkto6QCQGiUaVtDeFEVTKuLpA7HjuKItgZd3D4xqetzdkUFjMoR1S+uwtysL11NOfXXvIICxhe04tUtFv8of/vCH+NnPfoZYLIaGhgZs3boVP/jBD8bdpqmpCTfddBNUVYWiKFixYgU6OjqmxegjCZcwUTHVG7lHwaZBqSNSHT51cQ2LmmMTLtIBrKokrMlBxBouiuglUcSyhakS5xv1nJvtEG+0ncbK8fKW1xlKJ10R47++XCt/sxd9dvb72vhyReMJK8VXyWSO3kV/xkBrQwSXn70SK9uTeH57HyuxtBwQSpE3bHT1F9DmDfoWUF45stZQZAkLGob7HmIRGZZDsHFFA9J5C/u6hzsAHIdgX1cOy9sSWNGWgO0QHOzNY09nBnmv38Cypz5QnTM7VLQYK4oiYrHhAcMLFiyAJI1/ghfPmN27dy8efvhh/PSnP63YsIaG2MQvGoOmpviUt60mU7HLJRSRsMpSFoKAhoYYKABNk9HcFEdTfWnkPpl92A5LPQx4Ofr21uSo7Yv/XreiEb/68z7s7c7j5PXD8sc5g6k4JhMRLGhNTqq80vacTzSsoqkpHqRTAGDN8gYAQJ9nX2N9FJoqT+v4u2hYgUsotLCKoayJE9a2oKE+hqOWNWDnwTRCEbaOkUhGACmNrG5j5aI6xONhhMIa2hYMS03U4nlXziYtosFwgFMa4/jfP+/HGwfSOG4t+z637x+E4xJsXN2M5QuS+PmTu9AxoCOds6BIImyXQJQk1DfEJjXoZTrsnm1q0aZKqeibSqVSeO2114Lo7sEHH0QyObGWCgDs2LED//iP/4gbb7wRS5curdiw/v7clKKGpqY4enuzk96u2kzVLtshcBwXsiSiYNjo7c0inTMhCQLS6QKEw6iAoJQG5Y2qLMI0rBIbR9q8pDGClrow/vfpPVjWEvUUGCkGMwZWtiWQyxkY6J9MfyiQzbDFTUoIBvpz0Iuch+pdL/Z5A64Fl0CWxGn9fjVFQq5g4fXdfSAUiIdkDA7lEfdKV3fu60djMox9BwRs38vy2amogp6+LJJRNbClFs+7sWyyHYJ0ugASVbF2cQovbO/F245rgyyL2La9B6IooDGmwDBMtDVG8dLOXgxmLaxbWocXd/VjKGugtzdbNUc/l45lLSGKwpgBckX32TfffDM+/elPY9euXXjTm96Eb3zjG7jlllsm3O5vf/sbPvShD+FTn/oULrnkkslZzQHxSuJkUYCqiF7D1PDQkcOtPike2xdSpQkHZ0iiiDdtXIBM3sKzb7DFypd3D8AlFImoMukuT7ZfGaJQmjbyCWsyYmEFfWl2MYiG5WmroS/eB6umYdUnzXXhkv/vGdQBr468d1CHKApYUO91xIZrfyG2HIosQpaZqNrRKxpg2i6efb0HhFDs7khjUXMsmGOwvC2BrgEdpu3i6OX1UBWRzSTgOfo5RUW/zOXLl+OBBx7A3r174bouli1bhkKhMO42nZ2d+Od//md8/etfx2mnnTYtxh5puF5Zn+Tpzltew5RhsZF9leThJ8LvDo2ElAnz66IoYHFLHMvbEnjqxU6AAo89dxCLW2JYt6QOoSnUk0dCMm64eAMo6KgLlyyJSETYEGtRYOsJ09UVG+xfk9HV76I/bUAUhGBoeCysIKzJrPxwCYVhuega1NFaF2Za71bl8sK1SCysIq9bWNoax4KGCB577iCefb0HQzkLbz1+YfC65W0J/HFbJyIhGcsWJBDyppjxHP3coqJfzaWXXgpJkrBixQqsXr0aiqLgve9977jb3H333TBNE1/+8pdx8cUX4+KLL8ZPfvKTaTH6SMGXyJVFAZoqsaobsLpvpt1++PtIeOWBIU2aUKNGZBI4eNsJC6GbLh577iBWtidx5bmrIYkitCl2iC5ujbHPM2L3ssRKLAEgFmF6MtPt6KNh5rgyBQsNSQ2C11wkCAKa68LoHtSZuJntoLM/j7amqKfPLo6rM1PrxMIybIfpzF+7aR0ue8tyFjwIAtYsTgWva2+KIhKScfTy+qCklw1Unz3bOZNn3F/mBz/4Qbz00kswDAPHH3988DghBOvWrRv3jW+55ZaK0jucsXGKInpNkWC7rLzSsNxAnOxwScSYow+r8sSO3hvKvaAhitM3tMK0XZx/8iJIkhjYNBVEQWAdnyM+j1/ZAzDHJHnTrqaTaEiBaRP0DOpob4oiV7CZDlBEQXMqjG07+6BIInoHDVg2wcLGKCzHRTw8eSGzWiKkyoGypSgKOGppPY5aWg/XJUHn8VDWRDyi4MPvXB9UF2mqBMvhEf1cY1xH/z//8z8YGhrCzTffjC996UvDG8kympqaqm7ckc6woxegKhIcl8B1CSyHMNngaXB6/sCMkCpN+H6CIASThkZq3oNOvrTSRxKZhkq5NYcmT9yM1X5PfwTtT85K5y0ct7oRgncMCKForgvDcgjyhoNBrxt0YVMMtk0RS9V+/fx4aIoUyGUUX7B8J287BLIswbBIySAZf86sW2b4DKd2GfeXE4vF0N7ejnvuuQdNTU2IxWKIRqPQNA2ZTGa8TTnTgO/oZVGApohs6IXXvKMplY/rG49kZDiir6QsUhKFMaO5qTp6UcCYi6xN9WxRNBqWoSrTX+VR3D3sX1QakyHohluyINvRm4emSEzvBbSqpYUzgSgKiEXUEkmLYiyLIBVV4Y5YdJUE1mDGFSznFhWdrffccw+++tWvwrbtoItOEAS89tprVTXuSMf2BjzIfurGocNDR5TROe2pUOfVpEfCo6teyiFLbKB08TKkHxVOtSKGbVt+YbO1jqlERjSlKjnxWJGj94eMNyRC6EvraEqxY9MzpONQXw4Lm6KBvXOhUWoimuvCrFdAG/2cC4qGZAhZ3QrSOYRQlrqxXT58ZI5RsaP/yU9+gvXr11fbHk4R/nQpP0dfHNGrkxjAPR5NqRAuOXMZFjbFKhLnkkURllMaBbqETlqeuAQBkOXy2zbXhXDcqkasaEtWpcol5gmbyZLgqW/KUBUJ9fEQ0nkTqZiKQ715dA/qeNPRKdgOQTSkTPtawWwQDSmIRxSYlltybF1CIIusa7ghEULvkIFoWIRuuoiFFVg2KRlaz6l9KgqRmpqauJOfBYLUjTQcQWYLTFVQOxzHWoQkili+MAlZqmxgiCyxev5ibIccllSvKAhj1vCrsoS3HLcQjcnQtFfcAMPCZk2pMCgdVgStT4TguBRNKRb1Ugq0NUVhOQSJyNzOzxfTXBcpmRcLsMX++jirQIpHWPqGyRwTpGIqCKUwLZ66mUtU9Ms544wz8OMf/xjd3d0YGhoK/nGqS7AYKwpMVxxsNiswvTXcbKBEZRUtTakwNMVbkCME2bwNSoHGw5AlEISx8/vDIml0UtIKleJLFTNHT4NGobAmI6TKaEqGguag9sYoKC0dYj7XiYZkr/Jo2Nm7LoLpY6yRTkDecJCMaoFap697w5kbVHTGfvvb34ZlWfjCF74QPMZz9NXHLoroZW9KVMaL6Cc74GM8NFmCU2HOVVMlLG9Loi+to2tAR3MqhMZU+LCi7bAmj5nz9oeNG5Y77V2xABCPqAipEtqbooAglKwDxMNKsIaRiqmIhhVk89a8yM/7CIKAlvoIdnWkIXrrLKIwPDVLEAQ0JELY35PDsgUhRLrZ4yNn6XJqm4oc/YsvvlhtOzhlcLyGKUWWilI3LKKPatOXPlAVaVTefTxEUUBzXQQNydC0DN0QBQHiOE48pEiwLLcqAz5kScQ/XbIBmizCsElJL0EkrAR1/G2NUZa7lsVJDT+fC8TCCpa1xnGoL49swUVTKlRydxePqGhMhBDRZERDvqO3Z8tczhSoyNFbloXf//73yOeZXKzruti/fz8+8YlPVNW4Ix0/daNIYqAjk82zH1gkPI0RvSrBtCf/fjM1WSmkSSiY09M3UI6IpsCwbIS10nUPTRFRH9fQkAhhzeIUbHvu6ttMRCKqIRpWMJg1R42oDGsylrTGIQgCwiF/oDoZVYPPqV0qcvSf+MQncODAAfT29uKoo47Ctm3bcPLJJ1fbtiOewNErUtAs5Ef0kWmM6BVZrOm68JAqQ1OqlypQJAEZmyAZLa0zVBUJkiTiny5ZD0EQkM3bgTbQfEQSRTQmw2Wf8x168UB1Qikk7ujnBBWFZK+99hq2bt2Kt73tbbj55pvx05/+FNlsbUt2zgf8HL0iC8EiYc6wIXraN9NFNKSgKVX+B14LKLKISKh6DtavJBp5sRMFARFNLqoyolNS6JxP+A1mpu2A90zNHSpy9M3NzZBlGUuXLsX27duxcuVK6Lo+8Yacw8JfINWKcvR5nY0RlGt4IPV0E9HkoEu1GiiyCAoEF9NiYmGFTfailC3WVkGGYS7hXwwNi0sVzyUqOmsjkQgeeughrF27Fr/+9a/xxhtv8PLKGcBvmFIVCaoXwRdMf17skeNwBEGo6nqAv8BarvM2EpJBCOC6FJoizdi6RK3i30lyqeK5RUVn7ec//3m89tprOOOMMyCKIt7//vfj2muvrbZtRzzDi7ECtCInpCoTC5BxKkcUWWqsXDWNpkiAAFgOQTw8fxqlporoyT8YljtqqDindqko4XjffffhM5/5DADgv//7v6tpD6eIIEevSiVpBTZdarasmn9IooCoJpetIPGPte2UqjgeyYRUiQ0I535+zlBRRP/kk09W2QxOOWyHQAArryxu0uER/fSiyhLqk+U7ewVBQDTkDRCfR41Sh0NYk1jqhkf0c4aKIvr29nZcc801OP744xGNRoPHr7766qoZxmGpG0kSAnkCWRLguBTqNMyL5QyjyCIUeez6+Jg3zlA5whdifUKqDMt2QXlIP2eoyNGnUikAwKFDh6ppC2cEjkvZUA4veldlCY7rTMtgcE7lhFQ5GLPHYZU32YINhzv6OUNFjv4vf/lLMEsTAERRRDgcxic/+UncdNNNaG5urqqRRyq240ISh0fsqYqIgumnbmbZuCOIaEiGpkQnfuERQkSTYTluUCzAqX0qcvTnnHMO8vk8rrzySoiiiF/84hfI5/NYs2YNPv/5z+POO++stp1HJI5LIUkC/EBS9YTNtGmaF8upDEEQoIyhl38kEtZkthjLh4/MGSqKC5977jl88YtfxFFHHYW1a9filltuwY4dO/ChD32Ip3OqiO0QFtF7f/u19Jpa2TQoDqcahL2I3uYR/ZyhIkefz+eRy+WCv3O5HAzDqJpRHIbjEkjicPTu693wxVjObBINyaAUMLhU8ZyhotTNu971Llx++eU4//zzQSnFo48+ine/+9249957sXz58mrbeMTCHH2Z1I3Gy/w4s4evO1TgUsVzhooc/fXXX49169bhD3/4A2RZxuc+9zmceuqpePnll3HJJZdU28YjFtth5ZV+8sav49Zk7ug5s8ewo+dzY+cKFUvxnXnmmTjzzDNLHtuwYcO0G8QZhpVXDkf0vs5IjHdocmYRX9iswMcJzhl4kV4NwxqmxGFH7+XoQzx1w5lFfKliPk5w7sAdfQ3juASyODp14//QOJzZIBriUsVzDe7oa5iRqZuQl7oJ84ieM4v4qRsuVTx34I6+hhmZumltiCIaGh7QzOHMBpEgondg2byWfi7APUYNM1xeyTz9aetbkYwo0BT+tXFmD02RIHjSzUM5A5FQbLZN4kwAj+hrGHtEHT3ABoVLvCuWM4sIgoCQIsFxKfozJte8mQNwR1/DOA5lqRsMO3ZFEiFwR8+ZZTSVadJTSpErWLNtDmcCuKOvYUZ2xgII9Ok5nNkkrMkwLBdhTUJvujblUFzC7zR8uKOvUSilcImvXjns2FVZgixxR8+ZXcKaDNN2ocgSDNOtyZr6Q315GFbt2TUbcEdfo7he2Zo8Qni+pT6CkMoXYzmzS0STYVlMAkGUgMGsOcsWlUIoRSZvw+RVQQC4o69ZbIedoBKP3jk1SEiToFssRx9SJaTzteXobYfAsh0UdC68BnBHX7P4lQy8woZTi6xoSyKn2zjQk4MkinBcUlPVNyxQEpDnCpsAZsDR53I5bN68GQcPHqz2ruYVjje9R5b4tZhTe5xyVAs0RcRfX+thD1ABll07apaG5UCRBeiGU9K965LauiDNFFX1Itu2bcN73/te7N27t5q7mZf403v4JClOLRJWZWxY1oDX9g0ik7cgCEz7plbQDQeKLAKCALPoAtSfNjCQrc0qoWpSVUe/ZcsW3HrrrXx4+BRwvBy9wnP0nBpEFIFjVjUCAJ59vQeyJCBfQ7LFedOBIokApbC83xKlFAMZAwW9duycKarq6L/4xS/ixBNPrOYu5i3+7SVP3XBqEUEQkIyqWL0ohb9v7wMA5PWZbZyyvIatkRBCYdsuJEmELIuBXYblwrQJCjVYClptarZOr6Fh6voZTU3xabRk+piMXYNe1JFIhGb189TisaxFm4DatKtaNhFC0TFk4G0nLcb/d9+L2N9XwNql9airj1YcnBimA02VSvpEfCayu39IR2dfAYtb42iqi5Q8p5sOUikdiaiGmENAQdHUFEfXQAF1dRGAYlJ2VmpTLVOzjr6/PzclCdSmpjh6e7NVsOjwmKxdvX1sGLtp2LP2eWrxWNaiTUBt2lVtmzJpHfVRGamYiudf78GCujAOdaQDdcvxoJTijQNDaK4Loz4eKnluPLsppegd0tHZX0AkJOOlN7qxqj0VSCcDQKZgYSitw7VZsJQt2KiPyNh5MANJZBeCjs50yTYTUYvf70hEURgzQOZ5gRrFX4yVeOqGU6NIkgAKYGlrHPt7sqCEwrQrS4sYlgvTctHZX5hUFcxAxkRnfwHxiAJFFqGpEvZ1Z0vew7AcdPTl8MdtHewBSpEt2LBsB7LMfk9+3v5IgXuRGsVfjFX5YiynRoloCiybYHFLHLrpIl0wK54jm9dtSKIASih6BvWKtiGEonuwgFhYDqrRNEWC4xB09uWD1xV0B397oxe/e74DecMGIGAoawYpIkkSoB9h9fUz4uifeOIJtLe3z8Su5g3BYqzMp0lxapPmujAsh2BRM0sXdPQVkKuwomUwZ0JTRUTDMnrTekVaOZmCBdelo+5yo2EZA1kT6Rzrzs3pNg72Mse/vysHWRaQKdjQVLadLInQzdopBZ0JeERfo/ipGx7Rc2qVsCajIaEhpEiIRxQc6s3DtN0JUzG2Q2CYDhSZLcRqioj93blxnT2lLPIPaaNdliAIiIZlHOzNQzcddA/qQU3/3q4sVFmE4zIBNoA5+iOt8oY7+hrFcdhCtKryiJ5TuzSlIiCgWNQcw77uLEBJoNM0FrrpwHJIEIGHVBkUBDsPDqEvrZctwsgbjtftWv73IEsiBAAdfXkc7GGLps2pMPZ1ZSFJIuoTIaTzFr738GsYypkg7sR2zie4o69R/KhIkbij59QumiKhKRVGa30E2YKNTMFGrjB+/judN/Hk84dwxwOvIJNnNe4hVUY0rKCjN4/9XRmQEfXxvUM6VHl8dxUJy8gZLG1TF9ewYXk9eoZ0L08P/O2NHhzsyeON/UMABNjOkZO+4Y6+RvEdvarw1A2ntmlIhLGwMQoA6E0b6BjIB851JIRS9AzpeOPAECyb4NFnDwTPiaKAeFRB75COrv4CKKUghKJnsIBM3kSognLIeFhGR18eS1rjWNrK6t73d7FS7Rd29LO/u1np8pEU0ddsHf2Rji9qpvJB4JwaR5FFLGqOIaRKONiTw4ZlddjXlcWq9iREUWD6MhkDiagKTZGxY/8QHJdi9aIUXt07iF2H0lixMAmA5dtTMQ37OoYAUBQMBwXTQTyiAmCiZDsPpvHCjn44hOCCU5egLq4FtvQOGdAtF0taYljQGIEii9jblYUoCcjpNhIRBft7chBFoGA4SMa0Mp9o/sEj+hpleDGWf0Wc2qcuEcLCxij2dWfZIiuAfd05vLF/CN2DOmRZRDpvoaMvj9f2D6E+ruGyNy9HfULDr5/ZH5QTA8zZJyIKeocMOIQgEVUhigK6Bwr4xs9fws+e2IVDfXkc7Mnju796Fbs7MsG2+7pZfn5JaxySKGJxcwz7urJ4fnsfoiEZZx7TBt10kC1YKBxBJZbci9QoDh88wplDREMy2hqjGMiYyBYshEMyLNuBKouIRxTIkoiwJsOlBAd6cti4sgGyLOKCU5dgIGviwT/thVmkfikIApIxNZimZtkufvHkbgDAFW9biY+/eyOuu3AdYmEFP3psO/64rQOEUOztyiIZVZHyIvUlrXH0DOnYcXAIx6xsDNI5h/ryweCUIwHu6GsUxyWQJYF3xnLmBJoiYVV7EoIAPPbsQVBKEQkpQSeqz0u7WJ584/IGAMDytgTefGwbXtk7gDsfeAV7OjOj3hsAHvnrAfRnDFxy1jKsXpSCKAqoT4RwzaZ1OGpJHX73fAf+3yOvY19XNnDmAIL/phQ4bnUj6hMaoiEZB3ryILS2pJWrCfciNYrtEoiiALGM4BOHU2sIgoCVC5M4Y0MrXt4zgBc9h14MpRQv7urHkpYYUnENphdRv/nYNnzoHWshSQLu/c12/OB/X0VfmmnGE0Lx/PZevLCjD2/auADLFiRK3lNTJFz65uW45Mxl6B00oJsulnjOfSBjojEVgiqLWNwSQ0MiBEEQsKglhv3dWaiygL2dmRK9+vkKX+mrURyXQhZFPniEM2eIRVScsKYJB3rzePiZ/WhIhtA7ZOD1fYMYzJrBwurpRy+A4xKYtgvbIYhFFCxqjuEfLzoKT73Uhb+82o1tO3vRUhdBX1qH41K0N0fxlmPbyu5XEAQcvaIBi1vjeHFXP45aWgfLdhFSJdg2xXvethKJqBq8fnFzDK/vY1U/qixid0cay9uS0JT5W8rMHX2N4jgEkiSAB/ScuUJYkyCKIt75pqX49kOv4nv/+zoAIBVTsaAhikhIRiKi4Ohl9SgYDlobImzB1SWQJRGKLOHs4xbi3FOW4uE/7UZXfwEnrGlGW0MEqxenJgx6klEVZ25cAADI5C0014XRO6iPugtY3MIi/gM9OaxfVg/DdLC3c7hKaD7CHX2N4bgEkijAIez/eeqGM1eQRBGxsALLdnH52SuxuyODNYtTWNAQKdGcd71zuz4egiqJ2N+TK4m44xEV5520aMp2UEohQEBDIox0zobjkJK1gtZ6Vna5v5s5+pAmI5O3MZgz0ZAIjfPOcxfu6GuIvGFjX2cGmirDtNiEHO7nOXOJuriGfd1ZLGmNB7nykRR0Fy0NYciSiERMQyRtwLRcaNMk92FaLpIxFYosoi6honfACBx9Tmclle1NUezryrKLgiAgGpbQ1V9AMqpWdaqbSwgonfnJcXwxtkYYzBrYdSgNRRFhOy7SOROSKEAA9/ScuUMyqiIWUmBY5UXDCKEQBATDRkRBQFtjFKbtwqpgUdR1CfK67cktWNDLyCLbDgmaqOJhFS5o8LgkCKAEWLu4Dj1DOn79zH5QSiGJYjDUpJoMZkz0p2d+OPkR5+hzuj2lyVXVwiUEnf157O/OIRqWociS1+rN8vM8oufMJQTPcVsWGVWjbjsEmYKNxmSoJKKNhBSsXJgEpQIyeQvuGL9Py3ZRMBw0pcJYtiCBVQtT0FQJmbwd7IsQClEUEA0pAICQKkEWBbgumxXb1hhFMqZi/bI6nL6hFc+90Yv//fM+UEqZZPKQPuZFyudwau8HcyaMCoezTCdHTOrGdphD7c8YaK2PYEFDdNr3QSnF/u4swpqMxlR4wvy6bjo42JuDYTpIRJVReUyFp244c5CwJqOpLoz+tIFISIZhuXBcCk2RsLQlhkR0tOxAJKRgVXsSfWkdBcNGNs9SLLIsQJFFOC6B6wIrFiYR8Zw4ACxdkEDPQAG9aR2AAEIoWurCwaKqIAioj2s41J9HXUxDIqpCkkQM5Uy87YSFEATgTy91QVMlnHviIiiyiEO9eSxrS5T9/Xb05aHKIhpT4UkfF8t2kTccUDrzbnfeO3pCKTI5E4f6ChDAKgB6hnTEIypiYWXC7SdDJm9hKGcinWe3lu1NsTHzjjndxp7ONFRZQtxbiCKEIpO3IMsibK8SodzgZA6n1mlKhZHOWzAsF6mYikSUNSqNdz6LooDmuggaG2M42DEE3XRQMBzkDQeiIGLZwvio35MoCGhtiKIhGQbxRNBGlknGoyq0jIm2xigEQUAkJEOWRBBK8dbjF0I3Hfz55W6sWZTC4pY4MjkLA2ljlDMfypnoHiggFdem5Ojzhg1RwKyoZs47R+8S6qnSUVjeiLGC6QRfLgBENBn7u7NY1Z6CMoH0aaU4LsGhvjyiXjegbjp448AgoiEZqXgI8cTwiZHTbezpYMOJZZnlBl/fN4QnXziE3qHh/N2KtgSP6DlzElkSsXJhkq0zTfIkFgQBIVVGSJVRV349dxTj/Y4jmozlC+KBnIIoCGhIhNA3ZCAakXHeSYuwuyODh/60F9dftB6xiIKO/jyiYSUYIK6bDg505xCLKCgYdrCIOxmGchZURYRt06CkdKaYd45+X1cGe/YPQBAEUEqhqVJJ6RbATgrLZqkcv6b2cOlL63AJDVb3w5qMsMZu1zr78shbBLZpIRnT0NmfR8hz8ns6M/jtcwfR2V9AYzKE809ZDFFgw4tb6yN8MZYzZ5npypKxYFF86d17Mqaie7AAAFAVCZtPX4ofProdTz5/COeetAiaKmF/dw4NSQ0uoehKm1AUAZIogFCWClYn0WDluAT7u7P42RM78c4zmYzDTE4JnXeO3nEIQqo04ZcQDcsYzJrBbeVkMS0XhuWAUIAQgp5BHdmChb+90YPjVzcFFxdVYbYkYxq6dROdfQVWt1uw8NizB7D9QBrJqIqL37QURy9vKGnYyBZsHtFzOFUgpMqIhhTkCjZiEQXL2xI4fnUjnnm1Gyvbk1i2IAHddNA1UIAgAI11Mew4mMavnt6HS89ajqWt8Uk5+oLp4PkdfTAsFx19BTiEQMPMefp55+jHglKKgYwJ03bR5g1JiIbYnMnVIWXC6MNv2dZNB4NZE7rpsGhbYCvw6ZyJH/92J0zbxZ9e6sKxKxtx6voWNCSHGzB8p9/Rl8e9v9kOCpYjPOWoljFvPXmOnsOpDkta4zjUm0c6ZyIWUXDuiYuwvzuHn/9uF67dtK7kt7vz0BDu/+MeUArs6czguNWNgUZ+JXQPFPDavkEAwGDWDOZNzBTz3tFn8haefrkL2w8MYSjHxpatbE/ivBPb0ZgKw7BcdA8UsLApVrJd3rDRM6DDdFw4DgEFBbxEiqaKJemgviEdP/vdLmiqhCvOWYmXdw/ghR19+Nv2XrQ1RrBhWQNOOboNAoDO/gJ++Oh2hDUJHzh/TSCnOhJKKeZpNzaHUxPIEhM76w1J6OzLIx5R8d5zVuHuX72Gnzy+A9duWgdFErHzUBpb/7AbCxoicF2Kjr488roDpCrbj+MSPPNKF2yHIKLJGMqZsMfoGXBcAkLopO4WKmHeOnrXJXjm1W78YVsnCKFYsTCB0za0wrJdPPViF+544BW89fiFOH1DK/oyBsIaW/yRRAH9GQN9Qzo0VYIiC9CU0dUClFJ09hew/cAQ/r69DwKA95+3Gg3JEJa0xHHWMQvw8u4BvLxnAI8+ewCPPnsALXVhZAoWNEXCB94+2skTwhZpLJvVICeOkOk3HM5sIQgCmlMRgALdAzrq4houf+sK3Pub7fjW1pdhWi4IpVjQEMX7zlmFP73cib+82oN0zqxoQZZQigPdWTy/ow/tTVHUJ0LY05mBMYajT+ctEELRNIWqnvGYV47+YG8OP39yN3oG8hjKWTBtF2sWpXDeyYtKxo0du6oRv35mPx7/2yFoioRjVzbiUG8OgOD/D4oioC+tI6fbyBsObIfAJRR5w0ZXfwFdAzpL3whAe1MMm05bUnKrF4+oOG1DK07b0Ir+tIEDfQVs294DQhVc8bZVSHn2OC6BbrgAKCRJhKZIaG0IIR5Wp60lnMPhjE9jKoy8YaNgOFjcEsdlb1mBbbv60JgMobU+ghPWLYCum1jSEsefX+5GZ38BRy0df0GWUorOvjxe3N2PoZyFtx6/EINZEy/uspHNm8CILAIA5HQrqPSZTuaVo0/nLAxkDMTCTPZ09eIUVnqzKIuJhhRcetZyOO5OPPzMfkRCMo5aWg+A5c+eeaULz+/oDwZ0FyOJAprrwli7OIXFLTGsai9t4Ch3lW9IhrBySQOOXVFf8rpswYYsiWhvjiIZ1eatch6HU+uIgoD2pjh2HByCZbtYsziFNYtTwfMhTYaum2ipY5H2wd48rAkqb3qHdPQM6nhhZx9iYQXrltRh264BAED3oIF1S0tfTylFTne4o5+I9cvq8cn3HY/evuyEOS5RFPCuNy/HDx/dga2/34NHnz0I4kXsgiBg4/J6rFlSh3hYQTTEpAlkSYAsldeIJ4QNMqZg02wAQBIEhEIiJFEc9dpswUZLfRhNqfCo5zkczsyjyCKWtsaDGbTFPoRSilzBhqrIaEyG0NGXg246YzZdDmQMHOrN4/G/H8S+rhw2nbYElAJ1Mba215/WA7kGH9shVRuCMq8cfSX4OjeCACiyhCvethJ/eKEDpu0yjYywghOKyiMnwnEIdNNlJVipEBoSYQgCq5/PGTb6BnW4xEUoxNI/AthC78KmGBqT05uH43A4h0ckpGDFwiR2d2ZAqBP03CDPSrFbG6JY1BTFy3sHkc6bZXPp6ZyJ/V1ZPPH3g3h17yDOObEdJ6xpwlDOwtoldQCAwZwFxyVQxeGLiWm7cOzRWYTpYN47ev92CJQCAmt48B93CYUsiXj7KYvH3NZxKWzHBSGAKLJbPJdSEBegoFBlEQsaI0hGtZISSVkSEQkpaEywBVhZU5DP6zAdgsUtcdTF56fuNYcz1wlrMla2JbG3KwPLJoiHVSxrr4OlmxAEAWsW1+H5nf3Y25nFirZkkKq1HRfpvIW9nRk89txB7DiYxttOYAUfBcNGXUxDa30EsbCCgYwBx6VQi24ICoaDavVHzmtHbzsuCqaL5lQYjckQJEksESqybBed/QUM5kxoCnPSlFK4Lg1kI8OahPoEU9tzXALbYc49pEpQZAlhTZpQvyMV09DUFEdM4SkaDmcuoKkSVi9KBb/tVFxDr8HKszcsbwCe2IkDPTkc6MlBEAQ4LkFOt5DOWXjo6b3oSxs4/5RFOHldCxyHgBCg1RvA0pQKsVp6Uhq9d/bn8cBTe3DleatZJdA0Mu8cvSQIMCyXDe6QRaxoS46ZR1MVCYtbYqiLq8gWbIiCAEEEwqoMVZGgKRJfIOVwjlDGCuBa6sNIxVR09ueRNyyAAh39Bew4mA6Gol957mosb0vAsl2YNsHS1nggttZSF8HzO/o8TS4GIRR/296Lzv5CVbrh552jX9QahyYCkiRUJKgkCAISUW1KMggcDufIQxJFLFuQwPM7+vCfP9lW9LiAle1JnHNiOxoSIRiWw6SV25KIhIZd7YLGCJ551cVQxghGFxqWg1f3DmJhY7Qq4wznnaNXFYnXn3M4nKqy+bSlTH1WEkAoRWMyjDWL2CAU1yXI5m2ENAnLWkdLK7d5szA6BgpY0Z4CAOw8mMZg1sSJa5qqYu+8c/QcDodTbdqbYzh5XTPikeGBQZbtIpu3IIoiFjZFkYprZYeX+HIrXf2F4LGnX+mCLAlYsyhVFXu5o+dwOJxJosgikjENmbzlVfQBEU1Be3MYsbA6rj5+YzIEQQD6M0ZQ2ffirv7gjqAacEfP4XA4U2Bxc8ybasWK9CrV35clEfXxEAYyJtJ5C39+uQuG5eKYlY1Vs5U7eg6Hw5kCgiBAEgRMZb5KS30YHX15PPKX/Xh+Ry9iYaaJn9PtCWdNTwVe2M3hcDgzzIL6CIZyFh599gDyuoO3nbAQhukioiljSpcfDjyi53A4nBnm7acsBgWwYmECrfURGKYLWRaxdEG8KiMYqxrRP/TQQ7jgggtw7rnn4kc/+lE1d8XhcDhzhsZkGKetb0U0pCJXcCDLEpa2Jqo2Z7dqEX13dze+/vWvY+vWrVBVFVdccQVOOeUUrFy5slq75HA4nDlDexNbzFVkseqD1Kv27k8//TROPfVUpFIpRCIRvP3tb8cjjzxSrd1xOBzOnEJTJa/pqvpLpVWL6Ht6etDUNNzl1dzcjBdffLHi7RsaRk9fqZSmpviUt60mtWrXeNSizbVoE1CbdtWiTZVQi3bXok2VUjVHT+noKecT6c4U09+fC7TjJ0NTUxy9vdlJb1dtatWu8ahFm2vRJqA27apFmyqhFu2uRZtGIorCmAFy1e4ZWlpa0NfXF/zd09OD5ubmau2Ow+FwOGNQNUd/+umn489//jMGBgag6zoeffRRnHXWWdXaHYfD4XDGoGqpm5aWFnziE5/ABz7wAdi2jcsuuwwbN26s1u44HA6HMwZVbZi68MILceGFF1ZzFxwOh8OZgJrtjD2cyU61OhWqVu0aj1q0uRZtAmrTrlq0qRJq0e5atKmY8ewTaLnyGA6Hw+HMG7ioGYfD4cxzuKPncDiceQ539BwOhzPP4Y6ew+Fw5jnc0XM4HM48hzt6DofDmedwR8/hcDjzHO7oORwOZ57DHT2Hw+HMc2ZMAiGXy+GKK67AnXfeifb2dmzduhXf/e53IUkSTjnlFNx0001Ip9O45pprgm2y2SwGBwfx/PPPI5PJ4P/8n/+DAwcOoL6+Hv/93/9dMtjEp6OjA5/+9KfR39+PZcuW4atf/Sqi0Wjw/C9+8Qs899xz+PKXv1zWru9973v4xje+Add10dLSgq1bt8JxnMCuoaEhZDIZAKiqXWPxjW98A47j4He/+x3uvPNOdHZ24rrrroPruhAEAe3t7XjwwQereix37dqFz33uc8jn8wiFQvi3f/s3LFq0aNT3e+edd6KnpweKouD444/HLbfcgo985CPB+3d3dyOTyeDVV1+tik3r1q0b87wbHBxEe3s7fvKTnyCdTuOKK67AwYMHoSgKCCEghEyLXTt37sQtt9yCQqGAZDKJL3/5y0gmk7N6rMrZtHDhwpo+53y6urpw0UUXYevWrWhvbx/1/f7oRz/CV7/6Vdi2jbq6OmzZsgWqqgZ25fN59PT0QJKkqto1FiN/5x0dHdi0aRMWL14MAGhsbMTdd9895vZThs4AL7zwAt28eTNdv349PXDgAN21axc988wzaXd3N6WU0ltvvZV+73vfK9nGdV161VVX0QcffJBSSun//b//l951112UUkp/+ctf0n/5l38pu6/rr7+e/upXv6KUUvqtb32L3nbbbZRSSg3DoP/5n/9Jjz32WHrjjTeOadeGDRvoj3/8Y0oppe9617vo+9///pLtjznmGHr66adX1a5yZDIZ+tnPfpZu2LCBnnbaaYHNt912Gz3++ONn9FheccUV9IknnqCUUvr000/Tc845p+z3+8EPfpD+6le/orfeeiu95pprSj7zbbfdRteuXUvf9773VcWmCy+8sOz3+6Y3vYl+/OMfpxs3bqSXXnppcKzuvvtueuedd077sbrqqqvo73//e0oppT/+8Y/p1VdfPevHaqRNn/zkJ8tuX0vnnP+e11xzDT322GPpgQMHyn6/xxxzDP3a175GKaX0Ax/4AL3ooouCbe+++2560kkn0RNOOKGqdpVjrN/5I488Qj/3uc+V3WY6mZHUzZYtW3DrrbcGg0feeOMNHHvsscHfZ599Nn7729+WbHPfffchHA4H6pdPPvlk8N+bN2/GH/7wB9i2XbKNbdt49tln8fa3vx0AcOmllwZzap999lkQQvDpT396TLteffVVuK6Ld7/73QCAK6+8Es8//3zJ9ueccw4kSaqqXeV4/PHHsXTpUixfvhxvfvObA5v/9re/QVVVXH/99bjhhhtwzDHHVP1Yvvvd7w5mC6xZswadnZ2jvt9jjjkGL774It7+9rfj7LPPRiaTKfnMr7/+OlasWIFFixZVzaZy511LSwvWrVuHq6++GkuXLg2O1UsvvYQ//elPeOtb34qdO3fixBNPnBa7vv/97+Oss84CIQQdHR3o6uqa9WM10qZEIoFy1NI5BwDf/e53cfrpp6Ourg5Aeb9CKcV73/teAMAHP/hBbN++HbZtY9euXdi1axc2bdoEURSralc5xvqdv/TSS9i+fTsuvfRSfOADH8Abb7wx5nscDjPi6L/4xS8GPxwAWLt2LbZt24bOzk64rotHHnmkZBqV67q444478KlPfSp4rHgGrSzLiMViGBgYKNnP4OAgYrEYZJllpJqamtDd3Q0AeNOb3oTPfOYzCIVCY9rV2toKSil6e3vhui7+8pe/wLKsYPtPfepTeOqpp7B+/fqq2lWOd77znbj++utxzjnnoK2tLXh8wYIFIITgjjvuwJlnnonbbrut6sfy0ksvhSRJAIDbb78dF1544ajv94UXXkA4HIYgCHjkkUeQTqeD7U877TTs2bMHF1xwQdVsOuecc8qed729vbjwwgshCAJ27doVHKt4PI6rrroKoijiPe95Dz7xiU9Mi12yLCOTyeCss87CT37yE3zta1+b9WM10qbLL78c5ailc+7ll1/GX/7yF1x99dXB68t9v4ZhwHEcuK6Lxx57DIIgYGBgAKtWrcIXvvAFPProo8EFs1p2lWOs37mmaXjnO9+JrVu34tprr8U///M/Bz5nOpmVxdhly5bhU5/6FD784Q/jyiuvxJo1a6AoSvD8H//4Ryxbtgxr1qwZ931EsdR8ephzahctWoRYLBbYtXr16pLt//jHP6KxsRHJZHJG7RqPr3/96/jXf/1XfPjDH8ZDDz2EfD5fsv9qHUtKKb7yla9g27ZtuPnmm0tet2zZMlx//fUYGhoq+X797X2bWltbZ8wm3y7/vNu6dSsaGhqC8+4LX/gCVFXFsmXL8PGPfxw7d+5ENlt+Ruhk7UokEnjqqafwX//1X/jwhz8M13VLbJqNYzWeTRMx0+ecruv4whe+gH//938ftU0xy5YtgyRJ+MhHPhIcy+L9/PGPf0RraysikciM2jUeH/3oR3HFFVcAAN785jcjEolg9+7dU3qv8ZgVPXrTNLFx40bcf//9ADDqKvvb3/62JIIBgObmZvT19aG1tRWO4yCXyyGVSuHiiy8OXvOLX/wCuVwOrutCkiT09vZOOKf24osvRnd3N6677jr8/Oc/h23buO+++yBJEn72s59B07QSuzZu3AhCyIzY5fPAAw+UfQ2lFN/85jdxwQUXBMfymGOOCRZ2fJun+1g6joMbb7wR3d3duOeeexCPxwEgOI6yLOPrX/86NE3Dj370Izz++ONoaWmBYRgzblPx93v//fcH5903v/lN7Nq1C6qqghCCu+66CwcPHiyxS5blw7br4Ycfxjve8Q4IgoCzzjoLhmEEEftsHauxbCqOSmvpnHvuuefQ19eHD3/4wwBYFH799dfjW9/6Fr70pS8Fx/Kuu+5CY2Mj7rrrLrS2tuLXv/41ACCVSgV2nXrqqXjxxRdnxK6enh4AwLe//W20tLSUPZ733nsvNm/eHKR9KKXBncN0MisRfaFQwAc/+EHkcjlYloV777235MR44YUXSm7JAHa180+shx9+GCeeeCIURcEDDzwQ/FMUBSeeeCIefvhhAMD9998/4ZzaBx54AC0tLfjOd74D27ZBCMHWrVthWRbuuusuHH/88SV2LV26dMbs8v+NhSAIeOyxx/C+970PuVwOv/jFL6CqKjZv3lzVY/mVr3wFuVwO3/ve9wKHCiA4jvfccw+uvfZaHHvssXjwwQdx7733Ih6PB9vPpE3F32/xeee6LrZt24YLLrgAoijisccew1NPPYUTTzwR999/P4455hiEw+HDtut73/seHnvsMQDAM888g7q6OtTX18/qsRrLplo9584880w88cQTweuam5vx7W9/G8uXL8d3vvOd4FjG43Fks1ls2bIFlmXh9ttvx6pVq4K7thdeeGHUXUY17fIfH8vJAyx3/4tf/AIA8Ne//hWEECxfvnzM10+Zqi/3FnH22WcHq9JbtmyhF1xwAT3vvPPo7bffXvK6jRs3UsMwSh4bHByk//iP/0gvuOAC+p73vGfM1e2DBw/Sq666ir7jHe+g11xzDR0aGip5/r777htV3VJs17e//W16zDHH0PXr19O3ve1tJdtv3LiR/vSnPy3Zvpp2leP222+nt99+e2Dz9u3b6TnnnEM3bNhAjz76aPrFL36x5PXTfSz7+/vpunXr6Lnnnksvuuii4N/I47hlyxZ67rnn0qOPPpqecsopJZ/Zt6n4M1fLpnJ2XXDBBfSkk06iV155ZfCa7du30zVr1tDzzz+fXnXVVbSjo+Ow7aKU0h07dtArrriCXnTRRfTKK6+k27dvn9VjNZ5NYzHb59xIio/dyL9/8IMf0GOPPZauX7+ennXWWSWv27hxI/3DH/5Ar7rqqhmxqxwjf+ddXV30Qx/6EN20aRO99NJL6WuvvTbu9lOFT5jicDiceQ7vjOVwOJx5Dnf0HA6HM8/hjp7D4XDmOdzRczgczjyHO3oOh8OZ53BHz5kXXHPNNRgYGMB1112HnTt3VnVfBw4cwEc/+tGq7oPDmU5mpTOWw5lu/vSnPwEAvvOd71R9Xx0dHdizZ0/V98PhTBe8jp4z5/nsZz+LrVu3YvXq1di5cye2bNmCQqGA//qv/0JzczN27NiBcDiMj370o7j33nuxZ88enHfeeYEezhNPPIE77rgDtm0jFArhxhtvxHHHHYddu3bhX//1X2FZFiiluOyyy3DFFVfg/PPPR3d3N0466STcfffduPPOO/Hb3/4WpmlC13XceOONOPfcc/HNb34T+/fvx4EDB9DT04ONGzfijDPOwP3334+DBw/i05/+NDZv3oxvfvOb2LFjB/r6+tDf34+1a9fii1/8ImKx2CwfWc68oSptWBzODLN69Wra399Pzz77bPriiy/SZ555hq5bt46+8sorlFJKr732Wvqe97yHmqZJ+/v76fr162lXVxfds2cP3bx5Mx0YGKCUsg7ZM844g+bzefrZz3420Crv6emhH//4x6nruvSZZ56hmzZtopSyjsn3v//9VNd1Simlv/rVr+jmzZsppTToJs1kMlTXdXrSSSfRL33pS5RSSh977DF63nnnBa8766yzaG9vL3Vdl37yk5+kX/7yl2fu4HHmPTx1w5m3tLe346ijjgIALF68GPF4HKqqor6+HtFoFOl0Gs8++yx6enrwoQ99KNhOEATs378f5557Lm688Ua8+OKLOO2003DLLbeMUilcuHAhvvKVr+Chhx7Cvn37sG3bNuTz+eD5008/PdDeaW5uxplnnhnYMzQ0FLzu/PPPR2NjIwDgsssuw3/8x3/gxhtvrMZh4RyB8MVYzrxFVdWSv8upAhJCcNppp5WIWG3ZsgWrVq3C2Wefjd/85jd4xzvegddeew0XXngh9u/fX7L9K6+8giuuuAK5XA5nnHEG/uEf/mHSNgAItPR9m6Yqe8vhlIOfTZx5gSRJcBxn0tudeuqp+NOf/oRdu3YBAH7/+9/joosugmma+NSnPoWHH34YmzZtwq233opYLIbOzk5IkhRMIXr22WexYcMGXH311Tj55JPx+OOPT0rb3efxxx9HNpsFIQRbtmzB2WefPen34HDGgqduOPOCc889F+973/tK0iaV4E8e+uQnPxlogd9xxx2IRCL4p3/6J/zrv/4rfvazn0GSJJxzzjk4+eSTkclkIEkSLrvsMtx555149NFHccEFF0BRFJx22mlIp9PI5XKTsqOxsRHXXXcdBgcHcdJJJ+GGG26Y1PYcznjwqhsOZ5b55je/icHBQXz+85+fbVM48xSeuuFwOJx5Do/oORwOZ57DI3oOh8OZ53BHz+FwOPMc7ug5HA5nnsMdPYfD4cxzuKPncDiceQ539BwOhzPP+f8BGzg98J4V3VcAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"granted_burst\", data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 131, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 131, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABhcUlEQVR4nO2dd5xU1d3/P3fqltnCVnRBQERQEYhiQSM++NCkqAEiRCzRPBg10ccSHo2oGBODGp/woEZRoyYak4iKqITfimBiQCxIIlhAioDU7W122r33nN8ft8ydsrsz95xhZ3fP+/VStt0z37kzc77n2yVKKYVAIBAIBBYc3S2AQCAQCLIPoRwEAoFAkIBQDgKBQCBIQCgHgUAgECQglINAIBAIEhDKQSAQCAQJCOUgEAgEggRc3S0AL5qa2kFI+iUbpaU+NDT4MyARG9kqV2dkq8zZKJeQiR/ZKHc2yhSPwyGhX7/8Dn/fa5QDIdSWcjCuzUayVa7OyFaZs1EuIRM/slHubJQpHYRbSSAQCAQJCOUgEAgEggSEchAIBAJBAkI5CAQCgSABoRwEAoFAkIBQDgKBQCBIQCgHgUCQdciK2t0i9HmEchAIBFnH4foAiJhD1q0I5SAQCLIOlVJA6IZuRSgHgUCQVVBKAWE1dDtCOQgEgqxDMxyEguhOhHIQCARZBYWuHIRu6FaEchAIBNmFsBqyAqEcBAJBVkFBheWQBQjlIBAIsgpNMVCIdKXuRSgHgUCQlQjLoXsRykEgEGQdlIqoQ3cjlINAIMgqKAWI8Cp1O0I5CASCLIPq/xfaoTsRykEg6IH4g3J3i5AxjM4ZIubQvQjlIBD0MAihqG8JdrcYGUUohu5HKAeBoAfSmzdPzWroxU+whyCUg0DQw9CKxHrx5klF+4xsQCgHgaCH0ds3TmpoBxGQ7laEchAIeiDRKuLehxmQ7m5B+jhCOQgEPQxNJ/TuRE+jv5Kg+xDKQSDogRCgVx+thWLofoRyEAh6HL17hKbhMuutbrOeglAOAkEPw9w8e62GoL1d//UIMq4cHn74Ydx1110AgO3bt2P27NmYMmUKFi1aBEVRAACHDx/G/PnzMXXqVNx4441ob2/PtFgCQY/F2DR5HKxDEQUt/jD7Qhwxg9FCO3QrGVUOH374Id544w3z+4ULF+Lee+/FO++8A0opVqxYAQD4xS9+gSuuuALV1dUYOXIknnzyyUyKJRD0eHi5XBSVIiyrXNZqD8mQFcK8jpGqK3RD95Ix5dDc3IylS5fihhtuAAAcOnQIoVAIY8aMAQDMmjUL1dXVkGUZmzdvxpQpU2J+LhAIOoDjxkkpBWHfzwEA/oDMTdFoAXehHrqTjCmH++67D7fddhsKCwsBALW1tSgvLzd/X15ejpqaGjQ1NcHn88HlcsX8XCAQJIf3GE3CaSGue7mY59DtuDKx6KuvvorjjjsO48aNw8qVKwEkN4MlSerw5+lSWupLX1Cd8vIC29dmkmyVqzOyVeZslMuuTIGQjIZ2GWVlPnjcTiYZPP4wXN6IKQvLfQoToNDnRWG+h1mmgoYgSkt9KCnMSema3vT6ZgsZUQ5r1qxBXV0dLr30UrS0tCAQCECSJNTX15t/U1dXh4qKCpSUlMDv90NVVTidTvPn6dLQ4Ach6Z81yssLUFfXlvZ1mSZb5eqMbJU5G+VikSkYVtDcHERdXRuzcmhtj6A1EEauk/0+1Tf6EQlFEA6wKYfWQAStrdrzU8Ndtybvba/vscLhkDo9VGfErfTCCy9g9erVePPNN3HLLbfgoosuwpIlS+D1erFlyxYAwKpVqzB+/Hi43W6MHTsWa9asifm5QCDoGG4xBwCUU8yBEk5ymam6gu7kmNY5PProo1iyZAkuvvhiBINBXH311QCAxYsXY8WKFZg2bRo+/fRT3HrrrcdSLIGgR0F5bp6UwobBnRTCOcVIFMF1LxlxK1mZNWsWZs2aBQAYMWIEXnvttYS/qaqqwksvvZRpUQSCXgLl2rSU1ybMay+nCV/YR1YIAmEFRYxxkL6IqJAWCHoinCqkeXY/5VazrWsZwmE1WSEIhnrvSNVMIpSDQNDD4Jwxyu/ET8BlMQo9k5FLLISf26yvIZSDQNDDoFQ7VfPY1LUGd3wi0oRTHIRSABI/y4hXHUdfIyXlkMwn2dLSwl0YgUCQIhyDyFzdSpwUlgQ+lkNvHoqUaVJSDkZA2coPfvAD7sIIBILU4OTBATivw2Mj1txKHMcZCd1gi06zla655hp8/vnnCIVCOOOMM8yfE0JwyimnZFw4gUCQCOUYKNDcSlyW4tYsT7Mcsq89SF+jU+Xwu9/9Ds3Nzbj77ruxZMmS6EUuV0yfJIFAcOygsf9jX4sjXFw4VAtI8wgk81R+fY1O3Uo+nw8DBgzAiy++iKKiIlRVVaGurg4ffvghVJVP90WBQJAmlF8KKiX8Tta8iuAIKDTTgX0tCkAV6Uq2SKkI7rHHHsP+/ftxxx134KabbsJJJ52EzZs348EHH8y0fAKBIAm8PEtaEJmfi4pHbQI1dQMn7SCwRUoB6X/84x/41a9+hbVr12L69Ol48cUXsWPHjkzLJhAIksC1cI1vFRwfF47pVuJU5Cf8SrZIuc4hNzcXmzZtwrnnngsAiEQiGRNKIBB0AuV32uc6OMj8H+s6FBInt5KYC2GflJRDv379cP/99+OLL77Aeeedh0cffdRWW22BQMBOdIY0j5M1zzoHPq4gSriFHACA26S7vkZKyuHhhx9GRUUFnn76aeTm5kKSJDz88MOZlk0gEHQAv7RRgHIK2FJC+RSugeqDwHisBRBePcn7GCkFpO+44w788Y9/jPleIBB0D4RwDCKD82wIXgFpCVzaevDsHdXXSMlyaGtrQyAQyLQsAoEgHXicrAnvCmkO6wBa+wz2pcTQIAZSshxyc3MxYcIEDB8+HHl5eebPly9fnjHBBAJBcowsHn62A3v8gnKUSWu8xydWQITpYJuUlMOcOXMyLYdAIEgXDpse0fNPWVeKBslZJdIWkSQO60SXE9ggJeXwve99L9NyCASCFOGZt2+69blFt/koLAmc6hyEW8k2KSmH73znO5CSqPJ//etf3AUSCASdQwFA4jz0h3U1/XJuFdISvw6vRrfYZHuYoGNSUg6rV682v5ZlGWvXroXT6cyYUAKBoGPMOgAefYx4pcTq6oVbhTSntUQWq31Sylaqqqoy/xs8eDCuv/56VFdXZ1o2gUDQAVodAK+QNHvnUvN6XtlKEqdsJU4xlb6IrTGhe/bsQUNDA29ZBAJBCvDMVqKUcPNP8aq25upW4qi0+hppxxwopZBlGQsXLsyoYAKBoAM4++QBfu4gPmNCif4v+1owFameHytImbRjDpIkobCwED6fL2NCCQSCjqHQJqVxceGYMQfWOgc+6xgr8HIrGTEVkc6aPikph6qqKmzYsAGbNm2Cy+XChRdeiLFjx2ZaNoFAkARqtLTmlBnEB6r3MeI4z4FTlpGIONgjpZjD8uXLsWTJEuTk5MDpdGLRokV4+eWXMy2bQCBIgpnKymnP45EZxPNkblRISxzSdQnP5lF9jJTdSitWrDBdSddeey2uuOIKzJ8/P6PCCQSCRIyTNZeRnPpiPCqkJfBrj609PwrmUAHlmGLbx0jJcvB6vcjPzze/LyoqgtfrzZhQAoGgE3RXC58OqFo1Mg9Fw6vGzFBY4PAco4aD0A7p0qnlsHbtWgDAkCFDcNNNN+H73/8+nE4nVq1ahZEjRx4TAQUCQSxED0hzKfCinOY167swn/RT3VzgZRlBWA526FQ5vPTSSzHfv/DCC+bXos5BIOgeqOlq4dRegsOJn+r5tVz2YKq17IbEXpxn+JSEbkiftJRDMh566CHcdddd3AQSCARdQHlWEPMLSGvr8Ku94FVtLbCHrQppKx9//DEPOQQCQYrw2tABcBsRaq7HrX2G9j8uWVQiY8kWzMqBZ/tggUDQNUZ7CR77ullwxikgzcWaiXliHPxKnIL3fQ1m5SDa4AoExxoa8w/TSuapmr1COvYLhrUsX7EuR/TBQeIMmz7MyqEzli1bhmnTpmH69OlmMHvTpk2YOXMmJk+ejKVLl5p/u337dsyePRtTpkzBokWLoChKJkUTCHos0aph1g1dn7jG5XynObv49FbSDp285kgL7JEx5fDJJ5/go48+wltvvYXXX38dL730Enbs2IG7774bTz75JNasWYMvvvgC77//PgBg4cKFuPfee/HOO++AUooVK1ZkSjSBoEdjFpxxqAHQvuAzJpSHW8lUWOBi0IBwnH3R18hYzOHss8/Giy++CJfLhYaGBqiqitbWVgwaNAgDBw6Ey+XCzJkzUV1djUOHDiEUCmHMmDEAgFmzZol5EQJBBxCib8SsdQ5G6TC3wC81+yGxiBT7vdjVuwtm5fDzn/+8w9+53W489thjmD59OsaNG4fa2lqUl5ebv6+oqEBNTU3Cz8vLy1FTU8MqmkDQi+EU6+MZM5Qk9uXiel3wiDnoJYNsC/VBOq1zGDFiRKcB5+3bt+Occ87p9AFuueUWLFiwADfccAP27duX8PuOJlqlG+guLbXfQry8vMD2tZkkW+XqjGyVORvlsitTTVsERCXIzXExPS9VJShqCgIASkrymWTytkfQGJABCpSVFcDhsKclVEJR1BhEkc8LhzuMsjIf8nLcXV7XkdxHW8LwRhSUlvrgy/PYksku2fieS4dOlcOHH34ISimWLVuGqqoqzJ07F06nEytXrsThw4c7XXjPnj2IRCI45ZRTkJubi8mTJ6O6ujpm9nRtbS0qKipQWVmJ+vp68+d1dXWoqKhI64k0NPhBbOT2lZcXoK6uLe3rMk22ytUZ2SpzNsrFIlNTUztAKYIBF+pyUuqdmRRFJWhpCQCQ0JDrQpHPa1umtkAELS2aoqmra2NQDgQtLUEQRUFbQEZdnR95XTzHzu5lU1M7ZIWgrt6PYG7XSoYX2fiei8fhkDo9VHfqVurXrx9KSkrwxRdf4Prrr0dRURF8Ph+uvvpqfPLJJ50+8MGDB3HPPfcgEokgEolg/fr1mDdvHvbu3Yv9+/dDVVWsXr0a48ePR1VVFbxeL7Zs2QIAWLVqFcaPH2/j6QoEvR8jW4lyaa6krccjIA2w92nSWoNwTNUFn/YgfZGUjh3BYBDffPMNTjzxRADA119/DVmWO73mwgsvxNatW3HZZZfB6XRi8uTJmD59OkpKSnDzzTcjHA7jwgsvxNSpUwEAjz76KO655x60t7fj1FNPxdVXX8341ASCXgoF4OCRGQSz8R4P7WA0zOOWGcRBMKuuEaRHSsrh1ltvxdy5czF8+HBQSrF79248+uijXV53yy234JZbbon52bhx4/DWW28l/O2IESPw2muvpSi2QNB3YR1xEIMxG57HUgDzThwtygOfFFtiZGMJDZEuKSmHyZMn48wzz8SWLVsgSRLOPPNMlJSUZFo2gUCQBKOmgH2wjsV9w7h5aldzKkU2klE4tfUQCbH2SCmVlRCCVatWYf369Rg3bhxeffVVqKqaadkEAkESKNEH9LCuw6PKLHYxDhs6TfqlXcwcFaEd0iYl5fDII4/g66+/xrZt2wAAGzZswJIlSzIqmEAg6ASJvemlebXEr0I6buX017EqLMbnaFRbiyoHe6SkHD788EM89NBD8Hq98Pl8eP755/HBBx9kWjYBB2SFQFE5DfYVZAW8WlVoC+gxBy6ZQZR5LS2eIsV8z7KWhui8Z4eUlIPL5YLDEf1Tj8cDl8t+frXg2NEWiMAf7DyzTNBzoHqLCgn8gqw81iK8/Dc0tucTs3YQSsE2Ke3wJ598Ml5++WWoqopvvvkGf/jDHzBixIhMyybgAKEUEIZDr8JsTMe8D0fdN4TDJmp0UeW5HTOvJXGZkN0nSclyWLRoEb788ks0NDTgBz/4Adrb23H33XdnWjYBByilXD74guxAc8nzeT0px5M1MeaEAkw7euw2zuYOohYzRHwE0icly8Hn8+HXv/51pmURZABCAWfXfyboKegBB15T1wBOs5+jRgh7hbS18R6LSHoFHE8XXF8iJeWwe/duPPfcc2hubo65ycuXL8+YYAI+UEpBRPuAXoN14+Uy/1lP52HPVqL6jAkOp3SzzoF9EpyBsJ7TJyXl8D//8z8444wzcNZZZ4mxoD0MSrIvjY9QiraAjKL8Y9sls9fAqfWQtX0Ga5smYpRtMyqs+D2cLZUVZqsRQfqkpBxkWcY999yTaVkEGYCCchlEzxNVJWhtDwvlYAPTVcKletiIE0jsIVtzKQ4DHUwXFYcxoZLEx23WB0lJpw4aNAi1tbWZlkWQAXgGHXlBKWy1VxfoRIuRuU1dY7UcqNHxifJwK0X/ZdvULcV0jCL1RVKyHAghmDFjBk477TR4vV7z5yLmkP0QgqwzqymYvQ99lgS3C+w34TMODpLEXkNsKhfGTqrWbCzWpqxGtbUEiX2kah8kJeUwadIkTJo0KdOyCDKA5lbKrjgRpVRYDizwKnSwrMX+ctCoWCyrWPtwSABhVDRGtbWodEiflJTD9773PRw6dAiffPIJFEXB2WefjUGDBmVaNgEHCKFwZpvlINxKDFjum5HPyqL7KdvlBtECaX6vK3Og3AxfCMvBDiltGxs2bMDs2bOxbt06rF+/HnPmzMG6desyLZuAAxRsp69MwSu1UFYIIjKfDsGyomZ94NLamI69pkDXDBIfN58RkGbqrRTTMoMtUG5cK3GZZtT3SMlyWLZsGf70pz/hpJNOAgDs2rULCxcuxMSJEzMqnIAdStmDjbyh1OgRRJlTowMhGYpKUFqUyyxXfXMIxQVe5Hqzt28Yjfuatcld9Bu2zdNMZeWxB0fHOXB77wpDNX1SshxkWTYVAwAMGzZMzHPoIVBKsvCDoaXX8hCLAlA5nfZ5rZNRKMClT4V+OaWUT8qotaqZMYPKej2Pauvsirj1HFJSDjk5Ofj888/N7z///HPk5rKf1ASZh1r+z0p9S5BL+29TMXAQixDK73RJ+FXkHhO4uYLY3Xxm8JdDmYN1CS7V1pzcZn2NlOznhQsX4oYbbsCgQYNAKcW+ffuwbNmyTMsm4ICWysrnkxEMqyjIo3BxaNakuZZZo6napsYruE0pzfqsFqt8rIVwRuSCh0veaLxHGec+a4aRpEvGtqlbewH2LK2fHaSkHMaOHYu//e1v2Lp1KyilGD16NPr165dp2QQc0NJG+RjWKuHTi8OIN3D5vFI214MVlWT/HmKVj9kq5NTgzlhAj22z57Ia10tgfEEssy8YVumrdKoc3nzzTVx66aV44YUXYn6+d+9eAMC1116bOckEXCCUwsmpHxalfLKMKADKyYVDwLE5G8myyH1H8GrZDZ6zIXQkfhsxn/YZ4Davoq/RqXLYv38/AGDnzp3HRBgBf3ht6ADH2gTK4dRrLEV01xkHiG7RZD1SNCDNljaqWV08Uj1NORhTqDS3kn4946ZOdcFEJqs9OlUOt9xyCwBgyZIl5s/8fj9aW1tx/PHHZ1YyAReMUzoPVE6nfWMAERfLgVAQTqPuCO0hbqWYOgCWtaL+G/YDBGXvuYfY+8+lfYZkVEgL0iWlbKV3330Xv/zlL+H3+3HJJZfg0ksvxR//+MdMyybgACX8Qqy8ArZGuiKfkAO/bCWV473KHLTTb9NdSTIbTLChKRdJm8HAsI6RWmt+zyyZsS6nhfoQKSmHp59+GpdffjnWrl2LMWPG4O9//zveeuutTMsm4AAFP3cQP8sBZo49j7W4bSA9wHSIKXNgXctaIc1BwZrOLoZbaLVgJLYpobqiMdbpIfGkLCIl5UApxfDhw7Fp0yaMHz8ePp+vZ/hm+zhGRhCXEzrH19uwGvi4lXgpGX6FeRmHU5EYr75KgK5cJPAZySlFv2DZ1KMeOJGtZIeUlIPD4cCaNWuwceNGnH/++Xj//fczLZeAA5qhr4d+WYucqOGTZ/+Y8Yo3AAAB4ZZBhR7QLTamaynjfSTm1slX+TMFkS0VzTy6cRjribNs+qSkHO68806sWLECt99+O8rLy/HUU0+JyXA9AY4BOQrKrYKYp+XALVtJl6VHpTwyRoCNMAGXGdLGps5oisS4zVjdXYZlxOie6qukXAT3hz/8AYCWrfTb3/5WZCv1AIjhUzKOYAwfXKNPDY/PWLSWjocVwtGaAd8TdCaI7VrKupihG9jcN/pSMLpnMPVWIvxcXYZdJHor2UNkK/V2JAoeVcSU6m4ILi4cbR0+HhzK7bSvWUdclsoslr57bO0zoicG5iI4I8uI0QqhoGhpj6CpLay3z2DNZdXoURZhliCylXoxZk68xMMdRM24A/NKHDdgXvELSqNFYdlMjHXDOpKTgMv0NiC6D7MO6KEUeG/LIbz1wT7mGdLmlcJ0sIXIVurVUHNaGPvJ0Mh+4uPC0dbksBafdk8w1ELWZzzGZRixntIBXUGwJiwkWdfeQhShiIpQWNHdXUxiAZIxQ1rsV+kispV6MdoHixqffg5r8eutZP2XaS1KuXzwiR5TyXb3Q4x0HDLQOC0FUKNCmr1TrEIIZIVDa3jL+yK7X9XsJK1spdtuu83MVlq0aFGX1z3xxBOYPn06pk+fjkceeQQAsGnTJsycOROTJ0/G0qVLzb/dvn07Zs+ejSlTpmDRokVQFMXmUxIYRBMVOVgOekMkLsqBUDgkDiky0Ft28whK6y64nnDAjCj65smhZbdWJMZeB2C6lRitEEoBRaGQVaIXr7FJJkECp7danyMl5WBkK82dOxcA8Ne//hVnnnlmp9ds2rQJGzduxBtvvIFVq1bhyy+/xOrVq3H33XfjySefxJo1a/DFF1+YVsjChQtx77334p133gGlFCtWrGB8agLrJ4LHh59QPm4XI7+eV4U0wCdVl1oXzFYoxd8+3I81H+5nftJmKiuibkPbawHmyFcWsQgFFDVqOTBXWxvPrydo/Syj01TW//7v/8ayZcswc+bMpL9/++23O7y2vLwcd911FzweDwBg6NCh2LdvHwYNGoSBAwcCAGbOnInq6mqcdNJJCIVCGDNmDABg1qxZeOyxx3DFFVfYeU4CnZhTPqfPBpfeSgTcTnPmhm7Z6GytQy0ZWVkMBeAPytqAM9aTtaWPEWvTvJiANFsgxHQrMVs0NCaxS5AmnSqHBQsWAADuvffetBceNmyY+fW+ffuwZs0aXHXVVSgvLzd/XlFRgZqaGtTW1sb8vLy8HDU1NWk9XmmpL20Zo49XYPvaTMIqVyAko84fAQCUlvmQ602prCUp/qCMwoYAiovzO5UrFZmbggoUyYGSknyUFrGNmy1sDIIQirIyH5zOjg3hruRqD8ooagyiuCj3mL0f7DyOw+PSXF+ShMKiXJSW+lDk89p6/NaICk9QQY7XBYcrDErtyUQpRXFDAEX5XgRDCnz5btv3sC1CoKoUKqEoKsyDwx1BWZnPtEo6ItnjqZKEEAEK8jxwusPH/HOerftKqnS6W4wcORIAcPbZZ6Ourg4tLS1pP8CuXbvw4x//GHfeeSdcLpc5KMhAG3WYqNe7ejPE09Dgt9X6oLy8AHV1bWlfl2l4yBUIKWhtCYJSiro6D5NyaA/JaG4NId/tQE4He3CqMjc1taOtPYK6ehdIxH5siVKKpqYAJImitq4Nrg6UQypyBUIyWltDcFKKohz79ylV7L6+ja0hyIqWzdPcHEB9rguRYMSWDI2N7QjLKrxuJ9oCMihgSyZCKVqaAyCyglBEgRz2wOdOyWOdKFOT33Qp1TW0ISwT1Na1aTGqDujoXtY3BdDSGoQSkdEWkFFb25r2vmKXbN1XrDgcUqeH6pQ+BUuWLMHLL78Mny+6kCRJ+PDDDzu9bsuWLbjllltw9913Y/r06fjkk09QX19v/r62thYVFRWorKyM+XldXR0qKipSEU3QKboTiDFwCWiuAgl8CteIntnC3O8J0UQsHvFoSaJZ372TAlBUCkUlHEZyxnSqsH8TzRfCmN5mXyiiWw0ANCXB4jJMkvYrSh5SJyXl8O6772LDhg1pzY0+cuQIfvKTn2Dp0qUYN24cAGD06NHYu3cv9u/fjwEDBmD16tWYPXs2qqqq4PV6sWXLFpx55plYtWoVxo8fb+8ZCUxMfzyXVFbKp+MmtNoESeIwYMa4nFOqriYTm0iZhlIKVSVQVMo8kjOmNoHav4MxykBii0vJKrV8TeB0OGIqudOXyxpxt7VMnyUl5TB48GAUFhamtfBzzz2HcDiMhx56yPzZvHnz8NBDD+Hmm29GOBzGhRdeiKlTpwIAHn30Udxzzz1ob2/HqaeeiquvvjqtxxMkYnbw5NATyViKV9dS1kpaIJpey6MCvMd076SAQjTLgUe2kullkfgoatbX1VrfICsETrfDvkFDrM9PaIV0SUk5XHXVVbjyyitxzjnnwOWKXvLTn/60w2vuueeeDju3Jmu9MWLECLz22mupiCNIB31XZy9yivmHCUK1Ogce/Z60TY2D20z/L9uL4FRCQAjVLAeAsabAsoszuOair4O5lG0UVTW/lhWCHA9Lem1stp5dC6SvkpJyePzxx1FaWoq2tuwOsAhiMfO8Oex3PN1KlOgxB0YrJNo7ikOKLYWmsLJcOUT0k7WiEvaRnAAaWsKo6JfL7pqz7sMMy8RYDozWUYwcXPqL9S1SUg7BYBDPPvtspmUR8EYvbOooIyzNpTS3Eo+YA3j592nUbcbBraT9yypTZlEUTUBFZS8Saw8p+P3qr3DZBUMwqH8hk+VgZAFJjFHySJxbicXqFXYCGynlmw0bNgw7duzItCwCzmgbugTKZfMEt1nDhuXAqhxie0cxrgW+MZVMIetuF0qhD1+yL28orIBQTUkALFu61YJhO4hYLQfF/NreetYKcB4ZbX2NlCyH2tpazJkzB1VVVWbFM9B5hbQgC9AzNHj0ztHiBFyksqSgcnArQTItJBaI4epiWiXzWDdPQtjcSrJiSRnl9MxZvZhKXLYSwHCwsVSAi4B0+qSkHG6//fZMyyHIANreSQHKPidRO+2zp58am7gkScytKij0DCMOmzoxYiqIuuKyEcV6sma0HGRFs0I0F5X9E79pwemwWJeGTNrXulw21yLxhQ5Zr/qzi5SUA0uFtKD7MF0kjIFLQPug8XC7aPsI5ZPKalgMnLKxjCB5NvuqZTV601SVMu13xlrmmgy+/YN17di4bTeumnIyHJK96mhNFovloDDKFfP+EgHpdMlohbSge4mehin7pk70bB5WoUyfEqf22LxafxunTGO9LNUOVreSSghTlpZhhSgKZXMHUaCmMYgjDQEEQwryctwMMsVaDppc9p+jZPlK6Ib0yFiFtCAL0Dc5iXKIORA+lgMx4yBxefY2oJaqV2Z3lzHYnurxlSzVDtaTtULY2pkYFkM088mmWwnUXEMmbM7C+FRWzc1nb62EeyO0Q1qkZP/ZqZAWdD9E98mDR00BqJl+ylwLIEX9+2wyIdojmnExTelJ7JXCGUaWrW4ltjoAQ9HIKmF+TQ3loCpscRDF8j5VjN5KNiGE4vk1O/D1t80AONTC9DEyViEt6H6s7QOYg7+cvC5mhhGHtNjovsFD0fAZc5lpVBIbc2BzK+kBacXaZyJ9KI0qB03R2F7KdHU5HJJm2TDEk0IRGbVNQRxtDOD4snz7QvVRRIV0L8YIrUqgzMFfQiyZH0w+eT1bCRwskOhyXNJizaeXxdohYmkvoahsQVajoM6oRGZay7BCGGc/G9fnepx6V1bArnlkBtx1CySLX9asRFRI92LMNhWUvY+R5qIymvjZ1w7RwjU+6adGoIA1uE20rElku/PB2NAB3YpgSWU1Yg4KYXo9rJaDyphea7iVcjwuvULa9lJxrUayM4aUzYgK6V6MmcnKI+ZgfL4Ye9QYbiUJEns1sh6t5NHziYDoBVPZXQmXUOdgcx1Ko837ZOYmftSUS1bsu5Uo1dZxOR3wuB3RE79NqYz4jJn1lMWvazYiKqR7MYYniII9bdRUNIw+eaNwjUcGqumg4pWtZLT+z2LtEFvnwObfN+IXWvtvxmprXcEoKlt0SyUEbpcEl9PBHHNQrDIxuKf6KqJCujeju10kyqOPEbFs6PYXMwvXwG7NGLuGBHblYIqS5b5pa3sJLSBtDwprnQNjQBpRtxJLHMRYx+V0wO1yICKrTPt5WA+4s7qn+iopuZXOPvtseL1efPLJJ/jggw/MnwmyG6Knn/KwqY3QNg//PiSJS1dWLfvUaP/Ntpah/CQe1dYZRLYEpFXCkGhAo/59hTEgHZOtpBD7I2CpplxcTgfcToe5qdt1GZquLg6WUV8kJeWwatUq3HLLLWhpaUF7ezvuuOMOrFixItOyCRgxCrt49DGKBmzZ/EHmDAbA/iZirkXNOAHz4CBNImbLKNNYLQfta3uyxhSuqZRpEwb0Vh5gc1FpMlG4dcvBqL+wK1XEEgfh0mKlj5GSW+kPf/gDXn31VVRUVAAAFixYgB/96Ee4/PLLMyqcgA3js8CjjxGhFE7dNmfZiKm1GRpl62NkZCvxCLgTAjgcyPoTpqIS7VStEm0qnF3lQGN98ixQSmPTRgFb6c6GBeJySppy0APJdl8QxZqtJEiblCwHQoipGACgsrISDof95lqCY4PWXVT7mt29r7uoGH3y1noCMA4hInoDP+1r+zJpchnV5Nl9wlQUCo9b++wx1zlYNnRCGIvXrE38GOIXikrgcjlM5QCGfl6yxXLgVlfTh0hphy8uLsa6devM79etW4eioqKMCSXgA9GnnbBO5wIs/mgOm6f1eiZFo+9DPAYHmVYIsnsTMQK2LqeDuX2G9UTNklYcG5DW4xc2BVP1mIPLiDno6cp2sCoH+w64vktKbqV7770XN910E375y18CANxuN373u99lVDABO4QADgmgYJ/DYDTe02CLE1jdSixEK8A5zKug2kqkB7iVXE4JLqfEWOcQH79gcBdaXVQKsV0LQymgkGi2klFQZ/c5xmZQZfOrmp2kpByGDRuGN954AzU1NVBVFUVFRaisrMy0bAIeGKmsrNk85hfsQ+3NDyqjFWJUgEMCVB7ZWBKYfNzHAsVysmarc6AxloPCYIUQGlsEZx8jIK3FHABNWdh9jtY4SLa/rtlISm6lNWvWYNasWRg6dCjcbjcuu+wyvPfee5mWTcAI0WMOrJ0DrNPbmDd0Gh+nZIk5aP/yqH6NBrTZq60zSdStJOmbO0tAOn42hD1US+GbrA8gsm05GHUOTm1r0iqu018stgKcbaJcXyUl5bB8+XK8+OKLAIAhQ4Zg5cqVePzxxzMqmIAdM9WTsYKYaosB4OCTp9Y0FraaCUKjMQc2kai+lsRlal4mibqVHFBUyvS6qiqFQx8MztK222otsPYxsgakDRntpcVGu84Sog+7ymKln42knK3Uv39/8/vjjjsOhNVPIcg4pvcGElOqpzGKEwCzT54ClpkJjD00LHUOhCFX13ApGTD3fMogsW4l9iByrsepr2vffROxKIeoorBx2oelCE5XDjKxqbRo3GwIxrYefZGUlENJSQn++te/QlEUqKqK1157DWVlZZmWTcCINZWVcQ+OFq6B7QAWv/GyqBrVkmHEdFaxiCBxCN5nEpVY3EqE2H/eehA516uFHVWGoG1EjrMcYDPFlmouKsMyAgDVrlvJ0gwQAGSidvLXgmSkpBweeOABrFixAqNHj8aoUaOwYsUK3H///RkWTcBKTFdW5g3PetpnO7FKhluJV80Ea0xFy70EoFsQ2asb9JO1BKfuVrKLUSFtKAeF2H8tIglzn+359xVCQCjgdsVZDvbEih2pqrLPUe9rpJSttHv3bqxcuRItLS1wOp3w+XyZlkvACKUUskLw0Vc1OGtEBZNvn1r9Low+eWodGsQYKTe8XRJrTIUi6uqS2KfmZRJrQDooq7aVvkooVEKR69XcSqpKQAA4baxluJVyPM6o+8aGWBFZUzJWt5Kq2OsflRBwV7L3Nc1WUrIcli5dCgAoKioSiqGHQAEcqvPj3c0HcbCuPaYbatprxV3HcgLTmgFGhWSrkNayUNhjKlFrhkerkUyiqhQulxZzYKlzMDbOHI8r+r1NH5Wsb+q5XhdTKqtxbUy2kkpsux5jUnUJyWp3YTaSkuVw8skn46mnnsLYsWORl5dn/vy0007LmGACRigQ1j+04YjKNPvZeh2rT95oBqgvxhwLMdtnMKwTG/ngMIQoQ1BK9SIxI1uJPcPIsBwUhvbfEXMtF9pDsiarjdUMmWLqHFQ2y8Fow8FSx9FXSUk5bN26FVu3bsWrr75q/kySJKxfvz5jggnYoKDmh80w11m0gxlxYPTJE0rRHlLMgCNzzMHB3niPWvwgmhLNzl2EwtpeQtJiDnbjBHJ0Qwd0txJjzCHP69TGmNqMJRkyuVwOuEzlYNdy0Ooc8rwutCgRLeaQpa9rtpKSchAFbz0PSqNKIawPTbHbAzXGUmCtRqYUK97bg5MGFOG80/p3/fddyOXUZ0MwZ2NZ3UpZuodQqsUJYiqkba4l6xt6rulWorYPD1ZFQyiF3Ti5MavC6laym2JLqWaJ+HLdaGmP6HUc9uTqq6QUc2hvb8cDDzyAa665Bs3NzbjvvvvQ3t6eadkEjBjmflhWmWc/G7AWwREKtAUiaG2PMM8QoIRGM58Y5DJGlwLQAtJZGnMwUjOdDsmskGZ3BeluJcLg21dirRC7G3rEGnOwWA523XxaNlY04C4Mh/RISTn86le/QkFBARoaGuD1euH3+3HfffdlWjYBA5rlYLiV2D4YMXUOksQUsFUIRUQhCOuZNiwKSyYU/9x6WLeM2JrQtbSH8dW+Ri2mwhjByBSG+8awHDR/vM3aBH0j9ridkCS2VE9jHKepHGwGpY33q1tvLAhocllTZVNFK6gjyDNlolnrLsxWUlIO27dvx2233QaXy4Xc3Fw8+uij2L59e0oP4Pf7MWPGDBw8eBAAsGnTJsycOROTJ082s6CMx5g9ezamTJmCRYsWQVEUG09HYEBp9EOlWQ72ewZRULQEIth7pNX83i6RSFQmVnfQ0YZ2/HPrEXxzqJV5NsS2PY14/f1vNIWVnboh6pPXA9JG11I7WDODXE6H7Q0dSGI5EMtpIp111KhMkqQFpVWVIiynLxslWswhN8eQiWTt65qtpKQc4gf7qKqa0rCfrVu34gc/+AH27dsHAAiFQrj77rvx5JNPYs2aNfjiiy/w/vvvAwAWLlyIe++9F++88w4opWIMKSMU0dOhEXuwu3dSCmzZUYvX/rEHEgCWwVrBiBKViTIIBS0LCwBCsgrbVbnQFGk4oppxmixNVjK7jBoBae1nbMrB7dL8+4pqP9UzwUWlqLaUvqmwdJeSlq5LtGB5mi9KNEjuMtcWlkN6pKQczjrrLPzmN79BKBTChg0bcPPNN+Occ87p8roVK1Zg8eLF5hS5bdu2YdCgQRg4cCBcLhdmzpyJ6upqHDp0CKFQCGPGjAEAzJo1C9XV1faflQCwuJXCsj4Ji2G5UERFMKzq1cTsG3o4ojIV1FFKLam6CpvbDNHNRNvosnMTiT/tA3o3VRuvh7XgzJwNYTeQrBA4HRI8LsO/bzdbSZPJCEa7XbpFI0kxNQupEJb5xEH6MillK/3sZz/DM888g4KCAixduhQXXHABbrrppi6ve/DBB2O+r62tRXl5ufl9RUUFampqEn5eXl6OmpqaVJ8DAKC01H5xXnl5ge1rMwmLXMGwYp6UCAWKinJRVupDjjellzwGjz8Mw+ubl5+DosKcDmXrSmb3nkYA2iZcVJSHfv3yUVacm7ZMhFA49c1IcjpRVJSH0lIfPO7kNb6dyZUTiJgZNp4cDwoLc4/JeyLdx2gKajUExYW5cOjPPS/fi7KyArO7aqp4c+sBAKX98uH1uDRPALX3nnPo7S76FWmvoyfHjdJSHwrzPenJlOMxZepXmIMcjxOSw4Giwlz0K8lHXo67w2vj5Q7plkZxYS7cTgdcLheK++Ud0896tu4rqZLSTvHRRx/hJz/5CX7yk58wPViyE47Uga9YSrO1QkOD31ZArby8AHV1bWlfl2lY5QqGFQSCmgunPRhBc0sQdfVtZkVsOrS2R9AeiAAAauv9oLKKQk/iJpyKzHWNfgCaJdLSHECdSwKV048vqYSguTUEAGhuDaGlNYi6urakyqErudoCEbTrG299YztynBLqfOltbOli5/WtrdPvXSiCSFiTt6kpgLr6NjjS/Lw0NgUAAIFACA4JCARlEFBb7zm/PwKnQ0IopL1HWlpDaGjwIxzoeDNPRlOzJlN7IAQQFQ5JQnswgpbWII7WtKIgL/lrkuxeHq3Vvo+EZTidEvyBMBobA8h3peQsYSZb9xUrDofU6aE6pTv1xBNP4KKLLsLvfve7tE/0ViorK1FfX29+X1tbi4qKioSf19XVma4ogX1MV4lMmJrcUcRWW7P45ENyNPOEpWo1oY4D9p1BFNG1Igz9iuzgD8poaAmm9LdRt5KlaymxVwin6L2G3GbmE0O1tUoS0k/trGXEVKxuJVl386XblsPMfNKtGq2OQ/iV0iEl5fDKK6/g2WefRSAQwOWXX44f//jHWLduXdoPNnr0aOzduxf79++HqqpYvXo1xo8fj6qqKni9XmzZsgUAsGrVKowfPz7t9QVRrKms0Wwl+4uZabEKAWVI+whFosohrBCmqlWrwtLktLmQtdWIrB7THjyqShAMp5aqGRtzkPTr7aVoxqTFuhxMilpWVLhd0TiIFvy1s070+Wn/SmY8I9101pgmfk5NyYgK6fRI2cYaOnQoFi5ciMcffxxNTU24/fbb034wr9eLhx56CDfffDOmTZuGE088EVOnTgUAPProo1iyZAkuvvhiBINBXH311WmvL4hCQWOL4CwtItKFWDbPCKPlELYoB1lWbacXUhpX5Af7nVk1yyFaE3IsD5iySmKG5XRGJElA2m6gNZoZJMGtt+KwX20dO9rTbsGZrBA4JJjxE7fLqSsHh/kap0pEjbccRCpruqTkgG5oaMCbb76JN954A4QQzJkzB08//XTKD2JtvzFu3Di89dZbCX8zYsQIvPbaaymvKegcVY0OkA/LWpaR3Q+/qhJzMwkrKtPmaf2QRxT76ZPWbCWrNWIHQkiMi+pYni8VlaR8Ko51K+lFYnY7qSoEkgQ4HVHLwe7mKeszJmL7IaVPRFFNpQfA3NSdDgnhSJpuJf09YXawZcm/7qOkpBwmT56MyZMn4/7770dZWRkGDRqUabkEjBgbZ16OC4GQYju9EAACkWjAmNXtYrUctFM6+2k/HNF7R9kUKxSJKoRQRGVq4pcuskJTLkBT1ETLwe6oUOO0DyBa58DQPsPjtgzosS0ThdMZDawbMQenU0I4RdebuZYlfmGsk631K9lKSm6ll19+Gdu2bcMNN9yA2bNnY+LEidizZ0+mZRMwEAxrG3pBrpYxEknTLLcS0NswA9CLxVjiBNGNMMJihcTFCbTmSPYWM+4VEC2CO1ZBaVUl+uCdrhVEfPsMAHqVdPqPq53SJXM9loCtrBCzmA6A7bXkeMvB6YCsEkiS1kY9lXtkYA1ImzEVQVqkpBx+85vf4L/+67+wefNmfPrpp7jxxhvxi1/8ItOyCRgwXC0Feq55xOYsXgAIhixB5AibWykiq3DqPmWWamQS71ai1PZaAYtyCMnR2RfHAm0ONE3JAkjqVlLtFe0ZGUZA1H1juwhOX8vhkOCQJCiq/Qppq3JwuSRLlhJNayyqoUjdTqMCnDIlUvRFUlIODQ0N+N73vmd+P3v2bDQ1NWVMKAE7pnLQLQet0Z29tYLxbiUG+zwsq2ZxFItbybge0DZIhnHKMZYDc+ZTmsgKgcOhp6R2gZI0IM3uVnLp2Tx2n7KSoGhSt2YisormNq1eRVMyFreSualTgEpaoDtF4lNZWZ5fXyUl5aCqKpqbm83vGxsbMyWPgBOmcsjTlINss98NAISsm6ds/0NmNAM0ZAozBaS1jcXYlFjqE4xUUpdTipt9kXlUlUKSpJSUg5Gt5LTUObBkK0XdStrjKzaVvqwQ06VkthJPUaiQrOJQQwCKnvQQH5A21qdIT77YbCw2y6ivklJA+sorr8TcuXNx8cUXAwD+3//7f7jmmmsyKpiAjZB+2o+e0u37lI3N0+t2mkVslNK0q9iN2ouCPA9cToe+odsSCbKiQiUUpYVeNLSGEI4otrdzw3IozPdEez4dg42EEGP8UmrKQVEpnA5tuJERuDXWSJeYgLSxCdvM+tLGcRrpp4600mIVhcAfkNGqD+SJVQ7GCFMtsyqdzrGyosJhycYSw37SJyXlMHfuXJxwwgnYuHEjCCFYvHgxzjvvvEzLJmDAcI/4jIC0yrB56msV+fTNk9qbKUdBEZFVeN1OeN1syqE9pJgyNbSGmOoTDEValO9BY2uYKfMpHbTqZgrJIZmT2TpDjgsiA3pWjm3LIepW0tayMTeBarGAmE6qaVgOEUVFrteJ2uYgZIXA40puObhdzphMt64Ixyg/a+xCkCopN9oZN24cxo0bl0lZBBwx4gRGPxpem2eLPxKdnWDDcggrRFMOHqfmorIZJDTcZoWWgLvdJxjSLaOifA+ONATYB2WnCCHaPXQ6JMgpzCyQVQKnxX0DMKSyqtGNOOqaS/+1MFt/xwW3U719sqy9H8KyVmuSZ2kMaawpKwQ5XldahXCyxeXodjq0saEiYyktjk0XKsExx9w8jZiDbN+/HwwrcDkl5Oe4dEVhz+1CqW45eBzwup1M2UpBvfGc8fzCMptlJEmAL8+jB+6PjQvCeD0ckoRICqd2RaFmppekKxWV2CtvTOpWsjFxzazajrEcaMo1E2FFhcMhwet2JIk5GDMrjBYa6biVSFQm4/kxpHP3RYRy6KWEI5rPNU+fhBVhaKwWiqjmaT9ktuJIH8N60dxKTqZq66irywtAC5TbzaIKRRR4XE7keJz6YPpjE3NQiTa72uFIze0R75N3OR1QbfrSlRi3kp5abMP1knQGQxoyhWVt4/d6nDFBcsCyqRv9ldLY3COWtawWiCB1hHLopYQiij4fWILH5YDMUNkcNpSD24mITGwXiRmB39i17NZeRF1dAFsrDk35adYMoCuxY+RWIpSm7FYyWkkYGJlBdpAtQWQz88nG5mmdKAdEq61TuX2EUBCVmL2UVELN18BYy5BV0htHpvp8rZls0cptoRzSQSiHXkpYVs0Phcft1IrgbH42QrIKr0c7WQNGQV366xhxEMMKYUs/jc/Gsm/RhCKxzy8sK8fOctAbzSkpzINOKBJzOpjST+M3T1uWQ5xycOnZSqkoakUlMXErRaVwu5NkKxlySanHWGSVxFgzgLAc0kUoh15KKKyag2+MU7rd03DUraS7qGw2pzNSYj0eLVuJpTDPSKnNz3GZLgeVg2UExLb4yCSqSixDemiX6axGnyEDl1Oy3TPLWrhmZivZeN7Wlh4A9A6vqVkO1uerZT11XOdgyp1iCw1ZJjFxkPh1BF0jlEMvJSxrrhIAWtqoYr89djiiIsfjRI65edo78VvdSh49Q4XY7CoaCMVbIQwxFVmB16OtAxy7gT9adbSx2UtdnorlBLeSQ98s05PVTD+N88nbGdJjrUQGdMtBSe0gYlUixnN3W5WDLp+xqUuIuhO7QrMcorUXxs8EqSOUQy8lLKvmwHcPo38/+eaZ/joxbiW3E4Sk1y8nfi23S+vnk+N2agrLtvIjyHFb3ErHqNWCUdwFAKBdt9BQ4txKTqdDH/aTHglDdRhabRsZTrExh9SsGcOtZjy2JlNsV1Yg6rryehxo8odTkitisRxEQNoeQjn0UsKy1kYZ0N1KDO0zwhGiWQ765hmymRIYMiutHaaiCdns8hoKWywjj9OcWWGHsB5TMQPSjM0FU0VWadStJKHLrqPxvYfsBqTNOIHFFQTom2eazzscl61kVCOnchAJR1RI+g4kq7EKCwByPC543U7UNmljVN0uJ0JhJeXMLnec8lPVruM6ndEWiMT04ertCOXQSwnL0ZiDx+3Q3C42gpeKSqComnIwLYcIY0DashHbbYgW1LOxjPWMLKp0MYYGGe4pAGkVW7GgWDJ1UrUcnI64VFYbLbujfYfi6gBspbLGrmWe0uUUekVZOvQaFqRhLQBaoH7I8QX45nBrdFOXpJQ2aGv7b2sdB4vOb/EL5SDoBRhtKoCo5WBnGzaK6bzuuJiDnbUsPZqiwV/FVpaREUQGYHEr2SsGI4Qix+OEx+WAJBmtxDNvOiiWgLTDgS4nwvGyHGQlrjbBGnNI88VIsEJMF1XXCjaiqKayU5JYDgAw9PgitLRH0NAa1n8voaW9a9dS/DAjAFpmF8PLGpaVtGdZ92SEcuiFmKdhayqrbC+V1Rj0k+NxcYk5OB1aR1HDJSQr9ip8Q2ElRvmFZXvKzxoklyTJXCvTQQdKKdpDCp556yscrPNrhXBdZAspany2ksOWqyRa1RzXp8lGirJRdeyOywxKpeLbKIAzHtt6vcGJxxcCAL453AJAe51a2yOdKm+iB9zNOIihsFIMlHcm77GyKrMBoRx6IUYdQjSVVXM/2MnWMAbh5HiccDkd2jxfuzGHiGr28/Ey1hSELJaDWTNhQ/kZldY53vi038xCCEVNYwC1zUHs2N+sFcJ14daRFQpXnFvJTp2DuRHrazkcklZrYSt+EZfKagSRu1B0hGqpuw7TrRSrsAz6FXhRUuDFnkOtpqyEUNMKTYZ1KJJVtnTmTCTIS7RxrunOsu7JCOXQCzFcQUZA2lAS6XS1NAgEo3ECQFMSYZuZT6GwElN7Adhr9gYYhXnWgDRJqe11PEFT+bn0f522U3XTQSUUja3akJsDtZrl0FkRmlEHkFjnkL4ii8Sd9o217LS1NmpCrLMhgK7jF6qeNvuvnXVo8YfNmEO85QAAJ1YVYt/RNnPYjyRJ8IciHa5tPL9oY0GjR5P9U7+iEiDNFh49HaEceiEhS8qo9d9QJP1gWiBiuJWip3S7cYJQRI3JoALsV1uHI9GAuxkLsfP8dLeZ121J+2WZbZ0iKqFobNN854fr20G7sOyUJHUA0dnP6T12fEDaWNdu+wyXUzJne6RajayoFE2tIazetB8vv7sL/qBsyhHP0OMLISsEB+raAWjvwea2zpRDrItKkjRXpp3nZ5VX0m91X5lHLZRDL8QwueNP6UE7lkMoGkQGtI3YrttFcyvFymTnlK4SgohCYhQWYC/F1iymi7OMjoXl0NASgiRpX9c0B0E6SQE1NlunZfN0OiWoxIblEBdEBqKjQtPVNBFLVpCxjvUxOkIlFAfrAwCAhtYQqj/+NuZ6K4P7F0KSgD2HtLiD2+VAOKJ2uEmbrq642RCKjcwuA2uw3m6b9J6GUA69kKjlYLiV7PfrD1piDoClpsCOW8mSfup0SmYQNt2VrJPprP8GO/FDd4QZU3HHPr9MZysR3a10UlURAOBgrR+QOq6SlpMUiRmWQ7qvRbLgb7rdVM215LiWF2Zwu/PXQlUJDtb64ct145LzB5uuUOvzM/B6nBhQ7sM3h1stP6UdWifG+zx+cJBiQ/mZayqqlllG7cXueiJCOfRCDAvBOA3HpI2mu1Y4/mTtst0Tyeh+CiCaGaSknxlkzTCyyha04VZKeH6M1eSp0tQWQjCiYshxBSj2eXCgVnOZdFQIZ2y28QFpAKYvPlWiJ+skiiatlQzLIUmb7S5kCskKDtX7Mah/AUafVIYLRh0Hl9NhtpiPZ2hVIY40BNDSHnUndWSdJHebSUwB6Yis6vEeKe373VMRyqEXErUctA8aU0A6pA36MfLRWfoYWWMOmnwOWw0B462ZHIbnl2CFMGQ+pcPBmjYAQFlxLgZU+HCg1g9CSIfWnelWcmlT+NoCEUugNT1h46e3AZqikG0o6ki85WDGHDpf6EhDEP6ggkGVPgDAhDOqsPAHo83EgHhGDS2F0yHhn58dBgA4nBJCHRSkhePqOAy5WE78RtqtwxFdv7cjlEMvJD5bKZo2ai+V1dpj36wpYHQrGWvZqZnoyHKwpfzCstmjyVhTJTTjxU7f6sqhvCgHA8t98AdlBMNKzMnYinVDV1QCWY4W0KXrLoxvlmddN10i8dPbUuxjZMQPBvUviF7rcnb05yj2eTF2eDk+212P+uYg3E6H6RKMx6gXSQze209RDuuWg9Mh9Zl0VqEcbCArKupagt0tRocYAWnTj65vAnbqE4JxyiFHn9iV7ilMa8MRO8zF63HaylYyTvtGbYLZ88mm5RBjzRhrpdj90y6H69rhdjlQmO/BgArt9FzbFEJbIJLUKjPut9PhQEQhyM91w3DmpNv2ItosL/paGAHpdK24hOltKQ4O2nukFfk5LpQV5ST8jlKKlrbE+/DdUcfB7XTg7/8+DFcnyqHDgHQas62tEEpxuL4dL72zExGFIGLDPdsTEcrBBu1BGbWNwWPSYsEOoUisH91jyQxKl2A48bQPAKFQemvFWzPGWppMNt1K8am6PJSf/nXARvwiHQ7X+1FWlANJklDZLxdulwOH6v1QCU2q5BQlGpAmBPDluqO/S9etpHeDtbb/thuQjs9WSqU9diCk4ECtHydUFpgpsLFrEjicicWW+blunHtaJbbvb8KRxgAIIUmfe9QyShZTSf8zq6oUX+1rwrc1fuw90nrM5n10N0I52KA1ICMsq7bcGMeCYFhrZy2ZfXskuF0OW7GCoKX7KRA9pQfCctoyAUhwUdkZHGScGD26LG69J1LYRofXYERJsGa0n2f2ta1pCKBUPzU7HBKqyvO1oDRF0hNxfNfSwnyPeTJO1wUWHycw1rWe9lNVOLJCYk7oXRXBEULxxd4GtAXkGJdSjHwRFcU+T1J32bjT+iPP68L7nx0GICX9m0gSy8jIVrI7GGnfUS1T6pvDrVAJtT2vvCchlEOaEErRqgcDOzJruxtrmwoDj8thq213QpzAdOGk99w7VA42PrDGY+fqwUsj88lO182gZWIeYHl+NtJiUyUQ0mIL5RaXysAKH2qaAgjJClqTxB2Mzdbh0E78+bluSwfU9APSWuFa9Gdup0Prc0UpapsC+PpAc0qWZkQmMb59reCs41Yg9S1Bs0/SoP6+pH9DAZQW5iKJUQGvx4kzh5djz6EWtAflpCmzkSQxB7sxFQCobQ6gsTUMp0PCnsOtoDR9t2pPRCiHNAlHVBCqvUmTfYizgVAkdsMDtI1YltP3uSaLOWg/t+dWsq7l8ThstSMIhLUGftaAqlG8lu7zC4WVGEUaLRjMnOI/0qClrZYX55o/+86wMkiShE+218IflBNSWo3NllKgIM8Nh6QpCCD9bKV4VxCgZyupBIfrAzjaEAAIUN/cdVxNVmKn0wHR4G88wbCCo40BHKoPINfrRIXl+UfXU5Gf40Jejgt5XlfS98fIE0tAKbD7UHPS1ylqOcRnK9k77X+5twkAMG5kJQIhBbVNoT6RziqUQ5r4gzL+8a+DOFTnhz8oZ6V5GdKnpFnRgr/pt9oORlTzNA3Yr7buyHJQVNplwVTiWsmVn50sKs1tlhhzyITlQAhFMKyYmToV/aKbY7HPi+8MK8O/d9WjxR9JiDuYJ3EqmfGGwnwPgNRdQO0hGaGIYraztvr7DbdSRFbhdjvgcGqVy1YLsT0kJzxWRFFj3EpA8rRRQikO1fnx1b5GfLm3EaefWJo03hCOEBQXaBZVcUFO0syg8uJc9C/JxY79zWYFvxVj/KrDEVt/YSemAgDb9zWiIM+Nc06pBADsO9pqe4JhT0IohzT5+78O4bPdDVi1YR9CEdVWv6JM06FbSdY+HO0hGYfr/V365xWVQFZIB5ZDmjGHiBEnSDylp5tlFAzJ+uyF6IffqE9Il45iDnYynzqDUoq9R1qx62Azdh1sgcspodjnjfmb7446DhKAT3fUoD0oIyKrONrYjp0HmnGgzg9A8+kbtQA+vWCsq5oCQPebH2nDrgPN8AflpKd9Qik+3l6D/3t1G5556yu0hxTUNAVBKUV9SxA7DzSjpjEQc52skIR+SMn6NDW2hPDZ7gas3XwQwwYUYdJZAzq8T74ct/783KBS8uc28sRSHG4I4EhD4vs4LKtwOx2wPkPDrUTTLGBRVILdh1px4nGFyM9147jSPOw/6u8Tcx2ySjm8/fbbmDZtGiZNmoSXX365u8VJoKk1hPf+fRCV/XLhD8r46MujZm8eK4RQbkqjsTWE5hTn5hpYM4xUog2zMRrKNfnD2Lq7Hofr27tc1wz8Wt0uNn3y0fTaaJGTXeUQCCsx1oyxVrqV28mUXzTzia/Sr2kMYOvuem2jr/Wjol9ezMkWAIryPTjj5DJ8sbcROw+2YOvuOuzY14yaxna06K9VXo7LlDFPtyDCsopASIE/mHiyN9h5oAn/3lWH1oAMf0A2u7vKimomMADAhm1HcOLxhQjLKl77+24cqG3Dgdo2bN5ei/01rahpDppWICFa2+34lheuOMshGJbx7qcHUP3JtxhY4cOc/zjRLKpUCTEtR0Uh8Lijc0O8Hic8Lqe+qVMEQgqCYQWUUowcUgIA2L6vKdGakYlW/W2NqbgcoBRpu5b2HGpBWFYxdIA2V2JoVREON0Rfj95M8nLEbqCmpgZLly7FypUr4fF4MG/ePJxzzjk46aSTuls0k7+s3wVVpZgzYSg+/qoGW76uwxknl6HM4jttbAvhnY+/hdftxMSxA03T3wqlNKlJbUVWVLy54Rt8sr0WIwb3w2XfHYKCvMS1khGKqPDluqGoBIGgAippG3yLP4JHXv43GlpDKPZ5MOXsE3DRGQMSXFAGyVxBOXZP+8ksB1PRpLcRW2c5mHLpldvpBB2Spdc6HBI8LgdXt9LX3zZh+ZtfxhS4/efYgQC090JEJmYriDOGleNfO+vxwprtCYrO6ZBQmO82lYrxWrz1wT6s+Wg/3E4Hhg0sxvjRx2H4wH7m323eUYPn/7ZdT8E8BAAYVOmDohD9XrowoNyHs06pxKgT+6Gq3IdDdX78ae0uvP6PbyBJQLNfk/34sjzMHj8UY4aVJcxyMHA7HQiENPeZQ5Lw8rs78c2RVpxQ6cO8i07SahRCChRVUyxulwOtARmKTDCgMj9mrZICLw7Vt8PhkFDi84JCG9cpSRJOqPRhx7fNSVxwWkxFQmL9RVcNAeP5/JsGSJI2kQ4ATqoqxMZtR7Dj2xYMrSpOa62eRtYoh02bNuHcc89FcXExAGDKlCmorq7GT3/604w9pqISbNx6CLX1/thfUO13YVnLoza6QH76dR3OPbUSpYU5mPCdKmzf14S3N+2Hog+K//pAMz7+qsacK7Bh2xH8x3eOR0mh5kNtbA1jx7dN+OZwK3K9LpxUVYShVYUghCIQVkAJRX6uG06HA+9vPYyaxgCK8j34+78OYdvuBkw6ayByvbGbokoomtvCaGwNozUQgawQNLWF0b80D4GQgsHHFYBQLUdcVgl8uS6MGno8PtvdgFfe241vDrdicP8CBMIKCKXI87qQl+OGyymhUR/NaN2InU4HXE4JOw8248k3vkB9SxDFPi8GlOfjhOOL0NQcNPPoPS6nmWa660CL9jNnolvp37vqUa/PNjDuP4W2cVJoB0BJ0j/qEtDQEkK/gliXjNfjRDCsYOPnRxNOsoUFOWhtCyGe9mBsSqx1raONAWzYdjjhmgR0WQnVvpAkXVZdhKONAbzz8bfw5Xkwf9IwVPbLQ16OC6UlPhypaUFEISjI9aCiXy5yPC4cbWzHpLMGoLktjMJ8Lwrz3XA6HXBI2qZb7ItmOJUW5eDsUyrQ4o/A63GiLRDBJ9tr8PFXNehX4MWgSh/yc9344POjqOyXi5nnD0Z9Swh7DrVg8HHaaz64fwFyvS6ohGDWf5yEYDCMiKyiyOfF3IuG4rV/fIPyfrkYP/p4OBwS1ny0H8+u/gqnDOqHXQe12El+XD+kvBwXdh1swYMvbQGgndovPucEjB1RDkWlaAvIKPJ5UVaUg1yvCw5JQjiioi0YQWHcAaggz4PSQhXlxbnI9eputFKCJn8IwwYUYf2WQ3jz/T3Isxw6jjQEEl1d+gHo469qUOxL7ZAFAFu+rsPxpfnmYw8o98HrdmLL17UozHd3eF1H7zneOCQJ3xlW3mFPKhayRjnU1taivLzc/L6iogLbtm1L+frS0uRpcZ3xzaEWPPLSpym7IipL8nDBGQPhcLuQ73Zh6nmDsfLvu/HH6q8BaCfQMSeX49yRx6G2KYAPPz+CNzfuM6+XAPQvzcfpQ0sRCCvYebAZ/9pZB0A7FUpStG9/ZUke5k4chlMGl2D7vias/Xg//rp+V1K5JAC+PDfyctxwOiQcX56P04eV48zTjjODezd9Px9f728yT+vnjq7Cux9/i83ba7B5Ry0kSXujJRuYU9IvDw6LO6ikMAdH6gMIRQgK8z34ttaPrXvqu7yPJYU5cHo8cOgn9X66xbVuy8HOL0zC8EElcLhcpuugpCgPEaUOL73zddprlcU9v34FOfi2pg0vrNmR9lrJOH1oKaadP8TcYACguS2EivICDKgoMLOOAKDq+GL0ryhETWMgwe2kqgQDq4pRZIlV3HbFmdh9sAUOB+BwONDYEsS23fXYd6QVe4+2odUfwXdOLse084bA7XZgwHFFGDOiEpRQDD6+yLR6+5X48PX+RkguF0oKclHs86C5LYwhA0tMpQwAA48rxOvv7cbOAy0YdFwBhg0sxuknlcFh2Ywv+4+TcKjOj/aQjHBExYhBJSgu8AIU8OZKOG1YOYp8iZXRHXHCgH4JPzseRSgr8WHjtqN4Zd3OhN+ffEIxHB43jFtYXKQ9zzf++U3Kj2sw6ewTzPeHA8DwQf2wbXc9t/cHK1ddDFw+8WTu60o0043rU2T58uUIBoO47bbbAACvvvoqPv/8czzwwAMpXd/Q4LeVOaRIEnbta0z4uVsP/HncWiArFFGR43GiMD/2xNrUFjYHlfhyXSj2eWNcRg0tIdN3n+d1ol9Bjvmhp5SipT0CpwTkel1wuZwIhRW0BSI4YWA/hNvD5lqKSnC0IZAwFtIhAcX5HuTlaopBO3Frysa6GQGamyh+8/cHIlCIZjE49arUtnY52vPf5UBZUU6M60CWVQQjCgrzvWZgMxRRQZwOtLWF4HU6AAlmoaDxkL4cF4oLYu9Piz+M1kBicFuS9ICYca8IhVX00kIv8nKim6qWnx9M6jYoLs5FcwdpmS6nhLKi3BjXmqwQ1DQFUj40SBLglDQfN42T1emUUB63PgCUlxcg0BZM6l6kVLckkzx+nteVoDTir40oxMxuCsuqObfbikNCzP0DgMLiPNTXtcVkgkVkNek9JZYRn+lgjJvlRWt7BAGFoCnu9S3Kdyd8Vq2fxVRxSEBpYQ5yLJ8lVR/x2tnkwc7eczxxSMBxpflJ3dddXuuQOj1UZ43lUFlZiU8//dT8vra2FhUVFRl/3OPKfHAx6EdrGwM7v4+PI/hy3SgrzkV5ST7q4gJtqcYcOiJeWXQkX0VxVwu5EX+Wy/W6UF5egLq6trRk8uW6UZXWFR3T0f2xI1e824o3vlw3gv7kbgdJkpCf0/n7piOMgkDDZdfV+8+K1+1MSBH2JPlZNlGY78HQFF/fdO5FVxR1sRnbec9lG1mTrXTeeefhww8/RGNjI4LBINauXYvx48d3t1gCgUDQJ8kqy+G2227D1VdfDVmWMWfOHIwaNaq7xRIIBII+SdYoBwCYOXMmZs6c2d1iCAQCQZ8na9xKAoFAIMgehHIQCAQCQQJCOQgEAoEggayKObBgJ+eax7WZJFvl6oxslTkb5RIy8SMb5c5Gmax0JV/WFMEJBAKBIHsQbiWBQCAQJCCUg0AgEAgSEMpBIBAIBAkI5SAQCASCBIRyEAgEAkECQjkIBAKBIAGhHAQCgUCQgFAOAoFAIEhAKAeBQCAQJJC17TP8fj/mzZuH5cuXY8CAAVi5ciV+//vfw+l04pxzzsFdd92FlpYWXHfddeY1bW1taGpqwr///W+0trbiZz/7GQ4cOICSkhL83//9X8yMaoPDhw9j4cKFaGhowJAhQ/Doo48iPz/f/P1rr72GTz/9FA899FBSuZ5//nksW7YMqqqisrISK1euhKIoplzNzc1obW0FgIzK1RHLli2Doij4+9//juXLl+PIkSNYsGABVFWFJEkYMGAA3nrrrYzeyz179uDee+9Fe3s7cnJycP/992PgwIEJr+/y5ctRW1sLt9uNM844A/fccw9++tOfmuvX1NSgtbUVX331VUZkOuWUUzp83zU1NWHAgAH4y1/+gpaWFsybNw8HDx6E2+0GIQSEEC5y7d69G/fccw8CgQCKiorw0EMPoaioqFvvVTKZqqo6nt+3bNkyOBwOXHvttZg3bx5uuukm/OIXv0BOTg4aGxuRk5ODSy+99Jh9ho8ePYpLLrkEK1euRHFxccK9fOaZZ1BbWwun04lhw4bh/vvvx8KFC83r6+vr0djYiO3bt2dEpgEDBnR4L+M/54cPH8b06dNxwgknAADKysrw3HPPdXg9EzQL+eyzz+iMGTPoaaedRg8cOED37NlDL7jgAlpTU0MppXTx4sX0+eefj7lGVVV65ZVX0rfeeotSSukvfvEL+vTTT1NKKX3jjTfof//3fyd9rOuvv56uXr2aUkrpE088QR955BFKKaWhUIj+5je/oWPGjKF33nlnh3KNHDmS/vnPf6aUUjp79mx61VVXxVw/evRoet5552VUrmS0trbSn//853TkyJF03LhxpsyPPPIIPeOMM47pvZw3bx597733KKWUbtq0iU6cODHp63vNNdfQ1atX08WLF9Prrrsu5jk/8sgjdMSIEfSKK67IiEwzZ85M+vp+97vfpbfeeisdNWoUnTVrlnmvnnvuObp8+XLu9+rKK6+k77//PqWU0j//+c/02muv7fZ7FS/T7bffnvR64z03atQoevfdd5ty//a3v6W//vWvj/ln2Fjzuuuuo2PGjKFr165Nei/vuusu+vTTT9PFixfT2267zXwcVVXp73//e3rqqafSqVOnZkSmAwcOJL2+o895dXU1vffee5New5usdCutWLECixcvNmdIf/311xgzZoz5/YQJE7Bu3bqYa15//XXk5uaaw4L+8Y9/mF/PmDED//znPyHLsYPsZVnG5s2bMWXKFADArFmzUF1dDQDYvHkzCCExJ4h4ub766iuoqorvf//7AID58+fj3//+d8z1EydOhNPpzKhcyVi/fj0GDx6ME088ERdeeKEp85YtW+DxeHD99dfjhhtuwOjRozN+L7///e+bI1+HDx+OI0eOJLy+o0ePxrZt2zBlyhRMmDABra2tMc95x44dGDp0KAYOHJgxmZK97yorK3HKKafg2muvxeDBg8179fnnn+ODDz7ARRddhN27d2Ps2LFc5HrhhRcwfvx4EEJw+PBhHD16tNvvVbxMhYWFSIbxnrv22mvx1VdfmXLv2LEDGzduRCAQwH333YcjR44ck88wAPz+97/Heeedh379+uFvf/tb0n3l448/xsyZMzFhwgQcPXrUfJw9e/Zg/fr1OPnkk1FWVpYRmTqio8/5559/jp07d2LWrFm4+uqr8fXXX3e4BitZqRwefPBB88MGACNGjMDWrVtx5MgRqKqK6upq1NfXm79XVRVPPfUU7rjjDvNntbW1prnncrng8/nQ2NgY8zhNTU3w+XxwuTTvWnl5OWpqagAA3/3ud/E///M/yMnJ6VCu/v37g1KKuro6qKqKjz/+GJFIxLz+jjvuwMaNG3HaaadlVK5kXHbZZbj++usxceJEHH/88ebPjzvuOBBC8NRTT+GCCy7AI488kvF7OWvWLDid2pD6xx57DDNnzkx4fT/77DPk5uZCkiRUV1ejpaXFvH7cuHHYu3cvpk2bljGZJk6cmPR9V1dXh5kzZ0KSJOzZs8e8VwUFBbjyyivhcDgwd+5c3HbbbVzkcrlcaG1txfjx4/GXv/wF//u//9vt9ypepssvvxzJMN5zTqcTF110kSm3z+fDFVdcgfz8fIwZMwa33nrrMfkMf/HFF/j4449x7bXXAgB+9rOfJd1XampqUFJSgurqajQ0NJiPc+KJJ+Lo0aOYP39+xmTqiI4+516vF5dddhlWrlyJH/3oR/jJT35i7jm8yUrlEM+QIUNwxx134MYbb8T8+fMxfPhwuN1u8/cbNmzAkCFDMHz48E7XcThiny5N0pBWklJvsztw4ED4fD5TrpNPPjnm+g0bNqCsrAxFRUXHVK7OWLp0KRYtWoQbb7wRb7/9Ntrb22MeP1P3klKKhx9+GFu3bsXdd98d83dDhgzB9ddfj+bm5pjX17jekKl///7HTCZDLuN9t3LlSpSWlprvuwceeAAejwdDhgzBrbfeit27d6OtrY2LXIWFhdi4cSN++9vf4sYbb4SqqjEydce96kymrrjtttswf/583HHHHaiursa2bdswePDgjH6Gg8EgHnjgAfzyl79MuMbAeH0VRcHVV18ds684HA5TJsMCOxYydcXNN9+MefPmAQAuvPBC5OXl4ZtvvrG1VldkbUDaSjgcxqhRo7Bq1SoAwNq1a2NesHXr1sWclACgoqIC9fX16N+/PxRFgd/vR3FxMS699FLzb1577TX4/X6oqgqn04m6ujrT5OyISy+9FDU1NViwYAFeffVVyLKM119/HU6nE6+88gq8Xm+MXKNGjQIh5JjIZfDmm28m/RtKKR5//HFMmzbNvJejR482g1uGzLzvpaIouPPOO1FTU4MXX3wRBQUFAGDeR5fLhaVLl8Lr9eLll1/G+vXrUVlZiVAodMxlsr6+q1atMt93jz/+OPbs2QOPxwNCCJ5++mkcPHgwRi6Xy8Us15o1a3DxxRdDkiSMHz8eoVDItAy66151JJP19NvZe+5Pf/oTbrnlFvNejh07FlVVVRn9DH/66aeor6/HjTfeCEA78V9//fV44oknUF9fb97LxYsXo6qqCsuWLcO2bdswYMAAfPLJJyguLj6mMi1ZsgS1tbUAgGeeeQaVlZVJ7+dLL72EGTNmmC4pSqlpofCmRyiHQCCAa665Bn/729/g8Xjw0ksvxZi2n332GRYsWBBzzYUXXohVq1bhhhtuwJo1azB27Fi43e6EN/HYsWOxZs0azJw5E6tWrTL90B3x5ptv4qKLLsKzzz4LWZZBCMHKlStx6aWX4umnn8YZZ5wRI9cll1wSo9kzKVdXSJKEd999F3/605+wfv16VFdXw+PxYMaMGTEy876XDz/8MPx+P55//nl4PB7z7ysrK/Hss88iPz8fl112GcaMGYO33noLK1euRElJiXm9IdOWLVsyLpP19bW+71RVxdatW3H77bfD4XDg3XffRUNDA66//nqsWrUKo0ePRm5uLrNczz//PFwuFyZPnoyPPvoI/fr1Q0lJSbfeq45kSvU9t3HjRgwdOhRPPPEEbrrpJpx++ul45ZVXMvoZvuCCC/Dee++Zf3PRRRfhmWeewYABA1BWVhZzLy+44AK8/vrr+Oijj3DSSSeZj2PIdPTo0YzL9Oyzz3Z5LwEtFhEKhbBgwQJ88sknIITgxBNPTOnatDkmYW+bTJgwwYzmr1ixgk6bNo1OnjyZPvbYYzF/N2rUKBoKhWJ+1tTURH/84x/TadOm0blz53aYFXDw4EF65ZVX0osvvphed911tLm5Oeb3r7/+ekJWkFWuZ555ho4ePZqedtpp9D//8z9jrh81ahT961//GnN9JuVKxmOPPUYfe+wxU+adO3fSiRMn0pEjR9LTTz+dPvjggzF/z/teNjQ00FNOOYVOmjSJXnLJJeZ/8fdxxYoVdNKkSfT000+n55xzTsxzNmSyPudMyZRMrmnTptGzzjqLzp8/3/ybnTt30uHDh9OpU6fSK6+8kh4+fJhZLkop3bVrF503bx695JJL6Pz58+nOnTu79V51JlNHGO85Q+4NGzbQuXPn0u9+97t09OjR9KKLLjqmn+H4+xd/L6dMmUJHjx5NzznnnJjHMWT66KOP6JVXXplRmToi/nN+9OhR+sMf/pBOnz6dzpo1i27fvr3T61kQk+AEAoFAkECPCEgLBAKB4NgilINAIBAIEhDKQSAQCAQJCOUgEAgEggSEchAIBAJBAkI5CPok1113HRobG7FgwQLs3r07o4914MAB3HzzzRl9DIGANz2iCE4g4M0HH3wAACkXH7Fw+PBh7N27N+OPIxDwRNQ5CPocP//5z7Fy5UqcfPLJ2L17N1asWIFAIIDf/va3qKiowK5du5Cbm4ubb74ZL730Evbu3YvJkyeb/Zfee+89PPXUU5BlGTk5Objzzjvxne98B3v27MGiRYsQiURAKcWcOXMwb948TJ06FTU1NTjrrLPw3HPPYfny5Vi3bh3C4TCCwSDuvPNOTJo0CY8//ji+/fZbHDhwALW1tRg1ahTOP/98rFq1CgcPHsTChQsxY8YMPP7449i1axfq6+vR0NCAESNG4MEHH4TP5+vmOyvoVWSsvE4gyGJOPvlk2tDQQCdMmEC3bdtGP/roI3rKKafQL7/8klJK6Y9+9CM6d+5cGg6HaUNDAz3ttNPo0aNH6d69e+mMGTNoY2MjpVSrlD7//PNpe3s7/fnPf272+q+traW33norVVWVfvTRR3T69OmUUq1y9qqrrqLBYJBSSunq1avpjBkzKKXUrGRvbW2lwWCQnnXWWXTJkiWUUkrfffddOnnyZPPvxo8fT+vq6qiqqvT222+nDz300LG7eYI+gXArCQQ6AwYMwKmnngoAOOGEE1BQUACPx4OSkhLk5+ejpaUFmzdvRm1tLX74wx+a10mShG+//RaTJk3CnXfeiW3btmHcuHG45557ErpvVlVV4eGHH8bbb7+N/fv3Y+vWrWhvbzd/f95555lNACsqKnDBBReY8jQ3N5t/N3XqVHPGwJw5c/DrX/8ad955ZyZui6CPIgLSAoGOtQEfgKTdLgkhGDduHN58803zvxUrVmDYsGGYMGEC3nnnHVx88cXYvn07Zs6ciW+//Tbm+i+//BLz5s2D3+/H+eefj//6r/9KWwYA5iwKQya7LaAFgo4Q7yhBn8TpdEJRlLSvO/fcc/HBBx9gz549AID3338fl1xyCcLhMO644w6sWbMG06dPx+LFi+Hz+XDkyBE4nU5zWtjmzZsxcuRIXHvttTj77LOxfv36tGYjGKxfvx5tbW0ghGDFihWYMGFC2msIBJ0h3EqCPsmkSZNwxRVXxLh0UmHYsGF44IEHcPvtt5u99J966ink5eXhpptuwqJFi/DKK6/A6XRi4sSJOPvss9Ha2gqn04k5c+Zg+fLlWLt2LaZNmwa3241x48ahpaUFfr8/LTnKysqwYMECNDU14ayzzsINN9yQ1vUCQVeIbCWBoIfx+OOPo6mpCffdd193iyLoxQi3kkAgEAgSEJaDQCAQCBIQloNAIBAIEhDKQSAQCAQJCOUgEAgEggSEchAIBAJBAkI5CAQCgSABoRwEAoFAkMD/B6bTvcsWl5t3AAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"overcommissioned_burst\", data=x)" - ] - }, - { - "cell_type": "code", - "execution_count": 105, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 105, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5HElEQVR4nO3deXhTVfoH8O/N2iZpuiYFWpay71BlR1mUshZQQC2IjKCgjOAMOoCyDNAZBBHRwQ1lmfkJ6NABqeIgiDCCgkipUFrWUkpbaOmSLmnSJs1yfn+URkIX0jZ73s/z+Njk3tv7nkPbN+ece87hGGMMhBBCSD14rg6AEEKIe6NEQQghpEGUKAghhDSIEgUhhJAGUaIghBDSIEoUhBBCGiRw9A00Gg3i4uKwZcsWZGRkYNOmTZZj+fn56NOnDz799FOraxITE7Fx40aEhoYCAEaMGIFFixY5OlRCCCF14Bw5jyIlJQUrVqxAZmYmDh06hMjISMuxwsJCTJ8+Hdu2bUO7du2srvvb3/6G6OhoxMbGOio0QgghNnJoiyIhIQGrVq3CkiVLah3bsGED4uLiaiUJAEhNTUVWVhY+++wzdO7cGStXrkRgYKDN9y0p0cJsbnz+Cw2VQaXSNPo6b+Nr9eBr5a2Pr9eDL5efx+MQHCyt97hDE8XatWvrfP/mzZs4c+ZMvccVCgXmzZuH3r17Y9OmTYiPj8e7775r833NZtakRFFzLfG9evC18tbH1+vB18tfH4d2PdV47LHH8Pnnn1u6nt5++20EBQXhpZdeeuC1ZWVlGDVqFJKSkhwdJiGEkDo4fDC7LkePHsX27dvrPFZeXo59+/bh+eefBwAwxiAQNC5MlUrTpE8GCkUACgvLG32dt/G1evC18tbH1+vBl8vP43EIDZXVf9yJsQAAiouLodPp0Lp16zqPSyQSbNu2DSkpKQCAXbt2ISYmxpkhEkIIuYfTWxS3bt1CixYtar2/fPlyPPbYY3j88cfx/vvvY/Xq1dDpdGjXrh02bNjg7DAJIYTc5ZQxCmejrqfm8bV68LXy1sfX68GXy+92XU/Efsys6U93EUKIrVwymE2a51x6If73223cyFVDLhVh2XMPQ+YvdHVYhBAvRS0KD/Tdr9nIzFMjunMYisp0+OirVBhNZleHRTyEptKAT7+5iA/2XcCXP6TjRq7a1SERN0eJwgOZTAxRreR4YUJ3zJnQFVdzSvH54auuDot4ALW2Chu++A3JVwtRUFKJ4+dv462dyTh4Oou6MUm9qOvJA5nNDAJedY4f1L0Fcou0+PZUFh7urECfjmEujo64K1WZDpsSzkNVpsOfn+qN7u1CUKEz4l+HrmDvjxm4kVeOWaM7Qy4VuTpU4maoReGBTGYGHo+zvJ40NAotQyXYfeQaqgwmF0ZG3EmZtgo376hRoTPgRq4af/v8LEo1VVj0dB90bxcCAJD4CTB/cg/MGtMFqRlFWPXPM7iaXeLiyIm7oRaFBzKZzVaJQsDnYeboLnjny3M4eDoLTzza3oXREVdLvlqIL364hpJyveU9DkBooB+WTI9GqzDrxd84jsOI6Aj069kSa3f8ii1fX8S7C4aCx3EgBKBE4ZHMZgY+z/qXuFvbYAzsHo6Dp7MQoZChf1eli6Ijrnbst1sAgGce64hQuR8KyypRoTMipn9ryCX1dytFtQrEpEeisPXAJdzMK0f7VnJnhUzcHCUKD2Qyszo/7c0Y1QlFZZX4JDEN2YPb4olHo8DnUe+iL9FXmZB+qxSPPxyJMQPaNPr6Xu1DwXHAhYwiShTEgv6KeCAzq92iAIAAiQhLpj+EYX1a4r+/ZOHv/5eMrDu+OdPUV13JLoHRxNCzfWiTrpf5C9EhIhAp11V2jox4MmpReKD7B7PvJRTw8IexXdEjKhRfHLmG+P9LQq/2oRjUIxzhwRKrc5XB/pD60UQ9b5KWWQyRgIfOkbZv9HW/Ph1Cse/4DZSU6xEcILZjdMRTUaLwQHWNUdyL4zj076pEj3bBOHg6G79cvIMLGbU/Icr8hXhpUg/0iApxZLjEidJuqNC1bTCEAn6Tv0efDmHYd/wGUm+oMKxPKztGRzwVJQoP9KBEUUPiJ8S0ER0wZXh73MhVQ1tpsBwzmhgSf76BTXvO45nHO2F0/7qXfSeeo6C0EvkllXj84cgHn9yACIUUIXIxUq4XUaIgAChReKSGup7qwuM4dIyo3RXRMyoEm/ddwDc/Z1Ki8AIXb1S3Gps6PlGD4zj06RCGk6l5KFbrECL3s0d4xIPRYLYHMtnYongQsYiPnlEhqNAbUak32iEy4krnr6sQFuiH8GD/Zn+vMQPbABzw+eGr8MKdCEgjUaLwQOZGtigaUvNpsVits8v3I66hrqjCxcxi9O+qBGeHiXLKIH9MGdYBFzJUOH0p3w4REk9GicLDMMbs1qIAqmfrAoCKEoVHO3ulAGbGMKhH7d0jm2rUw5HoECHH7u+v4WjyLehpeRifRWMUHqamF8BeLYpQeU2i0D/gTOLOTl/MR4RCitbK+ncpaywej8Pc2O7YeuASdh+5hsSfbqBLm2B0aCVHn45htZYCId6LEoWHMd1dCtpeLYpAqQh8HkddTy5SqTeipFyP0EA/iIVNe6S1sLQS12+XYepw+6/xpQyWYNlzD+NaTilOpOQi47Yav10rxH9+zEBrpQwDuikxsFs4woKaPy5C3BclCg9Ts2eAvVoUPB6H4ACxz3c9Mcag1laB4zjweBw4rvppMR5392te9fv2XCivWK1D/P+dhVpbBQDwF/PhLxYgUCpCqzApIsJkiFRK4S8S4PTFfJy7XogAiQitQqUY1qclurQJBgD8encMYWD3cLvFdi+O49ClTbDlfiXlepy9WoAzl/Kx7/gN7Dt+Ax0i5BjYLRyDe7agSZxeiBKFh/m9RWG/4aVQuR+Ky3w7Uez9MQPf/Zr9wPNahkrQvpUcyiB/BEhFEN+d2CYQ8OAv5iNQKkZ4sD9ED2gdGIwmfLQ/DXqDCX8Y2wWaSgPKtFWo1BtRrNYj7UYxTqbesZwv4PPQu0Mo9AYTUm+o8MvFO+jfVQmTmeF8ehG6tA5CWKBzPtUHB4gR0681Yvq1RmFpJc5czsevlwrwxQ/p+O1aIZbMeMgpcRDncXii0Gg0iIuLw5YtWxAZGYk333wTycnJ8Pev/qFesGABYmJirK65fPkyVqxYAY1Gg379+mHNmjUQCCinAdXrPAH263oCqp98upbj23sQ5KkqECIXY9zAtmCMwcyqW2/VX1e/NhjNuFWgQeqNYksroC4cgNbhMswc3aXO+StmxvD54avIzFPjlSd74eEuijq/j7qiCrmFWpRq9egZFWrZF11vMOG701n47tdsiIV8jB7Q2mXzYBRB/pgwuB0mDG6H/Sdu4NtTN1Gq0SNIRkt/eBOH/vVNSUnBihUrcPPmTct7aWlp2LVrF5TK+pfBXrx4Mf7+97+jb9++WLZsGRISEjBjxgxHhuoxTHf3xrZX1xMAhAaKUXKpCiaz2WdXm9XqDFAG+ds8q9lgNKO8ogqGu/8eBqPZMt6QW6TFydQ7WLcrGaP7t8aAbuFoEy4Dn8eDwWjG9v9ewpnLBZg0tF29SQIA5BIR5G1rLwsuFvLxxKPtMXZgGwj4PAj47vFv1r+bEgdO3cT59CKMiI5wdTjEjhyaKBISErBq1SosWbIEAFBRUYHc3FysXLkSubm5iImJwYIFC8C754/T7du3odPp0LdvXwDAlClTsHnzZkoUd9l7MBuoblGYGUOZpspnZ+FqKg2IaMRTPEIBr8G6GjOgDf59NB2Hz+Tg8JkciAQ8KIP9YWZAbpEW00Z0wLiBjV8G/F5+IvdqZUeESaEM8sdv6YWUKLyMQ3/S1q5da/VapVJh0KBBiI+Ph0QiwUsvvYS9e/fi6aeftpxTUFAAheL3T1kKhQL5+TThp4ZlMNuOg6ph8t/nUvhqotDqjJD6228Q1l8swOzx3TD5kShcv12GG7lqFJRUokyrx9yJ3THYjvMd3AXHcXioswJHzuagUm+Ev9i9EhlpOqf+S7Zu3RofffSR5fVzzz2HxMREq0RR13IBjZ1pGhra9GfJFYqAJl/rDIa7dREU5G+3WDua735vxlm+pzvVwx2VFpV6I/h3nzzi83gIlIkgsdPTNYwxVOgMUIRI7V5uhSIAXTrU373kbppb/pED2uDQmWxkFVbgUQ9sVbjTz707cWqiuHr1Km7evIkxY8YAqP4FvX+QOjw8HEVFRZbXhYWFDY5n1EWl0lg+eTeGQhGAwkL33uinSKUFAGg1evvFaqxe5ynzVgm6tw50m3pgjOHAyZtI/DmzzuMBEiHGDmiDcYPaNus+Mrk/jCYGjpndotyuYo9/91CJEHKJED8mZ6NrpGftkOcuP/euwONxDX7AdmqiYIzhrbfewqBBgyCRSLBnzx48+eSTVudERERALBYjOTkZDz/8MBITEzFs2DBnhunWHDFG4ScSQOonQLEbzc42mc3YefgqTqTkYXCPcDzUWQGTmcFsrl7CpExbhR/P3UbytcJmJ4ryu08w0fP/zcfjcegRFYLLWb79FJ23cWqi6Nq1K+bNm4fp06fDaDRi9OjRiI2NBQDMnTsXr776Knr16oWNGzdixYoV0Gq16N69O2bNmuXMMN2a2QGJAqhe88mdJt39fCEPJ1LyEDukLZ58tH2d3Y+3CzW4llPa7HuVV1QnCpkdxyh8WYjcD2qtAWZW997uxPM4JVEcO3bM8vWzzz6LZ599ttY5W7dutXzdtWtX7N271xmheRyTnWdm1wiV+6GwtNKu37OpzIzh+6QctG0RUG+SAIBAmRhl2iowxpq1YqqmonpDJ6kfDb7ag1wqgpkxaCoNkEtqP95LPA/9ZngYR3Q9AdWfAi/dLMGJlFy0iwxCebltrQuJWIBWYdImr1NUl7QbKuSpKjBvYvcGE0CQVASjiUGrMzarNVBeebfriVoUdhEorU4Oam0VJQovQYnCw9h7racaXVoH4cdzt/Gv7640+loOQHiIBB0jAtGuZQCCA8QIkokRKBVBLhU1ekLY4TM5CA4Qo1/Xhh9iCAqonv1bqtE3L1HQGIVd1SQHtbYK8JwHvkgDKFF4GEe1KPp1VSK6cxhK1HpAIEBxidam69TaKtwu0iLrTjnOXy/Cz6l5Vsc5VD+dJJeKIRRUxywU8CERC+Av5sNPLIBIwAOH6mMGoxmXs0rw1IgOD0wwNZ9cyzRViGzGH6Tyu11PMn/6dbAH+T0tCuId6DfDwziqRQFULzQYdnd+RqHM9k/X/e7+nzGGknI9yrRVKNXoUaa5+39tFco0VTCZGRgYDAYzitU6yxasNctg1FAG+WNY31YPvG/NekKlmuY9rVVeUQWRkAehwH7dZ76MEoX3oUThYRw1mG0PHMchRO7ntNndgbK7LYpm/kHSVBio28mOpH4C8HkcyiooUXgL91hNjNjMUY/HeiI/kQBiEd8uLQpKFPbDcRzkUhG1KLwIJQoP44j9KDxZkFSEMk3z/iCVV1TR+ISdyaWiZrf0iPugvzYepmY/CnfsenKFQJkYZc1uURjo0Vg7C6QWhVehROFhavajoK6nakEyEUqbPUZBXU/2JpdQovAmlCg8jDsPZrtCoFTcrK4nxlj1GAV1PdmVXCpCeYXB0gImno0ShYexDGbTGjoAqlsUeoMJlXpjk67XG0wwmhit82RncqkIJjNDha5p/y7EvVCi8DAmGqOwUvOIbFOffNJWVv8ho64n+5JLq+uTBrS9AyUKD0OPx1oLvDvprqndT1pdzYKAlCjsKVBCk+68CSUKD0NjFNaC7s4CLtU2rUWhqaTlOxxBfjeBU6LwDpQoPAy1KKw1v0Vxt+uJxijsyrIOFyUKr0CJwsM4alFATyX1E0DA5zU9UVRS15MjSO4u40EtCu9AicLDUNeTNY7j7s6laOJgto42LXIEHschQCKkROEl6LfDwzhy9VhPFSgTQVWmQ5m2CnweBz6PA49Xs3B5NY7jIBTU/lykqTRAJORDZMeNl0g1uVQENS0M6BUoUXgYk5mB40B7Ed8jJMAPSVcKsOiDnxs8L7pTGCYNjULbFgGW97SVRsgl1O3kCLTek/egROFhzGZG4xP3eWpEB3RrFwyzmcFkZjCZWK0ZweUVVfgpJQ9r0pPQtU0QRkRHIFIhQ1FZJWS0XadDBEpEuF1o2wZYxL05PFFoNBrExcVhy5YtiIyMxJ49e7Bz505wHIeePXtizZo1EImsf1ETExOxceNGhIaGAgBGjBiBRYsWOTpUj2A2M+p2uk9YkD9G9I144HkTh0Thf+du4fj5XGz5+qLl/YcesOUqaRq5rHq9p1/S7mBAdyWteOzBHJooUlJSsGLFCty8eRMAkJmZie3bt+Orr76CVCrFG2+8gS+++ALPP/+81XWpqal44403EBsb68jwPJKJWhRNJvETYMLgdhg3qC2uZJWgvMIAsYiPfj1awqg3uDo8r/NIr5ZIua7C1m8vIfHnG5g9rhu6tg12dVikCRya4hMSErBq1SooldWf2EQiEVavXg2ZTAaO49C5c2fk5ubWui41NRWJiYmYNGkS/vKXv6CsrMyRYXqU6q4n+mTWHDyOQ/d2IRjYPRx9O4Yh2Ek78vmalqFSxL8wAAun9gKP4/DOl+fwnx+vw3jf1rfE/Tn0L87atWvRr18/y+uIiAgMGTIEAFBcXIzdu3fj8ccfr3WdQqHAwoUL8fXXX6Nly5aIj493ZJgexWQ2U9cT8Rg8jkN0JwVWze6PR/u0wnens7H282TkqWjswpNwjDl+HeDHHnsMn3/+OSIjIwEA+fn5ePHFFzF27Fi88sorDV5bVlaGUaNGISkpydFheoTNe87ht6sF+Ndfx7g6FEIa7ZfUXHyQcB5VRjPGDGyLmIFt0a6l3NVhkQdw+lNPGRkZmDt3LmbOnIk5c+bUOl5eXo59+/ZZxi0YYxAIGhemSqWxzDdoDIUiAIWF5Y2+zpkqKqoABofG6Qn1YE++Vt76OKMeOrYIwOrZA7DnWDr+ezIT3/x0A0+P7IixA9s49L628OWfAx6PQ2iorP7jTowFGo0GL7zwAv70pz/VmSQAQCKRYNu2bUhJSQEA7Nq1CzExMc4M062ZGA1mE88WHCDGy5N7YtOCoegZFYIDpzItizMS9+TURLF3714UFRVhx44dmDx5MiZPnox//OMfAIDly5fj6NGj4PP5eP/997F69WqMGzcOFy9exOLFi50Zplujx2OJtwiQiPD0yI7Q6U04fCbb1eGQBjhljMLZvLnr6aP9qbijqsDfXhzosHt4Qj3Yk6+Vtz6uqoctX6ch5boKG+YPRoALJz/68s+BW3U9keajFgXxNpOGRqHKaMLhMzmuDoXUgxKFh6EJd8TbtAqTonf7UPx66Q68sIPDK1Ci8DCUKIg3eriLEiq1Hln5vtn14+4oUXgY6noi3qhvpzDwOA7JVwtdHQqpAyUKD0MtCuKNZP5CdGkThN+uUaJwR5QoPAy1KIi3eqizAnmqCuQW0fIe7oYShYcxUaIgXuqhzgoAQDK1KtwOJQoPYzYz8Gl3O+KFggPE6BAhx/9+uwVVmc7V4ZB7UKLwMNSiIN7sudFdoDeY8e6e87TfthuhROFhzIyBz6d/NuKd2oQH4E/TeqNYrcP7CSnQG0yuDomAEoXHMZnM9NQT8WqdWwfh5ck9kXWnHP88eJkm4bkBShQexmRm4NEYBfFyfTuFYcrw9jhzuQAHT2e5OhyfR4nCw5hpmXHiI8YPaosB3ZT46sQNlJTrXR2OT6NE4WFoMJv4Co7jMHFoFBgDTcRzMUoUHsZMM7OJD4kIk6JlqATJVwtcHYpPo0ThYWhmNvE1/boocTWnFGVaelzWVShReBha64n4mn5dlWAMOEfdTy5DicLDUIuC+JpIhRThwf7U/eRCNiWKZcuW1Xpv4cKFdg+GPBi1KIiv4TgO/boqcTmrFJpKg6vD8UmChg6uWrUK+fn5SE5ORnFxseV9o9GIGzduODw4Yo0xRomC+KR+XZT47y9ZOHetEI/2aeXqcHxOg4li2rRpSE9Px9WrVzFmzBjL+3w+H9HR0Q4PjlirmaBKXU/E17QJlyEs0A9nr1KicIUGE0WvXr3Qq1cvDBkyBC1atGj0N9doNIiLi8OWLVsQGRmJU6dOYd26ddDr9Rg3bhwWLVpU65rc3FwsXrwYKpUKUVFR2LhxI6RSaaPv7Y1M5upMQS0K4mtqup+OJOVAqzNA6id0dUg+xaYxiuzsbDz33HOYNGkSJk6caPmvISkpKZg+fTpu3rwJANDpdFi2bBk+/vhjHDx4EGlpaTh+/Hit69asWYMZM2bg0KFD6NmzJz7++OPGl8pLme8mCmpREF/Ur4sSJjPD+fQiV4ficxpsUdSIj4/H1KlT0b17d3A2rjOUkJCAVatWYcmSJQCACxcuoG3btmjdujUAYOLEiTh06BCGDx9uucZgMCApKQkfffQRAGDKlCmYOXMmFi9e3KhCeStLi4LWeiI+KKplAELkYiRfLcTQXi1dHY5PsSlRCIVCzJ49u1HfeO3atVavCwoKoFAoLK+VSiXy8/OtzikpKYFMJoNAUB2WQqGodY4vMzNqURDfxXEc+nVR4thvt1CpN8JfbNOfL2IHNtV0p06dcPXqVXTp0qXJN6prqeD7Wye2nGOL0FBZo6+poVAENPlaRxOoq3f9Cgz0d3ic7lwPjuBr5a2Pu9dDzOB2+D4pByfS7mDm2G52//7uXn5XsSlR5OTkYOrUqWjVqhXEYrHl/QMHDth8o/DwcBQV/d63WFBQAKVSaXVOSEgINBoNTCYT+Hw+CgsLa51jC5VKY+nPbwyFIgCFheWNvs5Ziu8misqKKofG6e71YG++Vt76eEI9hEqEGNqzBRJ+uIYopQydWwfZ7Xt7QvkdhcfjGvyAbVOiqOvppMbq06cPMjMzkZWVhcjISHz77beYOnWq1TlCoRD9+vXDwYMHMXHiRCQmJmLYsGHNvre3sAxm0xgF8WEzYjrj2q1SbD1wCWvmDIDEj7qgHM2mp546d+5c53+NIRaLsX79eixcuBDjx49H+/btMXbsWADA8uXLcfToUQDVk/wSEhIwfvx4nD17Fn/+858bVyIvZmL0eCwh/mIBXoztDpVah9OX7rg6HJ9gUyoeNGgQOI4DY8wyZqBQKHDixIkHXnvs2DHL14MHD8Y333xT65x7B74jIiKwc+dOW8LyOfR4LCHVOkQEgs/jaEMjJ7EpUVy5csXytcFgwPfff2/1HnEOmnBHSDUex0EuFaFMQ0uPO0OjV48VCoWYMGECTp486Yh4SAOoRUHI7+RSEdQVlCicwaYWRWlpqeVrxhjS0tKgVqsdFROph4kSBSEWgdSicJpGj1EAQGhoKJYvX+7QwEht1PVEyO8CpSJk5/vm46zO1ugxCuI6ZkoUhFgEykRQaw0wM0aPjDuYTYnCbDZj+/btOHHiBIxGI4YOHYqXX37ZstQGcQ5qURDyO7lEBDNj0FQaIJeIXB2OV7NpMPvdd9/F6dOn8Yc//AGzZ8/GuXPnsGHDBkfHRu5Dg9mE/C5QVr1KhJrGKRzOpibBTz/9hH379kEorF4DfsSIEZg0aVKdW6QSx6HBbEJ+FyitbkWUaasQ6eJYvJ1NLQrGmCVJAIBIJLJ6TZyDxigI+d3viYIm3TmaTYmia9eueOutt5CdnY3s7GysW7eu0Ut4kOYz0VpPhFjI7yYKtdbg4ki8n02JYtWqVVCr1YiLi8MzzzyD4uJirFy50tGxkfuYaa0nQiz8RHyIhDxqUTiBTWMUMpkMU6dOxfr161FaWoqzZ88iKCjIwaGR+5lMZgA0RkEIUL1XTaBUhDItDWY7mk0tivfeew+bN28GUL339WeffUZ7WbuA5fFYfqNXXiHEK9F6T85h01+co0ePYseOHQCAFi1aYNeuXTh48KBDAyO1mWnPbEKsBErFUFOLwuFsShQGg8HqKSehUNikLUpJ85hoz2xCrFDXk3PYNEbx0EMP4fXXX8e0adPAcRwSExPRp08fR8dG7kOPxxJiLVAqgqbSAKPJDAF1yTqMTTW7cuVKKBQKrFu3Dhs2bKBFAV2EJtwRYk0uq3lElloVjmRTi0IikeCNN96o89hrr72GTZs22TUoUjdqURBirWbSnbqiCiFyPxdH472a3VbLzMy0RxzEBrTWEyHWaibd0ZNPjkWdeh7ESC0KQqzcu94TcRynrxP+n//8B7t27bK8vnXrFiZPnoy//vWvlvc+/PBD7Nu3D3K5HADw9NNP49lnn3V2qG6HWhSEWLMkCg3NznYkpyeKp556Ck899RQAID09Ha+88goWLFhgdU5aWho2bdqE6OhoZ4fn1kxmBo6jtZ4IqSEU8KEM9seJlDwMj46gfSkcxKVdT6tXr8aiRYsQEhJi9X5aWhq2bt2KiRMnIj4+Hno9fVoAqlsU1O1EiLWXJvWAuqIKH3+VCuPdZW6IfTU7UdTso91Yp06dgk6nw7hx46ze12q16NatG5YuXYr9+/dDrVbTciF3mc2Mup0IuU9USzlmj+uKa7fK8OXRdFeH45U4ZuNf+sLCQpSVlVm917FjR2RmZiIqKqrRN3711VcxevRoxMbGNnjepUuXsGzZMiQmJjb6Ht5m69ep+OFMNvasneDqUAhxO9u/SUPi8QysnDMQA3q0cHU4XsWmMYp169Zh9+7dkMlklvc4jsMvv/zSpCRRVVWFpKQkrF+/vtax3NxcnDp1CtOmTQNQ3WJp7N7cKpXGMvDbGApFAAoLyxt9nbNoNVXgAIfH6O71YG++Vt76eHo9jOvfGr9dzsd7X/6Gv70wwLJVqq08vfzNweNxCA2V1Xvcpr/AR44cwU8//YTg4GC7BHX16lW0a9cOEomk1jE/Pz+88847GDhwICIjI7F7927ExMTY5b6ezsRojIKQ+ggFPMyb1ANr/pWEDV+ew4i+EejfTYmgRiYMUptNYxTt2rWzPKpqDzk5OWjRwrppOHfuXKSmpiIkJATx8fGYP38+xo4dC8YYZs+ebbd7ezKTyUxjFIQ0oFWYFC9P7gEhn4cvj6bj9Y9O4p0vz+HnC3kwmWmgu6lsGqP43//+h88++wwDBw606ga6/7FWd+GtXU/bv72EK9mleOePQxx6H3evB3vztfLWx9vqIbdIizOX8/HrpXzkl1SiVZgUcY93RM+o0DrP97byN4Zdup4++OADhIaGorzcNyvRXVDXEyG2axUmxROPtsfkR6JwLr0ICceuY9OeFPTuEIpnHuuIlqFSV4foMWxKFJWVldi6daujYyEPQI/HEtJ4HMfhoc4K9GofiqPJt3DgVCb+uv0MnhzWHmMHtqEJrDawaYyiU6dOuHLliqNjIQ9gogl3hDSZUMDD2IFtsG7eYER3VmDvjxn4YO8FaCoNrg7N7dnUoigoKMDUqVMRGRkJkej3KfIHDhxwWGCkNmpRENJ8cqkI8yf3wLHWQfj30XSs2HoaM0d3wThFgKtDc1s2JYqcnBxs27YNPB6PtkB1IRMlCkLsguM4PP5wJDq3DsKOg5fxcWIaynRGPN63latDc0s2dT0tWLAA7733HlasWIFz584hKioKAwYMcHRs5D7U9USIfbVWyrBi1sN4uIsC/zmajpJyWleuLjYliunTpyMhIQFbtmxBWVkZ4uLi8Morrzg6NnIf6noixP74PB6eGtkRJpMZB07ddHU4bqlRiwLqdDpUVVWBMQY+n++omEg9TGYGASUKQuxOGeSPsYPb4aeUXOSXVLg6HLdj0xjFjh07sH//flRVVWHatGlISEhAWFiYo2Mj9zGbGQRC2pSQEEd4ZlRnHDmThe9OZ+H5cd1cHY5bsSlRXLx4EStWrMDAgQMdHQ9pAA1mE+I4wXI/tG8pR56KWhT3sylRvPvuu46Og9jAbGbg01NnhDiM1E+IXJXW1WG4HerH8CDUoiDEsaT+QmhpAl4tlCg8iJnWeiLEoWT+Qmh1xibv3OmtKFF4EFpmnBDHkvkLYTIz6KpMrg7FrVCicHOMMRSrdTAYzTThjhAHk/pXD9vS+k/WGrfHKHEavcGEHf+9jEs3i6HVGSES8mAyMfB5lNsJcRSZvxBAdaJQBPm7OBr3QYnCTSVfLUDSlQIM6hGODq0CkavS4kpWCdq3st9Og4QQazWJgga0rVGicFNnLhcgVC7Gi7Hdab18Qpzk3hYF+R31Y7ghTaUBFzOL0b9bOCUJQpxISomiTpQo3NDZqwUwmRkGdgt3dSiE+BSpHw1m14UShRs6cykf4SEStAmvf7NzQoj98Xk8SMQCaCuNrg7FrbhkjGLWrFlQqVQQCKpvHx8fjz59+liOnzp1CuvWrYNer8e4ceOwaNEiV4TpEiXlelzNLsXEoe1okyhCXEDmL4RGRy2Kezk9UTDGcOPGDfz444+WRHEvnU6HZcuWYefOnWjZsiVeeuklHD9+HMOHD3d2qC5x7LdbAIBBPVq4OBJCfJPUX0hdT/dxetfTjRs3wHEc5s6di0mTJmHXrl1Wxy9cuIC2bduidevWEAgEmDhxIg4dOuTsMF1CU2nA0eRbeLirEi1CJK4OhxCfJKNEUYvTWxRqtRqDBw/G6tWrodPpMGvWLERFRWHo0KEAgIKCAigUCsv5SqUS+fn5zg7TJX44mwNdlQkTh7RzdSiE+CyZvwB5tIKsFacniujoaERHRwMAJBIJpk2bhuPHj1sSRV2LcTW2rz40tOmDwApFQJOvbawyjR4HT91EVp4andsE4+hvtzGwRws81KOl02KojzPrwR34Wnnr4+v1oFAEQBEixfnrKp+vi3s5PVGcPXsWBoMBgwcPBlCdGO4dqwgPD0dRUZHldUFBAZRKZaPuoVJpYDY3fvVHhSIAhYXljb6uPmXaKlzJKkGZRo8KvREVOiMq9UZU6I3Q6ozIzFPDYDQjRC7GyQu5AIDR/SLtGkNT2Lse3J2vlbc+vl4PNeXngaFSb0TenTII+L7xYCiPxzX4AdvpiaK8vBybN2/Gv//9bxgMBuzfvx9r1qyxHO/Tpw8yMzORlZWFyMhIfPvtt5g6dapTYitW65CZp0Z5hQHlFVXQVhqgqzJBbzShqspc/X+DCVUGM/SG6q+NproTkt5gwp1i652y/MUCSMQCSPwE8BcLMLRXS4x6OBKtwqQoKdejvKIKbcLpUwwhrnTvMh6BMrGLo3EPTk8UI0eOREpKCp544gmYzWbMmDED0dHRmDx5Mj777DOEh4dj/fr1WLhwIfR6PYYPH46xY8c6PK7s/HLE/ysJdTVEBHwOIgEfYhEfIgEPYiEfIiEffiI++Hwe6uoY4/E4DO3VAj2iQqAM8oefSNDgEuHBAWIEB9APJSGudu8yHpQoqnHMC3foaErXk9FkRmaBFlqNHjKJEAESIWT+wupk4GMrtvpaF4Svlbc+vl4PNeW/mFmMd/ecx9IZ0ejSJtjVYTmF23U9uSsBn4chvVv59C8KIeTeFgXNzq7hWx+VCSHkAWo2L9LS7GwLShSEEHIPWmq8NkoUhBByD7GQDwGfo0RxD0oUhBByD47jaL2n+1CiIISQ+8j8hbQd6j0oURBCyH1kfkKoK6rqXFLIF1GiIISQ+wQFiJFxW41X//ETNu+9gIuZxT6dNGgeBSGE3Cfu8U7o0iYIN/PKcf56Ec7vOY8WIRJ0bROEqJZytG8lR8tQaYOrLXgTShSEEHKfQKkII/pGAH0Bg9GMXy/l4/SlO/j1cgF+PF+9gKdYxEf7lnJ0igxEz/ahaN9KDp6X7kpJiYIQQhogFPDwSO+WeKR3S5gZQ35xBW7kqnEjV42M22U4cOomvjl5E4EyEcKDqzcck4gFCAv0Q6RShj4dwxAoFbm4FM1DiYIQQmzE4zi0DJWiZagUQ3tV7xtToTPgQoYK59KLUF5RBcaAorJKXM4qgd5gAgegV4dQzH+iJ8RCvmsL0ESUKAghpBkkfkIM6tGi1j73jDHkFGiQdKUAB3/Jwv8duoK5sd0bvRGbO6BEQQghDsBxHNqEB6BNeABEAh72/5SJdi3kGN2/tatDazRKFIQQ4mAThrTDzTvl2HM0HddvlSJ2SDuP2qSM5lEQQoiD8TgO8yb2wPjBbZGWWYz4f51Fnkrr6rBsRomCEEKcQCziY+rwDlg8PRpmxpBbRImCEEJIHYLubq+qrvCctaQoURBCiBMFSKr3uyivqHJxJLajREEIIU4k4PPgLxagXOs5LQqXPPX04Ycf4rvvvgMADB8+HEuWLKl1fN++fZDL5QCAp59+Gs8++6zT4ySEEEeQS4Qor/ScFoXTE8WpU6fw888/Y//+/eA4Di+++CKOHDmCmJgYyzlpaWnYtGkToqOjnR0eIYQ4XIBEhHIPGqNweqJQKBR44403IBJVr33SoUMH5ObmWp2TlpaGrVu3IicnB/3798fSpUshFoudHSohhDhEgESIgtJKV4dhM6ePUXTq1Al9+/YFANy8eRMHDx7E8OHDLce1Wi26deuGpUuXYv/+/VCr1fj444+dHSYhhDiMp7UoOOai3TjS09Px0ksvYeHChXjyySfrPe/SpUtYtmwZEhMTnRccIYQ40M7vLmPvsXTsf3uiR+xp4ZLB7OTkZLz66qtYtmwZJkyYYHUsNzcXp06dwrRp0wBUL6wlEDQuTJVKA7O58flPoQhAYWF5o6/zNr5WD75W3vr4ej04s/x8xmA2M2TdKoHMX+iUezaEx+MQGiqr/7gTYwEA5OXl4ZVXXsHGjRtrJQkA8PPzwzvvvIOcnBwwxrB7926rgW5CCPF0AdLq5KDWesaTT05vUWzfvh16vR7r16+3vBcXF4djx47h1VdfRa9evRAfH4/58+fDYDDgoYcewuzZs50dJiGEOEyApPphnupJd1LXBmMDl41ROBJ1PTWPr9WDr5W3Pr5eD84sf3Z+OVb/Mwl/fKIn+nVVOuWeDXG7ridCCPF1cum9LQr3R4mCEEKcrGYA21MWBqREQQghTibg8yD1E1CLghBCSP1kHjTpjhIFIYS4gFwipBYFIYSQ+nnSMh6UKAghxAXkEiHU1KIghBBSH5lEBE2loUlzvpyNEgUhhLiAXCIEY4BG5/7dT5QoCCHEBX5fxoMSBSGEkDrIJdWT7so9YGFAShSEEOICNS2KMkoUhBBC6qII9ofMX4gfknNgdvO1WSlREEKIC4iFfDw9siMybqtx8kKeq8NpECUKQghxkSG9WqBjZCD+82MGNJXuO6hNiYIQQlyEx3F4bnQXVOiM+HDfBWjd9FFZShSEEOJCrZUyzJvUHTfy1HhrZzKSrxbgXHoh0m+VolJvdHV4AFywFSohhBBrA7qFQy4R4YOvUvHR/jSrY5EKGYb0bIHBPcIRKBO7JD5KFIQQ4ga6tg3G2y8PhqpMBwaGUk0VsvPLkZqhQsL/ruOrExkYER2B2MHtLDvkOQslCkIIcRMyf6Fl9zsA6NsxDJOGRiFPpcWhX7NxNPkWfr6Qh+dGd8Hgni2cFheNURBCiJtrGSrF7PHd8PcXB6KNUoat317C9v9ectqTUi5JFAcOHMD48eMRExOD3bt31zp++fJlTJ06FWPGjMHy5cthNLrHgA4hhLhSy1ApFs+IRuyQdjiVegeLPz6FPcfS8dOFXJxMzUOFzjF/K52eKPLz8/Hee+/hiy++wNdff409e/bg+vXrVucsXrwYK1euxOHDh8EYQ0JCgrPDJIQQt8Tn8TBlWHvEvzgQ0Z3D8H1SDv558Aq2//cyfrtW6JB7On2M4tSpUxg0aBCCgoIAAGPGjMGhQ4ewYMECAMDt27eh0+nQt29fAMCUKVOwefNmzJgxw9mhEkKI24oIk2LexB6YMaozdFVG8DgOIXI/h9zL6S2KgoICKBQKy2ulUon8/Px6jysUCqvjhBBCfifzFyIs0N9hSQJwQYuC1bH4FcdxNh+3RWiorPGB3aVQBDT5Wm/ia/Xga+Wtj6/Xg6+Xvz5OTxTh4eE4e/as5XVBQQGUSqXV8aKiIsvrwsJCq+O2UKk0TdpeUKEIQGFheaOv8za+Vg++Vt76+Ho9+HL5eTyuwQ/YTu96GjJkCH755RcUFxejsrIS33//PYYNG2Y5HhERAbFYjOTkZABAYmKi1XFCCCHO5fREER4ejkWLFmHWrFl44oknEBsbi969e2Pu3LlITU0FAGzcuBHr1q3DuHHjUFlZiVmzZjk7TEIIIXdxrK5BAQ9HXU/N42v14GvlrY+v14Mvl9/tup4IIYR4Fq9c64nHa9xTUva61pv4Wj34Wnnr4+v14Kvlf1C5vbLriRBCiP1Q1xMhhJAGUaIghBDSIEoUhBBCGkSJghBCSIMoURBCCGkQJQpCCCENokRBCCGkQZQoCCGENIgSBSGEkAZ5xRIeGo0GcXFx2LJlCyIjI/HVV19h27Zt4PP5GDhwIN544w2UlZVhzpw5lmvKy8tRUlKCc+fOQa1W4y9/+QtycnIQEhKC999/32qXvRq5ublYvHgxVCoVoqKisHHjRkilUsvxvXv34uzZs1i/fr1Tyn0/V9dDRkYGVq5cCa1WCz8/P6xevRrdunXz2vJev34dK1asQEVFBQIDA7F+/XpEREQ4rLz1cXU91Lhz5w4mTZqEr776CpGRkU4pO+D68iclJWHBggVo0aIFAKB79+5Yt26d08rvFMzDnT9/nsXGxrIePXqwnJwclpGRwR599FGWn5/PGGNs1apVbMeOHVbXmEwmNnPmTPbNN98wxhhbs2YN+/TTTxljjO3fv5/96U9/qvNe8+bNY99++y1jjLEPP/yQbdiwgTHGmE6nY++88w7r27cvW7p0qSOK+UDuUA9xcXHs2LFjjDHGTp06xSZOnGj3ctZwh/LOnDmTHT9+nDHG2BdffMFee+01u5fzQdyhHmq+55w5c1jfvn1ZTk6OvYtZL3co//bt29mWLVscUTy34fFdTwkJCVi1apVlF7yrV6+ib9++ltcjR47EDz/8YHXNvn374O/vj4kTJwIAfvzxR8vXsbGxOHHiBAwGg9U1BoMBSUlJGDNmDABgypQpOHToEAAgKSkJZrMZixcvdlxBH8Ad6uGpp56ybDLVpUsX5OXlOai07lHef/7znxg2bBjMZjNyc3Mhl8sdVt76uEM9AMC2bdswZMgQBAcHO6ag9XCH8qempuLkyZN44okn8PLLLzv0595VPD5RrF27Fv369bO87tq1K1JSUpCXlweTyYRDhw5Zba1qMpnwySef4PXXX7e8V1BQYGlqCgQCyGQyFBcXW92npKQEMpkMAkF1b51CoUB+fj4A4JFHHsGSJUvg5+e4zc0fxB3qYcqUKeDz+QCAzZs3Y9SoUY4pLNyjvAKBAGq1GsOGDcOXX36Jp59+2mHlrY871ENaWhp+/fVXzJ4922HlrI87lD8gIACzZs1CYmIihg8fjkWLFjmsvK7iFWMU94qKisLrr7+O+fPnw8/PD2PHjrXsnAcAP/30E6KiotClS5cGvw+PZ51DWR2L7HKc+y5J7Kp6YIxhw4YNSElJweeff97MUtjOVeWVy+X4+eefceLECcyfPx9Hjx61JEtXcHY9VFZWIj4+Hu+//36ta1zBFT8H8fHxlvemT5+Od999F+Xl5QgICGhOUdyK1yUKvV6P3r17IzExEQDw/fffo3Xr1pbjP/zwA8aPH291jVKpRFFREVq0aAGj0QiNRoOgoCBMnjzZcs7evXuh0WhgMpnA5/NRWFhoad66I1fUg9FoxNKlS5Gfn4/PP//cqb8orijvwYMHMW7cOHAch2HDhkGn06GsrAwhISGOL3A9nF0PZ8+eRVFREebPnw+g+tP5vHnz8OGHH6J9+/aOL/B9nF1+s9mMTz/9FPPmzbP6gFDT8vAWrv8IYGcVFRX4wx/+AI1Gg6qqKuzcudPqB+P8+fNWTVUAGD58uOUH6+DBg+jXrx+EQiG+/vpry39CoRD9+vXDwYMHAQCJiYmW/nh35Ip6ePvtt6HRaLBjxw6nf5pyRXl37NiBI0eOAABOnz6N4OBglyYJwPn18Oijj+LYsWOW85RKJT777DOXJAnA+eXn8Xg4cuQIDh8+bHm/T58+8Pf3d06BncWVI+n2NHLkSMvTFgkJCWz8+PFs9OjRbPPmzVbn9e7dm+l0Oqv3SkpK2EsvvcTGjx/PnnnmmXqf2rh16xabOXMmGzduHJszZw4rLS21Or5v3z6XPfVUw1X1oFKpWLdu3VhMTAybNGmS5T9Hc+W/e3p6OouLi2OTJk1izz77LLt27ZoDSmgbd/j5vz8OZ3Jl+a9du8aeeeYZNn78eDZz5kyWm5vrgBK6Fu1wRwghpEFe1/VECCHEvihREEIIaRAlCkIIIQ2iREEIIaRBlCgIIYQ0iBIF8Xlz5sxBcXEx5s6di+vXrzv0Xjk5OVi4cKFD70GIvXnX9EFCmuDkyZMAgK1btzr8Xrm5ucjMzHT4fQixJ5pHQXzam2++ia+++gqdO3fG9evXkZCQgIqKCmzatAlKpRLp6enw9/fHwoULsXPnTmRmZmL06NFYtmwZAODYsWP45JNPYDAY4Ofnh6VLlyI6OhoZGRlYvnw5qqqqwBjDtGnTEBcXh7FjxyI/Px/9+/fH9u3bsWXLFvzwww/Q6/WorKzE0qVLERMTgw8++ADZ2dnIyclBQUEBevfujaFDhyIxMRG3bt3C4sWLERsbiw8++ADp6ekoKiqCSqVC165dsXbtWshkMhfXLPEqLp3uR4gb6Ny5M1OpVGzkyJHswoUL7PTp06xbt27s4sWLjDHGXnjhBfbMM88wvV7PVCoV69GjB7tz5w7LzMxksbGxrLi4mDFWPUN36NChTKvVsjfffNOyx0FBQQH785//zEwmEzt9+jSbMGECY6x6pu9zzz3HKisrGWOMffvttyw2NpYxxtjmzZvZyJEjmVqtZpWVlax///5s3bp1jDHGjhw5wkaPHm05b9iwYaywsJCZTCb22muvsfXr1zuv8ohPoK4nQuoQGRmJ7t27AwDatGmDgIAAiEQihISEQCqVoqysDElJSSgoKMDzzz9vuY7jOGRnZyMmJgZLly7FhQsXMHjwYKxYsaLWiqQRERF4++23ceDAAWRlZSElJQVardZyfMiQIZY1s5RKJR599FFLPKWlpZbzxo4di7CwMADAtGnT8NZbb2Hp0qWOqBbio2gwm5A6iEQiq9d1rQZqNpsxePBgq8XjEhIS0KlTJ4wcORKHDx/GuHHjcPnyZUycOBHZ2dlW11+8eBFxcXHQaDQYOnQoXnzxxUbHAMBq1VKz2ewWy30T70I/UcTn8fl8GI3GRl83aNAgnDx5EhkZGQCA48ePY9KkSdDr9Xj99ddx8OBBTJgwAatWrYJMJkNeXh74fL5l97SkpCT07NkTs2fPxoABA3D06FGYTKZGx3H06FGUl5fDbDYjISEBI0eObPT3IKQh1PVEfF5MTAxmzJhh1e1ji06dOiE+Ph6vvfYaGGMQCAT45JNPIJFI8Mc//hHLly/Hnj17wOfzMWrUKAwYMABqtRp8Ph/Tpk3Dli1b8P3332P8+PEQCoUYPHgwysrKoNFoGhVHWFgY5s6di5KSEvTv3x8vv/xyo64n5EHoqSdCPNgHH3yAkpIS/PWvf3V1KMSLUdcTIYSQBlGLghBCSIOoRUEIIaRBlCgIIYQ0iBIFIYSQBlGiIIQQ0iBKFIQQQhpEiYIQQkiD/h+Ml3UhPMVpcAAAAABJRU5ErkJggg==\n", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "sns.lineplot(x=\"timestamp\", y=\"vm_count\", data=df.resample('1D').mean())" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} -- cgit v1.2.3 From f0f59a0b98fe474da4411c0d5048ccdf4a2d7c43 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 09:48:07 +0200 Subject: exp: Use LocalInputFile for Parquet readers This change updates the Parquet readers used in the Capelin experiments to use our InputFile implementation for local files, to reduce our dependency on Apache Hadoop. --- .../experiments/capelin/trace/Sc20ParquetTraceReader.kt | 1 - .../capelin/trace/Sc20RawParquetTraceReader.kt | 14 +++----------- .../capelin/trace/Sc20StreamingParquetTraceReader.kt | 17 ++++++++--------- 3 files changed, 11 insertions(+), 21 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt index a8462a51..7f25137e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt @@ -38,7 +38,6 @@ import java.util.TreeSet * @param performanceInterferenceModel The performance model covering the workload in the VM trace. * @param run The run to which this reader belongs. */ -@OptIn(ExperimentalStdlibApi::class) public class Sc20ParquetTraceReader( rawReaders: List, performanceInterferenceModel: Map, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt index bd27cf02..54151c9f 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt @@ -24,10 +24,9 @@ package org.opendc.experiments.capelin.trace import mu.KotlinLogging import org.apache.avro.generic.GenericData -import org.apache.hadoop.fs.Path -import org.apache.parquet.avro.AvroParquetReader import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.format.util.LocalParquetReader import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import java.io.File @@ -40,16 +39,12 @@ private val logger = KotlinLogging.logger {} * * @param path The directory of the traces. */ -@OptIn(ExperimentalStdlibApi::class) public class Sc20RawParquetTraceReader(private val path: File) { /** * Read the fragments into memory. */ private fun parseFragments(path: File): Map> { - @Suppress("DEPRECATION") - val reader = AvroParquetReader.builder(Path(path.absolutePath, "trace.parquet")) - .disableCompatibility() - .build() + val reader = LocalParquetReader(File(path, "trace.parquet")) val fragments = mutableMapOf>() @@ -81,10 +76,7 @@ public class Sc20RawParquetTraceReader(private val path: File) { * Read the metadata into a workload. */ private fun parseMeta(path: File, fragments: Map>): List> { - @Suppress("DEPRECATION") - val metaReader = AvroParquetReader.builder(Path(path.absolutePath, "meta.parquet")) - .disableCompatibility() - .build() + val metaReader = LocalParquetReader(File(path, "meta.parquet")) var counter = 0 val entries = mutableListOf>() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt index c5294b55..6792c2ab 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt @@ -24,7 +24,6 @@ package org.opendc.experiments.capelin.trace import mu.KotlinLogging import org.apache.avro.generic.GenericData -import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetReader import org.apache.parquet.filter2.compat.FilterCompat import org.apache.parquet.filter2.predicate.FilterApi @@ -33,6 +32,7 @@ import org.apache.parquet.filter2.predicate.UserDefinedPredicate import org.apache.parquet.io.api.Binary import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader +import org.opendc.format.util.LocalInputFile import org.opendc.simulator.compute.interference.IMAGE_PERF_INTERFERENCE_MODEL import org.opendc.simulator.compute.interference.PerformanceInterferenceModel import org.opendc.simulator.compute.workload.SimTraceWorkload @@ -54,7 +54,6 @@ private val logger = KotlinLogging.logger {} * @param traceFile The directory of the traces. * @param performanceInterferenceModel The performance model covering the workload in the VM trace. */ -@OptIn(ExperimentalStdlibApi::class) public class Sc20StreamingParquetTraceReader( traceFile: File, performanceInterferenceModel: PerformanceInterferenceModel? = null, @@ -96,10 +95,10 @@ public class Sc20StreamingParquetTraceReader( * The thread to read the records in. */ private val readerThread = thread(start = true, name = "sc20-reader") { - @Suppress("DEPRECATION") - val reader = AvroParquetReader.builder(Path(traceFile.absolutePath, "trace.parquet")) + val reader = AvroParquetReader + .builder(LocalInputFile(File(traceFile, "trace.parquet"))) .disableCompatibility() - .run { if (filter != null) withFilter(filter) else this } + .withFilter(filter) .build() try { @@ -164,10 +163,10 @@ public class Sc20StreamingParquetTraceReader( val entries = mutableMapOf() val buffers = mutableMapOf>>() - @Suppress("DEPRECATION") - val metaReader = AvroParquetReader.builder(Path(traceFile.absolutePath, "meta.parquet")) + val metaReader = AvroParquetReader + .builder(LocalInputFile(File(traceFile, "meta.parquet"))) .disableCompatibility() - .run { if (filter != null) withFilter(filter) else this } + .withFilter(filter) .build() while (true) { @@ -178,7 +177,7 @@ public class Sc20StreamingParquetTraceReader( metaReader.close() - val selection = if (selectedVms.isEmpty()) entries.keys else selectedVms + val selection = selectedVms.ifEmpty { entries.keys } // Create the entry iterator iterator = selection.asSequence() -- cgit v1.2.3 From 77075ff4680667da70bdee00be7fa83539a50439 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 12:36:37 +0200 Subject: exp: Use LocalOutputFile for Parquet writers This change updates the Parquet writers used in the Capelin experiments to use our OutputFile implementation for local files, to reduce our dependency on Apache Hadoop. --- .../experiments/capelin/telemetry/parquet/ParquetEventWriter.kt | 5 ++--- .../org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt | 8 +++----- 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt index 4fa6ae66..d8f7ff75 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt @@ -25,10 +25,10 @@ package org.opendc.experiments.capelin.telemetry.parquet import mu.KotlinLogging import org.apache.avro.Schema import org.apache.avro.generic.GenericData -import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.opendc.experiments.capelin.telemetry.Event +import org.opendc.format.util.LocalOutputFile import java.io.Closeable import java.io.File import java.util.concurrent.ArrayBlockingQueue @@ -52,8 +52,7 @@ public open class ParquetEventWriter( /** * The writer to write the Parquet file. */ - @Suppress("DEPRECATION") - private val writer = AvroParquetWriter.builder(Path(path.absolutePath)) + private val writer = AvroParquetWriter.builder(LocalOutputFile(path)) .withSchema(schema) .withCompressionCodec(CompressionCodecName.SNAPPY) .withPageSize(4 * 1024 * 1024) // For compression diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt index 1f9e289c..d0031a66 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt @@ -38,11 +38,11 @@ import me.tongfei.progressbar.ProgressBar import org.apache.avro.Schema import org.apache.avro.SchemaBuilder import org.apache.avro.generic.GenericData -import org.apache.hadoop.fs.Path import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.opendc.format.trace.sc20.Sc20VmPlacementReader +import org.opendc.format.util.LocalOutputFile import java.io.BufferedReader import java.io.File import java.io.FileReader @@ -109,16 +109,14 @@ public class TraceConverterCli : CliktCommand(name = "trace-converter") { traceParquet.delete() } - @Suppress("DEPRECATION") - val metaWriter = AvroParquetWriter.builder(Path(metaParquet.toURI())) + val metaWriter = AvroParquetWriter.builder(LocalOutputFile(metaParquet)) .withSchema(metaSchema) .withCompressionCodec(CompressionCodecName.SNAPPY) .withPageSize(4 * 1024 * 1024) // For compression .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) .build() - @Suppress("DEPRECATION") - val writer = AvroParquetWriter.builder(Path(traceParquet.toURI())) + val writer = AvroParquetWriter.builder(LocalOutputFile(traceParquet)) .withSchema(schema) .withCompressionCodec(CompressionCodecName.SNAPPY) .withPageSize(4 * 1024 * 1024) // For compression -- cgit v1.2.3 From 0eb4fa604efe4e0b84d69749f688a79c2249c8b3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 13:38:56 +0200 Subject: build: Eliminate most Hadoop dependencies This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 5 ----- opendc-experiments/opendc-experiments-serverless20/build.gradle.kts | 6 ------ opendc-experiments/opendc-experiments-tf20/build.gradle.kts | 5 ----- 3 files changed, 16 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 0dade513..324cae3e 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -44,10 +44,5 @@ dependencies { implementation(libs.clikt) implementation(libs.parquet) - implementation(libs.hadoop.client) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } - testImplementation(libs.log4j.slf4j) } diff --git a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts index 88479765..7d68cb3a 100644 --- a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts @@ -37,10 +37,4 @@ dependencies { implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(libs.kotlin.logging) implementation(libs.config) - - implementation(libs.parquet) - implementation(libs.hadoop.client) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } } diff --git a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts index 64483bd4..b088045b 100644 --- a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts @@ -38,9 +38,4 @@ dependencies { implementation(projects.opendcUtils) implementation(libs.kotlin.logging) - implementation(libs.parquet) - implementation(libs.hadoop.client) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } } -- cgit v1.2.3 From 1768292251957da5ce6411ecc7d2dffebf8709c8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 10 Jun 2021 17:32:43 +0200 Subject: simulator: Integrate power subsystem with compute subsystem This change integrates the power subsystem of the simulator with the compute subsystem by exposing a new field on a SimBareMetalMachine, psu, which provides access to the machine's PSU, which in turn can be connected to a SimPowerOutlet. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 001547ef..92e7080a 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -153,7 +153,7 @@ public class SimTFDevice( } SimResourceEvent.Run -> { _usage.record(ctx.speed) - _power.record(machine.powerDraw) + _power.record(machine.psu.powerDraw) } else -> {} } -- cgit v1.2.3 From 145153ddda7f9caf95831ab27244351772a121d8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 15 Jun 2021 11:43:48 +0200 Subject: exp: Fix execution of energy experiments This change fixes the execution of the Energy21 experiments which failed due to various changes in the OpenDC codebase. First, the directory structure is now required to be pre-generated before the writer starts writing the experiment output. Second, we must include the configuration of the Capelin experiment in this experiment in order to workaround an issue with harness filtering. --- .../opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt | 4 ++-- .../opendc-experiments-energy21/src/main/resources/application.conf | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt index 983b4cff..bfdf5f3e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt @@ -43,11 +43,11 @@ private val logger = KotlinLogging.logger {} */ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: Int) : ExperimentMonitor { private val hostWriter = ParquetHostEventWriter( - File(base, "host-metrics/$partition/data.parquet"), + File(base, "host-metrics/$partition/data.parquet").also { it.parentFile.mkdirs() }, bufferSize ) private val provisionerWriter = ParquetProvisionerEventWriter( - File(base, "provisioner-metrics/$partition/data.parquet"), + File(base, "provisioner-metrics/$partition/data.parquet").also { it.parentFile.mkdirs() }, bufferSize ) diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf b/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf index 3e011862..263da0fe 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf +++ b/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf @@ -6,3 +6,9 @@ opendc.experiments.energy21 { # Path to the output directory to write the results to output-path = output } + +opendc.experiments.capelin { + env-path = input/environments/ + trace-path = input/traces/ + output-path = output +} -- cgit v1.2.3 From 81c3b51169cc5dfafb80abb0cf55abb49646a72a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 16 Jun 2021 16:14:39 +0200 Subject: exp: Fix power tracking for energy experiments This change fixes an issue where the power in the energy experiments is always reported as zero due to the changes in commit 652b869. --- .../main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 37e10580..a6419441 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -87,11 +87,7 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { weighers = listOf(RandomWeigher(Random(0)) to 1.0) ) - val meterProvider: MeterProvider = SdkMeterProvider - .builder() - .setClock(clock.toOtelClock()) - .build() - + val meterProvider: MeterProvider = createMeterProvider(clock) val monitor = ParquetExperimentMonitor(File(config.getString("output-path")), "power_model=$powerModel/run_id=$repeat", 4096) val trace = Sc20StreamingParquetTraceReader(File(config.getString("trace-path"), trace), random = Random(1).asKotlinRandom()) -- cgit v1.2.3 From 0fd7b3116fbace7deb8202d1849cece7146462a9 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 17 Jun 2021 14:15:45 +0200 Subject: faas: Rename opendc-serverless to opendc-faas This change renames the opendc-serverless module to opendc-faas to better distinguish between the two terms (Serverless and FaaS) and be clearer about the intent of the module. The opendc-faas module holds the code for the FaaS platform on top of OpenDC. Although this is one approach of doing serverless, serverless can also entail other services that will not be covered by this module. --- .../org/opendc/experiments/energy21/EnergyExperiment.kt | 2 -- .../opendc-experiments-serverless20/build.gradle.kts | 4 ++-- .../opendc/experiments/serverless/ServerlessExperiment.kt | 14 +++++++------- .../experiments/serverless/trace/FunctionTraceWorkload.kt | 6 +++--- 4 files changed, 12 insertions(+), 14 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index a6419441..65915cc6 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -24,7 +24,6 @@ package org.opendc.experiments.energy21 import com.typesafe.config.ConfigFactory import io.opentelemetry.api.metrics.MeterProvider -import io.opentelemetry.sdk.metrics.SdkMeterProvider import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.channels.Channel @@ -50,7 +49,6 @@ import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.* import org.opendc.simulator.core.runBlockingSimulation -import org.opendc.telemetry.sdk.toOtelClock import java.io.File import java.time.Clock import java.util.* diff --git a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts index 7d68cb3a..65c31c4f 100644 --- a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts @@ -32,8 +32,8 @@ dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcSimulator.opendcSimulatorCore) - implementation(projects.opendcServerless.opendcServerlessService) - implementation(projects.opendcServerless.opendcServerlessSimulator) + implementation(projects.opendcFaas.opendcFaasService) + implementation(projects.opendcFaas.opendcFaasSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(libs.kotlin.logging) implementation(libs.config) diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt index 516bcc3e..3a016491 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt @@ -31,14 +31,14 @@ import kotlinx.coroutines.launch import mu.KotlinLogging import org.opendc.experiments.serverless.trace.FunctionTraceWorkload import org.opendc.experiments.serverless.trace.ServerlessTraceReader +import org.opendc.faas.service.FaaSService +import org.opendc.faas.service.autoscaler.FunctionTerminationPolicyFixed +import org.opendc.faas.service.router.RandomRoutingPolicy +import org.opendc.faas.simulator.SimFunctionDeployer +import org.opendc.faas.simulator.delay.ColdStartModel +import org.opendc.faas.simulator.delay.StochasticDelayInjector import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf -import org.opendc.serverless.service.ServerlessService -import org.opendc.serverless.service.autoscaler.FunctionTerminationPolicyFixed -import org.opendc.serverless.service.router.RandomRoutingPolicy -import org.opendc.serverless.simulator.SimFunctionDeployer -import org.opendc.serverless.simulator.delay.ColdStartModel -import org.opendc.serverless.simulator.delay.StochasticDelayInjector import org.opendc.simulator.compute.SimMachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode @@ -85,7 +85,7 @@ public class ServerlessExperiment : Experiment("Serverless") { val delayInjector = StochasticDelayInjector(coldStartModel, Random()) val deployer = SimFunctionDeployer(clock, this, createMachineModel(), delayInjector) { FunctionTraceWorkload(traceById.getValue(it.name)) } val service = - ServerlessService(coroutineContext, clock, meterProvider.get("opendc-serverless"), deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = 10 * 60 * 1000)) + FaaSService(coroutineContext, clock, meterProvider.get("opendc-serverless"), deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = 10 * 60 * 1000)) val client = service.newClient() coroutineScope { diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt index 7d824857..9a93092e 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt @@ -22,13 +22,13 @@ package org.opendc.experiments.serverless.trace -import org.opendc.serverless.simulator.workload.SimServerlessWorkload +import org.opendc.faas.simulator.workload.SimFaaSWorkload import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload /** - * A [SimServerlessWorkload] for a [FunctionTrace]. + * A [SimFaaSWorkload] for a [FunctionTrace]. */ -public class FunctionTraceWorkload(trace: FunctionTrace) : SimServerlessWorkload, SimWorkload by SimTraceWorkload(trace.samples.asSequence().map { SimTraceWorkload.Fragment(it.duration, it.cpuUsage, 1) }) { +public class FunctionTraceWorkload(trace: FunctionTrace) : SimFaaSWorkload, SimWorkload by SimTraceWorkload(trace.samples.asSequence().map { SimTraceWorkload.Fragment(it.duration, it.cpuUsage, 1) }) { override suspend fun invoke() {} } -- cgit v1.2.3 From df3c9dc3fcd2f89910575bfdc24a3db3af9eba0f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 20 Jun 2021 22:21:39 +0200 Subject: exp: Enable interpreter sharing across hosts This change enables the experiments to share the SimResourceInterpreter across multiple hosts, which allows updates to be scheduled efficiently for all machines at the same time. This is especially beneficial if the machines operate on the same time slices. --- .../main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt | 4 +++- .../main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 0fbb7280..0485415c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -50,6 +50,7 @@ import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.failures.CorrelatedFaultInjector import org.opendc.simulator.failures.FaultInjector +import org.opendc.simulator.resources.SimResourceInterpreter import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import org.opendc.telemetry.sdk.toOtelClock import java.io.File @@ -144,6 +145,7 @@ public suspend fun withComputeService( scheduler: ComputeScheduler, block: suspend CoroutineScope.(ComputeService) -> Unit ): Unit = coroutineScope { + val interpreter = SimResourceInterpreter(coroutineContext, clock) val hosts = environmentReader .use { it.read() } .map { def -> @@ -153,7 +155,7 @@ public suspend fun withComputeService( def.model, def.meta, coroutineContext, - clock, + interpreter, meterProvider.get("opendc-compute-simulator"), SimFairShareHypervisorProvider(), def.powerModel diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 65915cc6..2f14776a 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -49,6 +49,7 @@ import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.* import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.simulator.resources.SimResourceInterpreter import java.io.File import java.time.Clock import java.util.* @@ -120,6 +121,7 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { block: suspend CoroutineScope.(ComputeService) -> Unit ): Unit = coroutineScope { val model = createMachineModel() + val interpreter = SimResourceInterpreter(coroutineContext, clock) val hosts = List(64) { id -> SimHost( UUID(0, id.toLong()), @@ -127,7 +129,7 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { model, emptyMap(), coroutineContext, - clock, + interpreter, meterProvider.get("opendc-compute-simulator"), SimFairShareHypervisorProvider(), PerformanceScalingGovernor(), -- cgit v1.2.3 From b29f90e5ad5bcac29cde86e56c06e0b65a52cedc Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 21 Jun 2021 20:57:06 +0200 Subject: simulator: Re-organize compute simulator module This change re-organizes the classes of the compute simulator module to make a clearer distinction between the hardware, firmware and software interfaces in this module. --- .../kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt | 2 +- .../kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt | 10 +++++----- .../org/opendc/experiments/serverless/ServerlessExperiment.kt | 6 +++--- .../kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 4 ++-- .../org/opendc/experiments/tf20/util/MLEnvironmentReader.kt | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 0485415c..47f5f71e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -44,8 +44,8 @@ import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.Sc20StreamingParquetTraceReader import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader -import org.opendc.simulator.compute.SimFairShareHypervisorProvider import org.opendc.simulator.compute.interference.PerformanceInterferenceModel +import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.failures.CorrelatedFaultInjector diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 2f14776a..28928dcb 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -41,9 +41,9 @@ import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor import org.opendc.experiments.capelin.trace.Sc20StreamingParquetTraceReader import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf -import org.opendc.simulator.compute.SimFairShareHypervisorProvider -import org.opendc.simulator.compute.SimMachineModel -import org.opendc.simulator.compute.cpufreq.* +import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider +import org.opendc.simulator.compute.kernel.cpufreq.PerformanceScalingGovernor +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit @@ -156,12 +156,12 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { /** * The machine model based on: https://www.spec.org/power_ssj2008/results/res2020q1/power_ssj2008-20191125-01012.html */ - private fun createMachineModel(): SimMachineModel { + private fun createMachineModel(): MachineModel { val node = ProcessingNode("AMD", "am64", "EPYC 7742", 64) val cpus = List(node.coreCount) { id -> ProcessingUnit(node, id, 3400.0) } val memory = List(8) { MemoryUnit("Samsung", "Unknown", 2933.0, 16_000) } - return SimMachineModel(cpus, memory) + return MachineModel(cpus, memory) } /** diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt index 3a016491..231b491e 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt @@ -39,7 +39,7 @@ import org.opendc.faas.simulator.delay.ColdStartModel import org.opendc.faas.simulator.delay.StochasticDelayInjector import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf -import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit @@ -122,10 +122,10 @@ public class ServerlessExperiment : Experiment("Serverless") { /** * Construct the machine model to test with. */ - private fun createMachineModel(): SimMachineModel { + private fun createMachineModel(): MachineModel { val cpuNode = ProcessingNode("Intel", "Xeon", "amd64", 2) - return SimMachineModel( + return MachineModel( cpus = List(cpuNode.coreCount) { ProcessingUnit(cpuNode, it, 1000.0) }, memory = List(4) { MemoryUnit("Crucial", "MTA18ASF4G72AZ-3G2B1", 3200.0, 32_000) } ) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 92e7080a..d8f92155 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -28,7 +28,7 @@ import kotlinx.coroutines.* import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine import org.opendc.simulator.compute.SimMachineContext -import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel @@ -63,7 +63,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), + SimResourceInterpreter(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), SimplePowerDriver(powerModel) ) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt index eea079fb..3e61f508 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt @@ -27,7 +27,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.opendc.format.environment.EnvironmentReader import org.opendc.format.environment.MachineDef -import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit @@ -100,7 +100,7 @@ public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jack UUID(0, counter.toLong()), "node-${counter++}", mapOf("gpu" to isGpuFlag), - SimMachineModel(cores, memories), + MachineModel(cores, memories), LinearPowerModel(maxPower, minPower) ) } -- cgit v1.2.3 From be34a55c2c2fe94a6883c6b97d2abe4c43288e8a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 23 Jun 2021 16:54:31 +0200 Subject: format: Remove performance interference from trace readers This change updates the trace reader implementation to remove their dependency on the performance interference model. In a future commit, we will instead pass the performance interference model via the host/hypervisor. --- .../experiments/capelin/ExperimentHelpers.kt | 49 +- .../org/opendc/experiments/capelin/Portfolio.kt | 34 +- .../capelin/env/ClusterEnvironmentReader.kt | 122 ++++ .../capelin/trace/ParquetTraceReader.kt | 65 +++ .../capelin/trace/RawParquetTraceReader.kt | 137 +++++ .../capelin/trace/Sc20ParquetTraceReader.kt | 83 --- .../capelin/trace/Sc20RawParquetTraceReader.kt | 149 ----- .../trace/Sc20StreamingParquetTraceReader.kt | 283 ---------- .../capelin/trace/Sc20TraceConverter.kt | 621 --------------------- .../capelin/trace/StreamingParquetTraceReader.kt | 261 +++++++++ .../experiments/capelin/trace/TraceConverter.kt | 620 ++++++++++++++++++++ .../experiments/capelin/trace/VmPlacementReader.kt | 52 ++ .../experiments/capelin/CapelinIntegrationTest.kt | 13 +- .../experiments/energy21/EnergyExperiment.kt | 5 +- 14 files changed, 1290 insertions(+), 1204 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 47f5f71e..06251dd3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -41,10 +41,8 @@ import org.opendc.compute.service.scheduler.ComputeScheduler import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.experiments.capelin.trace.Sc20StreamingParquetTraceReader import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader -import org.opendc.simulator.compute.interference.PerformanceInterferenceModel import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload @@ -53,7 +51,6 @@ import org.opendc.simulator.failures.FaultInjector import org.opendc.simulator.resources.SimResourceInterpreter import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import org.opendc.telemetry.sdk.toOtelClock -import java.io.File import java.time.Clock import kotlin.coroutines.resume import kotlin.math.ln @@ -68,7 +65,7 @@ private val logger = KotlinLogging.logger {} /** * Construct the failure domain for the experiments. */ -public fun createFailureDomain( +fun createFailureDomain( coroutineScope: CoroutineScope, clock: Clock, seed: Int, @@ -100,7 +97,7 @@ public fun createFailureDomain( /** * Obtain the [FaultInjector] to use for the experiments. */ -public fun createFaultInjector( +fun createFaultInjector( coroutineScope: CoroutineScope, clock: Clock, random: Random, @@ -118,27 +115,10 @@ public fun createFaultInjector( ) } -/** - * Create the trace reader from which the VM workloads are read. - */ -public fun createTraceReader( - path: File, - performanceInterferenceModel: PerformanceInterferenceModel, - vms: List, - seed: Int -): Sc20StreamingParquetTraceReader { - return Sc20StreamingParquetTraceReader( - path, - performanceInterferenceModel, - vms, - Random(seed) - ) -} - /** * Construct the environment for a simulated compute service.. */ -public suspend fun withComputeService( +suspend fun withComputeService( clock: Clock, meterProvider: MeterProvider, environmentReader: EnvironmentReader, @@ -182,15 +162,13 @@ public suspend fun withComputeService( * Attach the specified monitor to the VM provisioner. */ @OptIn(ExperimentalCoroutinesApi::class) -public suspend fun withMonitor( +suspend fun withMonitor( monitor: ExperimentMonitor, clock: Clock, metricProducer: MetricProducer, scheduler: ComputeService, block: suspend CoroutineScope.() -> Unit ): Unit = coroutineScope { - val monitorJobs = mutableSetOf() - // Monitor host events for (host in scheduler.hosts) { monitor.reportHostStateChange(clock.millis(), host, HostState.UP) @@ -211,24 +189,23 @@ public suspend fun withMonitor( try { block(this) } finally { - monitorJobs.forEach(Job::cancel) reader.close() monitor.close() } } -public class ComputeMetrics { - public var submittedVms: Int = 0 - public var queuedVms: Int = 0 - public var runningVms: Int = 0 - public var unscheduledVms: Int = 0 - public var finishedVms: Int = 0 +class ComputeMetrics { + var submittedVms: Int = 0 + var queuedVms: Int = 0 + var runningVms: Int = 0 + var unscheduledVms: Int = 0 + var finishedVms: Int = 0 } /** * Collect the metrics of the compute service. */ -public fun collectMetrics(metricProducer: MetricProducer): ComputeMetrics { +fun collectMetrics(metricProducer: MetricProducer): ComputeMetrics { val metrics = metricProducer.collectAllMetrics().associateBy { it.name } val res = ComputeMetrics() try { @@ -247,7 +224,7 @@ public fun collectMetrics(metricProducer: MetricProducer): ComputeMetrics { /** * Process the trace. */ -public suspend fun processTrace( +suspend fun processTrace( clock: Clock, reader: TraceReader, scheduler: ComputeService, @@ -306,7 +283,7 @@ public suspend fun processTrace( /** * Create a [MeterProvider] instance for the experiment. */ -public fun createMeterProvider(clock: Clock): MeterProvider { +fun createMeterProvider(clock: Clock): MeterProvider { val powerSelector = InstrumentSelector.builder() .setInstrumentNameRegex("power\\.usage") .setInstrumentType(InstrumentType.VALUE_RECORDER) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index b70eefb2..460da303 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -32,29 +32,27 @@ import org.opendc.compute.service.scheduler.* import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.weights.* +import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor -import org.opendc.experiments.capelin.trace.Sc20ParquetTraceReader -import org.opendc.experiments.capelin.trace.Sc20RawParquetTraceReader -import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader -import org.opendc.format.trace.PerformanceInterferenceModelReader +import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.core.runBlockingSimulation import java.io.File import java.util.* import java.util.concurrent.ConcurrentHashMap -import kotlin.random.asKotlinRandom /** * A portfolio represents a collection of scenarios are tested for the work. * * @param name The name of the portfolio. */ -public abstract class Portfolio(name: String) : Experiment(name) { +abstract class Portfolio(name: String) : Experiment(name) { /** * The logger for this portfolio instance. */ @@ -70,35 +68,30 @@ public abstract class Portfolio(name: String) : Experiment(name) { */ private val vmPlacements by anyOf(emptyMap()) - /** - * The path to the performance interference model. - */ - private val performanceInterferenceModel by anyOf(null) - /** * The topology to test. */ - public abstract val topology: Topology + abstract val topology: Topology /** * The workload to test. */ - public abstract val workload: Workload + abstract val workload: Workload /** * The operational phenomenas to consider. */ - public abstract val operationalPhenomena: OperationalPhenomena + abstract val operationalPhenomena: OperationalPhenomena /** * The allocation policies to consider. */ - public abstract val allocationPolicy: String + abstract val allocationPolicy: String /** * A map of trace readers. */ - private val traceReaders = ConcurrentHashMap() + private val traceReaders = ConcurrentHashMap() /** * Perform a single trial for this portfolio. @@ -106,7 +99,7 @@ public abstract class Portfolio(name: String) : Experiment(name) { @OptIn(ExperimentalCoroutinesApi::class) override fun doRun(repeat: Int): Unit = runBlockingSimulation { val seeder = Random(repeat.toLong()) - val environment = Sc20ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) + val environment = ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) val chan = Channel(Channel.CONFLATED) val allocationPolicy = createComputeScheduler(seeder) @@ -122,14 +115,11 @@ public abstract class Portfolio(name: String) : Experiment(name) { val rawReaders = workloadNames.map { workloadName -> traceReaders.computeIfAbsent(workloadName) { logger.info { "Loading trace $workloadName" } - Sc20RawParquetTraceReader(File(config.getString("trace-path"), workloadName)) + RawParquetTraceReader(File(config.getString("trace-path"), workloadName)) } } - val performanceInterferenceModel = performanceInterferenceModel - ?.takeIf { operationalPhenomena.hasInterference } - ?.construct(seeder.asKotlinRandom()) ?: emptyMap() - val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, workload, seeder.nextInt()) + val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) val monitor = ParquetExperimentMonitor( File(config.getString("output-path")), diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt new file mode 100644 index 00000000..d73d14f5 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.env + +import org.opendc.format.environment.EnvironmentReader +import org.opendc.format.environment.MachineDef +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit +import org.opendc.simulator.compute.power.LinearPowerModel +import java.io.File +import java.io.FileInputStream +import java.io.InputStream +import java.util.* + +/** + * A [EnvironmentReader] for the internal environment format. + * + * @param input The input stream describing the physical cluster. + */ +class ClusterEnvironmentReader(private val input: InputStream) : EnvironmentReader { + /** + * Construct a [ClusterEnvironmentReader] for the specified [file]. + */ + constructor(file: File) : this(FileInputStream(file)) + + override fun read(): List { + var clusterIdCol = 0 + var speedCol = 0 + var numberOfHostsCol = 0 + var memoryPerHostCol = 0 + var coresPerHostCol = 0 + + var clusterIdx = 0 + var clusterId: String + var speed: Double + var numberOfHosts: Int + var memoryPerHost: Long + var coresPerHost: Int + + val nodes = mutableListOf() + val random = Random(0) + + input.bufferedReader().use { reader -> + reader.lineSequence() + .filter { line -> + // Ignore comments in the file + !line.startsWith("#") && line.isNotBlank() + } + .forEachIndexed { idx, line -> + val values = line.split(";") + + if (idx == 0) { + val header = values.mapIndexed { col, name -> Pair(name.trim(), col) }.toMap() + clusterIdCol = header["ClusterID"]!! + speedCol = header["Speed"]!! + numberOfHostsCol = header["numberOfHosts"]!! + memoryPerHostCol = header["memoryCapacityPerHost"]!! + coresPerHostCol = header["coreCountPerHost"]!! + return@forEachIndexed + } + + clusterIdx++ + clusterId = values[clusterIdCol].trim() + speed = values[speedCol].trim().toDouble() * 1000.0 + numberOfHosts = values[numberOfHostsCol].trim().toInt() + memoryPerHost = values[memoryPerHostCol].trim().toLong() * 1000L + coresPerHost = values[coresPerHostCol].trim().toInt() + + val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", coresPerHost) + val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) + + repeat(numberOfHosts) { + nodes.add( + MachineDef( + UUID(random.nextLong(), random.nextLong()), + "node-$clusterId-$it", + mapOf("cluster" to clusterId), + MachineModel( + List(coresPerHost) { coreId -> + ProcessingUnit(unknownProcessingNode, coreId, speed) + }, + listOf(unknownMemoryUnit) + ), + // For now we assume a simple linear load model with an idle draw of ~200W and a maximum + // power draw of 350W. + // Source: https://stackoverflow.com/questions/6128960 + LinearPowerModel(350.0, idlePower = 200.0) + ) + ) + } + } + } + + return nodes + } + + override fun close() { + input.close() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt new file mode 100644 index 00000000..2ebe65ea --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import org.opendc.experiments.capelin.model.CompositeWorkload +import org.opendc.experiments.capelin.model.Workload +import org.opendc.format.trace.TraceEntry +import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.workload.SimWorkload + +/** + * A [TraceReader] for the internal VM workload trace format. + * + * @param rawReaders The raw trace readers to use.. + * @param workload The workload to use. + * @param seed The seed to use for workload sampling. + */ +class ParquetTraceReader( + rawReaders: List, + workload: Workload, + seed: Int +) : TraceReader { + /** + * The iterator over the actual trace. + */ + private val iterator: Iterator> = + rawReaders + .map { it.read() } + .run { + if (workload is CompositeWorkload) { + this.zip(workload.workloads) + } else { + this.zip(listOf(workload)) + } + } + .map { sampleWorkload(it.first, workload, it.second, seed) } + .flatten() + .iterator() + + override fun hasNext(): Boolean = iterator.hasNext() + + override fun next(): TraceEntry = iterator.next() + + override fun close() {} +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt new file mode 100644 index 00000000..94193780 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import org.apache.avro.generic.GenericData +import org.opendc.format.trace.TraceEntry +import org.opendc.format.trace.TraceReader +import org.opendc.format.util.LocalParquetReader +import org.opendc.simulator.compute.workload.SimTraceWorkload +import org.opendc.simulator.compute.workload.SimWorkload +import java.io.File +import java.util.UUID + +/** + * A [TraceReader] for the internal VM workload trace format. + * + * @param path The directory of the traces. + */ +class RawParquetTraceReader(private val path: File) { + /** + * Read the fragments into memory. + */ + private fun parseFragments(path: File): Map> { + val reader = LocalParquetReader(File(path, "trace.parquet")) + + val fragments = mutableMapOf>() + + return try { + while (true) { + val record = reader.read() ?: break + + val id = record["id"].toString() + val duration = record["duration"] as Long + val cores = record["cores"] as Int + val cpuUsage = record["cpuUsage"] as Double + + val fragment = SimTraceWorkload.Fragment( + duration, + cpuUsage, + cores + ) + + fragments.getOrPut(id) { mutableListOf() }.add(fragment) + } + + fragments + } finally { + reader.close() + } + } + + /** + * Read the metadata into a workload. + */ + private fun parseMeta(path: File, fragments: Map>): List> { + val metaReader = LocalParquetReader(File(path, "meta.parquet")) + + var counter = 0 + val entries = mutableListOf>() + + return try { + while (true) { + val record = metaReader.read() ?: break + + val id = record["id"].toString() + if (!fragments.containsKey(id)) { + continue + } + + val submissionTime = record["submissionTime"] as Long + val endTime = record["endTime"] as Long + val maxCores = record["maxCores"] as Int + val requiredMemory = record["requiredMemory"] as Long + val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) + + val vmFragments = fragments.getValue(id).asSequence() + val totalLoad = vmFragments.sumOf { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs + val workload = SimTraceWorkload(vmFragments) + entries.add( + TraceEntry( + uid, id, submissionTime, workload, + mapOf( + "submit-time" to submissionTime, + "end-time" to endTime, + "total-load" to totalLoad, + "cores" to maxCores, + "required-memory" to requiredMemory, + "workload" to workload + ) + ) + ) + } + + entries + } catch (e: Exception) { + e.printStackTrace() + throw e + } finally { + metaReader.close() + } + } + + /** + * The entries in the trace. + */ + private val entries: List> + + init { + val fragments = parseFragments(path) + entries = parseMeta(path, fragments) + } + + /** + * Read the entries in the trace. + */ + fun read(): List> = entries +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt deleted file mode 100644 index 7f25137e..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20ParquetTraceReader.kt +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import org.opendc.experiments.capelin.model.CompositeWorkload -import org.opendc.experiments.capelin.model.Workload -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader -import org.opendc.simulator.compute.interference.IMAGE_PERF_INTERFERENCE_MODEL -import org.opendc.simulator.compute.interference.PerformanceInterferenceModel -import org.opendc.simulator.compute.workload.SimWorkload -import java.util.TreeSet - -/** - * A [TraceReader] for the internal VM workload trace format. - * - * @param reader The internal trace reader to use. - * @param performanceInterferenceModel The performance model covering the workload in the VM trace. - * @param run The run to which this reader belongs. - */ -public class Sc20ParquetTraceReader( - rawReaders: List, - performanceInterferenceModel: Map, - workload: Workload, - seed: Int -) : TraceReader { - /** - * The iterator over the actual trace. - */ - private val iterator: Iterator> = - rawReaders - .map { it.read() } - .run { - if (workload is CompositeWorkload) { - this.zip(workload.workloads) - } else { - this.zip(listOf(workload)) - } - } - .map { sampleWorkload(it.first, workload, it.second, seed) } - .flatten() - .run { - // Apply performance interference model - if (performanceInterferenceModel.isEmpty()) - this - else { - map { entry -> - val id = entry.name - val relevantPerformanceInterferenceModelItems = - performanceInterferenceModel[id] ?: PerformanceInterferenceModel(TreeSet()) - - entry.copy(meta = entry.meta + mapOf(IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems)) - } - } - } - .iterator() - - override fun hasNext(): Boolean = iterator.hasNext() - - override fun next(): TraceEntry = iterator.next() - - override fun close() {} -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt deleted file mode 100644 index 54151c9f..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20RawParquetTraceReader.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import mu.KotlinLogging -import org.apache.avro.generic.GenericData -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader -import org.opendc.format.util.LocalParquetReader -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import java.io.File -import java.util.UUID - -private val logger = KotlinLogging.logger {} - -/** - * A [TraceReader] for the internal VM workload trace format. - * - * @param path The directory of the traces. - */ -public class Sc20RawParquetTraceReader(private val path: File) { - /** - * Read the fragments into memory. - */ - private fun parseFragments(path: File): Map> { - val reader = LocalParquetReader(File(path, "trace.parquet")) - - val fragments = mutableMapOf>() - - return try { - while (true) { - val record = reader.read() ?: break - - val id = record["id"].toString() - val duration = record["duration"] as Long - val cores = record["cores"] as Int - val cpuUsage = record["cpuUsage"] as Double - - val fragment = SimTraceWorkload.Fragment( - duration, - cpuUsage, - cores - ) - - fragments.getOrPut(id) { mutableListOf() }.add(fragment) - } - - fragments - } finally { - reader.close() - } - } - - /** - * Read the metadata into a workload. - */ - private fun parseMeta(path: File, fragments: Map>): List> { - val metaReader = LocalParquetReader(File(path, "meta.parquet")) - - var counter = 0 - val entries = mutableListOf>() - - return try { - while (true) { - val record = metaReader.read() ?: break - - val id = record["id"].toString() - if (!fragments.containsKey(id)) { - continue - } - - val submissionTime = record["submissionTime"] as Long - val endTime = record["endTime"] as Long - val maxCores = record["maxCores"] as Int - val requiredMemory = record["requiredMemory"] as Long - val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) - - val vmFragments = fragments.getValue(id).asSequence() - val totalLoad = vmFragments.sumOf { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs - val workload = SimTraceWorkload(vmFragments) - entries.add( - TraceEntry( - uid, id, submissionTime, workload, - mapOf( - "submit-time" to submissionTime, - "end-time" to endTime, - "total-load" to totalLoad, - "cores" to maxCores, - "required-memory" to requiredMemory, - "workload" to workload - ) - ) - ) - } - - entries - } catch (e: Exception) { - e.printStackTrace() - throw e - } finally { - metaReader.close() - } - } - - /** - * The entries in the trace. - */ - private val entries: List> - - init { - val fragments = parseFragments(path) - entries = parseMeta(path, fragments) - } - - /** - * Read the entries in the trace. - */ - public fun read(): List> = entries - - /** - * Create a [TraceReader] instance. - */ - public fun createReader(): TraceReader { - return object : TraceReader, Iterator> by entries.iterator() { - override fun close() {} - } - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt deleted file mode 100644 index 6792c2ab..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20StreamingParquetTraceReader.kt +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import mu.KotlinLogging -import org.apache.avro.generic.GenericData -import org.apache.parquet.avro.AvroParquetReader -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.filter2.predicate.Statistics -import org.apache.parquet.filter2.predicate.UserDefinedPredicate -import org.apache.parquet.io.api.Binary -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader -import org.opendc.format.util.LocalInputFile -import org.opendc.simulator.compute.interference.IMAGE_PERF_INTERFERENCE_MODEL -import org.opendc.simulator.compute.interference.PerformanceInterferenceModel -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import java.io.File -import java.io.Serializable -import java.util.SortedSet -import java.util.TreeSet -import java.util.UUID -import java.util.concurrent.ArrayBlockingQueue -import kotlin.concurrent.thread -import kotlin.random.Random - -private val logger = KotlinLogging.logger {} - -/** - * A [TraceReader] for the internal VM workload trace format that streams workloads on the fly. - * - * @param traceFile The directory of the traces. - * @param performanceInterferenceModel The performance model covering the workload in the VM trace. - */ -public class Sc20StreamingParquetTraceReader( - traceFile: File, - performanceInterferenceModel: PerformanceInterferenceModel? = null, - selectedVms: List = emptyList(), - random: Random -) : TraceReader { - /** - * The internal iterator to use for this reader. - */ - private val iterator: Iterator> - - /** - * The intermediate buffer to store the read records in. - */ - private val queue = ArrayBlockingQueue>(1024) - - /** - * An optional filter for filtering the selected VMs - */ - private val filter = - if (selectedVms.isEmpty()) - null - else - FilterCompat.get( - FilterApi.userDefined( - FilterApi.binaryColumn("id"), - SelectedVmFilter( - TreeSet(selectedVms) - ) - ) - ) - - /** - * A poisonous fragment. - */ - private val poison = Pair("\u0000", SimTraceWorkload.Fragment(0, 0.0, 0)) - - /** - * The thread to read the records in. - */ - private val readerThread = thread(start = true, name = "sc20-reader") { - val reader = AvroParquetReader - .builder(LocalInputFile(File(traceFile, "trace.parquet"))) - .disableCompatibility() - .withFilter(filter) - .build() - - try { - while (true) { - val record = reader.read() - - if (record == null) { - queue.put(poison) - break - } - - val id = record["id"].toString() - val duration = record["duration"] as Long - val cores = record["cores"] as Int - val cpuUsage = record["cpuUsage"] as Double - - val fragment = SimTraceWorkload.Fragment( - duration, - cpuUsage, - cores - ) - - queue.put(id to fragment) - } - } catch (e: InterruptedException) { - // Do not rethrow this - } finally { - reader.close() - } - } - - /** - * Fill the buffers with the VMs - */ - private fun pull(buffers: Map>>) { - if (!hasNext) { - return - } - - val fragments = mutableListOf>() - queue.drainTo(fragments) - - for ((id, fragment) in fragments) { - if (id == poison.first) { - hasNext = false - return - } - buffers[id]?.forEach { it.add(fragment) } - } - } - - /** - * A flag to indicate whether the reader has more entries. - */ - private var hasNext: Boolean = true - - /** - * Initialize the reader. - */ - init { - val takenIds = mutableSetOf() - val entries = mutableMapOf() - val buffers = mutableMapOf>>() - - val metaReader = AvroParquetReader - .builder(LocalInputFile(File(traceFile, "meta.parquet"))) - .disableCompatibility() - .withFilter(filter) - .build() - - while (true) { - val record = metaReader.read() ?: break - val id = record["id"].toString() - entries[id] = record - } - - metaReader.close() - - val selection = selectedVms.ifEmpty { entries.keys } - - // Create the entry iterator - iterator = selection.asSequence() - .mapNotNull { entries[it] } - .mapIndexed { index, record -> - val id = record["id"].toString() - val submissionTime = record["submissionTime"] as Long - val endTime = record["endTime"] as Long - val maxCores = record["maxCores"] as Int - val requiredMemory = record["requiredMemory"] as Long - val uid = UUID.nameUUIDFromBytes("$id-$index".toByteArray()) - - assert(uid !in takenIds) - takenIds += uid - - logger.info("Processing VM $id") - - val internalBuffer = mutableListOf() - val externalBuffer = mutableListOf() - buffers.getOrPut(id) { mutableListOf() }.add(externalBuffer) - val fragments = sequence { - var time = submissionTime - repeat@ while (true) { - if (externalBuffer.isEmpty()) { - if (hasNext) { - pull(buffers) - continue - } else { - break - } - } - - internalBuffer.addAll(externalBuffer) - externalBuffer.clear() - - for (fragment in internalBuffer) { - yield(fragment) - - time += fragment.duration - if (time >= endTime) { - break@repeat - } - } - - internalBuffer.clear() - } - - buffers.remove(id) - } - val relevantPerformanceInterferenceModelItems = - if (performanceInterferenceModel != null) - PerformanceInterferenceModel( - performanceInterferenceModel.items.filter { it.workloadNames.contains(id) }.toSortedSet(), - Random(random.nextInt()) - ) - else - null - val workload = SimTraceWorkload(fragments) - val meta = mapOf( - "cores" to maxCores, - "required-memory" to requiredMemory, - "workload" to workload - ) - - TraceEntry( - uid, id, submissionTime, workload, - if (performanceInterferenceModel != null) - meta + mapOf(IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems as Any) - else - meta - ) - } - .sortedBy { it.start } - .toList() - .iterator() - } - - override fun hasNext(): Boolean = iterator.hasNext() - - override fun next(): TraceEntry = iterator.next() - - override fun close() { - readerThread.interrupt() - } - - private class SelectedVmFilter(val selectedVms: SortedSet) : UserDefinedPredicate(), Serializable { - override fun keep(value: Binary?): Boolean = value != null && selectedVms.contains(value.toStringUsingUTF8()) - - override fun canDrop(statistics: Statistics): Boolean { - val min = statistics.min - val max = statistics.max - - return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isEmpty() - } - - override fun inverseCanDrop(statistics: Statistics): Boolean { - val min = statistics.min - val max = statistics.max - - return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isNotEmpty() - } - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt deleted file mode 100644 index d0031a66..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/Sc20TraceConverter.kt +++ /dev/null @@ -1,621 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.arguments.argument -import com.github.ajalt.clikt.parameters.groups.OptionGroup -import com.github.ajalt.clikt.parameters.groups.groupChoice -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.defaultLazy -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.options.split -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.long -import me.tongfei.progressbar.ProgressBar -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetWriter -import org.apache.parquet.hadoop.metadata.CompressionCodecName -import org.opendc.format.trace.sc20.Sc20VmPlacementReader -import org.opendc.format.util.LocalOutputFile -import java.io.BufferedReader -import java.io.File -import java.io.FileReader -import java.util.Random -import kotlin.math.max -import kotlin.math.min - -/** - * Represents the command for converting traces - */ -public class TraceConverterCli : CliktCommand(name = "trace-converter") { - /** - * The directory where the trace should be stored. - */ - private val outputPath by option("-O", "--output", help = "path to store the trace") - .file(canBeFile = false, mustExist = false) - .defaultLazy { File("output") } - - /** - * The directory where the input trace is located. - */ - private val inputPath by argument("input", help = "path to the input trace") - .file(canBeFile = false) - - /** - * The input type of the trace. - */ - private val type by option("-t", "--type", help = "input type of trace").groupChoice( - "solvinity" to SolvinityConversion(), - "bitbrains" to BitbrainsConversion(), - "azure" to AzureConversion() - ) - - override fun run() { - val metaSchema = SchemaBuilder - .record("meta") - .namespace("org.opendc.format.sc20") - .fields() - .name("id").type().stringType().noDefault() - .name("submissionTime").type().longType().noDefault() - .name("endTime").type().longType().noDefault() - .name("maxCores").type().intType().noDefault() - .name("requiredMemory").type().longType().noDefault() - .endRecord() - val schema = SchemaBuilder - .record("trace") - .namespace("org.opendc.format.sc20") - .fields() - .name("id").type().stringType().noDefault() - .name("time").type().longType().noDefault() - .name("duration").type().longType().noDefault() - .name("cores").type().intType().noDefault() - .name("cpuUsage").type().doubleType().noDefault() - .name("flops").type().longType().noDefault() - .endRecord() - - val metaParquet = File(outputPath, "meta.parquet") - val traceParquet = File(outputPath, "trace.parquet") - - if (metaParquet.exists()) { - metaParquet.delete() - } - if (traceParquet.exists()) { - traceParquet.delete() - } - - val metaWriter = AvroParquetWriter.builder(LocalOutputFile(metaParquet)) - .withSchema(metaSchema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) - .build() - - val writer = AvroParquetWriter.builder(LocalOutputFile(traceParquet)) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) - .build() - - try { - val type = type ?: throw IllegalArgumentException("Invalid trace conversion") - val allFragments = type.read(inputPath, metaSchema, metaWriter) - allFragments.sortWith(compareBy { it.tick }.thenBy { it.id }) - - for (fragment in allFragments) { - val record = GenericData.Record(schema) - record.put("id", fragment.id) - record.put("time", fragment.tick) - record.put("duration", fragment.duration) - record.put("cores", fragment.cores) - record.put("cpuUsage", fragment.usage) - record.put("flops", fragment.flops) - - writer.write(record) - } - } finally { - writer.close() - metaWriter.close() - } - } -} - -/** - * The supported trace conversions. - */ -public sealed class TraceConversion(name: String) : OptionGroup(name) { - /** - * Read the fragments of the trace. - */ - public abstract fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList -} - -public class SolvinityConversion : TraceConversion("Solvinity") { - private val clusters by option() - .split(",") - - private val vmPlacements by option("--vm-placements", help = "file containing the VM placements") - .file(canBeDir = false) - .convert { it.inputStream().buffered().use { Sc20VmPlacementReader(it).construct() } } - .required() - - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val clusters = clusters?.toSet() ?: emptySet() - val timestampCol = 0 - val cpuUsageCol = 1 - val coreCol = 12 - val provisionedMemoryCol = 20 - val traceInterval = 5 * 60 * 1000L - - // Identify start time of the entire trace - var minTimestamp = Long.MAX_VALUE - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach file@{ vmFile -> - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val vmId = vmFile.name - - // Check if VM in topology - val clusterName = vmPlacements[vmId] - if (clusterName == null || !clusters.contains(clusterName)) { - continue - } - - val values = line.split("\t") - val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - - if (timestamp < minTimestamp) { - minTimestamp = timestamp - } - return@file - } - } - } - } - - println("Start of trace at $minTimestamp") - - val allFragments = mutableListOf() - - val begin = 15 * 24 * 60 * 60 * 1000L - val end = 45 * 24 * 60 * 60 * 1000L - - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach { vmFile -> - println(vmFile) - - var vmId = "" - var maxCores = -1 - var requiredMemory = -1L - var cores: Int - var minTime = Long.MAX_VALUE - - val flopsFragments = sequence { - var last: Fragment? = null - - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split("\t") - - vmId = vmFile.name - - // Check if VM in topology - val clusterName = vmPlacements[vmId] - if (clusterName == null || !clusters.contains(clusterName)) { - continue - } - - val timestamp = - (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - minTimestamp - if (begin > timestamp || timestamp > end) { - continue - } - - cores = values[coreCol].trim().toInt() - requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) - maxCores = max(maxCores, cores) - minTime = min(minTime, timestamp) - val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz - requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) - maxCores = max(maxCores, cores) - - val flops: Long = (cpuUsage * 5 * 60).toLong() - - last = if (last != null && last!!.flops == 0L && flops == 0L) { - val oldFragment = last!! - Fragment( - vmId, - oldFragment.tick, - oldFragment.flops + flops, - oldFragment.duration + traceInterval, - cpuUsage, - cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - cores - ) - if (last != null) { - yield(last!!) - } - fragment - } - } - } - } - - if (last != null) { - yield(last!!) - } - } - - var maxTime = Long.MIN_VALUE - flopsFragments.filter { it.tick in begin until end }.forEach { fragment -> - allFragments.add(fragment) - maxTime = max(maxTime, fragment.tick) - } - - if (minTime in begin until end) { - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", vmId) - metaRecord.put("submissionTime", minTime) - metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", maxCores) - metaRecord.put("requiredMemory", requiredMemory) - metaWriter.write(metaRecord) - } - } - - return allFragments - } -} - -/** - * Conversion of the Bitbrains public trace. - */ -public class BitbrainsConversion : TraceConversion("Bitbrains") { - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val timestampCol = 0 - val cpuUsageCol = 3 - val coreCol = 1 - val provisionedMemoryCol = 5 - val traceInterval = 5 * 60 * 1000L - - val allFragments = mutableListOf() - - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach { vmFile -> - println(vmFile) - - var vmId = "" - var maxCores = -1 - var requiredMemory = -1L - var cores: Int - var minTime = Long.MAX_VALUE - - val flopsFragments = sequence { - var last: Fragment? = null - - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .drop(1) - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split(";\t") - - vmId = vmFile.name - - val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - - cores = values[coreCol].trim().toInt() - val provisionedMemory = values[provisionedMemoryCol].trim().toDouble() // KB - requiredMemory = max(requiredMemory, (provisionedMemory / 1000).toLong()) - maxCores = max(maxCores, cores) - minTime = min(minTime, timestamp) - val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz - - val flops: Long = (cpuUsage * 5 * 60).toLong() - - last = if (last != null && last!!.flops == 0L && flops == 0L) { - val oldFragment = last!! - Fragment( - vmId, - oldFragment.tick, - oldFragment.flops + flops, - oldFragment.duration + traceInterval, - cpuUsage, - cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - cores - ) - if (last != null) { - yield(last!!) - } - fragment - } - } - } - } - - if (last != null) { - yield(last!!) - } - } - - var maxTime = Long.MIN_VALUE - flopsFragments.forEach { fragment -> - allFragments.add(fragment) - maxTime = max(maxTime, fragment.tick) - } - - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", vmId) - metaRecord.put("submissionTime", minTime) - metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", maxCores) - metaRecord.put("requiredMemory", requiredMemory) - metaWriter.write(metaRecord) - } - - return allFragments - } -} - -/** - * Conversion of the Azure public VM trace. - */ -public class AzureConversion : TraceConversion("Azure") { - private val seed by option(help = "seed for trace sampling") - .long() - .default(0) - - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val random = Random(seed) - val fraction = 0.01 - - // Read VM table - val vmIdTableCol = 0 - val coreTableCol = 9 - val provisionedMemoryTableCol = 10 - - var vmId: String - var cores: Int - var requiredMemory: Long - - val vmIds = mutableSetOf() - val vmIdToMetadata = mutableMapOf() - - BufferedReader(FileReader(File(traceDirectory, "vmtable.csv"))).use { reader -> - reader.lineSequence() - .chunked(1024) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - // Sample only a fraction of the VMs - if (random.nextDouble() > fraction) { - continue - } - - val values = line.split(",") - - // Exclude VMs with a large number of cores (not specified exactly) - if (values[coreTableCol].contains(">")) { - continue - } - - vmId = values[vmIdTableCol].trim() - cores = values[coreTableCol].trim().toInt() - requiredMemory = values[provisionedMemoryTableCol].trim().toInt() * 1_000L // GB -> MB - - vmIds.add(vmId) - vmIdToMetadata[vmId] = VmInfo(cores, requiredMemory, Long.MAX_VALUE, -1L) - } - } - } - - // Read VM metric reading files - val timestampCol = 0 - val vmIdCol = 1 - val cpuUsageCol = 4 - val traceInterval = 5 * 60 * 1000L - - val vmIdToFragments = mutableMapOf>() - val vmIdToLastFragment = mutableMapOf() - val allFragments = mutableListOf() - - for (i in ProgressBar.wrap((1..195).toList(), "Reading Trace")) { - val readingsFile = File(File(traceDirectory, "readings"), "readings-$i.csv") - var timestamp: Long - var cpuUsage: Double - - BufferedReader(FileReader(readingsFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split(",") - vmId = values[vmIdCol].trim() - - // Ignore readings for VMs not in the sample - if (!vmIds.contains(vmId)) { - continue - } - - timestamp = values[timestampCol].trim().toLong() * 1000L - vmIdToMetadata[vmId]!!.minTime = min(vmIdToMetadata[vmId]!!.minTime, timestamp) - cpuUsage = values[cpuUsageCol].trim().toDouble() * 3_000 // MHz - vmIdToMetadata[vmId]!!.maxTime = max(vmIdToMetadata[vmId]!!.maxTime, timestamp) - - val flops: Long = (cpuUsage * 5 * 60).toLong() - val lastFragment = vmIdToLastFragment[vmId] - - vmIdToLastFragment[vmId] = - if (lastFragment != null && lastFragment.flops == 0L && flops == 0L) { - Fragment( - vmId, - lastFragment.tick, - lastFragment.flops + flops, - lastFragment.duration + traceInterval, - cpuUsage, - vmIdToMetadata[vmId]!!.cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - vmIdToMetadata[vmId]!!.cores - ) - if (lastFragment != null) { - if (vmIdToFragments[vmId] == null) { - vmIdToFragments[vmId] = mutableListOf() - } - vmIdToFragments[vmId]!!.add(lastFragment) - allFragments.add(lastFragment) - } - fragment - } - } - } - } - } - - for (entry in vmIdToLastFragment) { - if (entry.value != null) { - if (vmIdToFragments[entry.key] == null) { - vmIdToFragments[entry.key] = mutableListOf() - } - vmIdToFragments[entry.key]!!.add(entry.value!!) - } - } - - println("Read ${vmIdToLastFragment.size} VMs") - - for (entry in vmIdToMetadata) { - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", entry.key) - metaRecord.put("submissionTime", entry.value.minTime) - metaRecord.put("endTime", entry.value.maxTime) - println("${entry.value.minTime} - ${entry.value.maxTime}") - metaRecord.put("maxCores", entry.value.cores) - metaRecord.put("requiredMemory", entry.value.requiredMemory) - metaWriter.write(metaRecord) - } - - return allFragments - } -} - -public data class Fragment( - public val id: String, - public val tick: Long, - public val flops: Long, - public val duration: Long, - public val usage: Double, - public val cores: Int -) - -public class VmInfo(public val cores: Int, public val requiredMemory: Long, public var minTime: Long, public var maxTime: Long) - -/** - * A script to convert a trace in text format into a Parquet trace. - */ -public fun main(args: Array): Unit = TraceConverterCli().main(args) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt new file mode 100644 index 00000000..a3b45f47 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import mu.KotlinLogging +import org.apache.avro.generic.GenericData +import org.apache.parquet.avro.AvroParquetReader +import org.apache.parquet.filter2.compat.FilterCompat +import org.apache.parquet.filter2.predicate.FilterApi +import org.apache.parquet.filter2.predicate.Statistics +import org.apache.parquet.filter2.predicate.UserDefinedPredicate +import org.apache.parquet.io.api.Binary +import org.opendc.format.trace.TraceEntry +import org.opendc.format.trace.TraceReader +import org.opendc.format.util.LocalInputFile +import org.opendc.simulator.compute.workload.SimTraceWorkload +import org.opendc.simulator.compute.workload.SimWorkload +import java.io.File +import java.io.Serializable +import java.util.SortedSet +import java.util.TreeSet +import java.util.UUID +import java.util.concurrent.ArrayBlockingQueue +import kotlin.concurrent.thread + +private val logger = KotlinLogging.logger {} + +/** + * A [TraceReader] for the internal VM workload trace format that streams workloads on the fly. + * + * @param traceFile The directory of the traces. + * @param selectedVms The list of VMs to read from the trace. + */ +class StreamingParquetTraceReader(traceFile: File, selectedVms: List = emptyList()) : TraceReader { + /** + * The internal iterator to use for this reader. + */ + private val iterator: Iterator> + + /** + * The intermediate buffer to store the read records in. + */ + private val queue = ArrayBlockingQueue>(1024) + + /** + * An optional filter for filtering the selected VMs + */ + private val filter = + if (selectedVms.isEmpty()) + null + else + FilterCompat.get( + FilterApi.userDefined( + FilterApi.binaryColumn("id"), + SelectedVmFilter( + TreeSet(selectedVms) + ) + ) + ) + + /** + * A poisonous fragment. + */ + private val poison = Pair("\u0000", SimTraceWorkload.Fragment(0, 0.0, 0)) + + /** + * The thread to read the records in. + */ + private val readerThread = thread(start = true, name = "sc20-reader") { + val reader = AvroParquetReader + .builder(LocalInputFile(File(traceFile, "trace.parquet"))) + .disableCompatibility() + .withFilter(filter) + .build() + + try { + while (true) { + val record = reader.read() + + if (record == null) { + queue.put(poison) + break + } + + val id = record["id"].toString() + val duration = record["duration"] as Long + val cores = record["cores"] as Int + val cpuUsage = record["cpuUsage"] as Double + + val fragment = SimTraceWorkload.Fragment( + duration, + cpuUsage, + cores + ) + + queue.put(id to fragment) + } + } catch (e: InterruptedException) { + // Do not rethrow this + } finally { + reader.close() + } + } + + /** + * Fill the buffers with the VMs + */ + private fun pull(buffers: Map>>) { + if (!hasNext) { + return + } + + val fragments = mutableListOf>() + queue.drainTo(fragments) + + for ((id, fragment) in fragments) { + if (id == poison.first) { + hasNext = false + return + } + buffers[id]?.forEach { it.add(fragment) } + } + } + + /** + * A flag to indicate whether the reader has more entries. + */ + private var hasNext: Boolean = true + + /** + * Initialize the reader. + */ + init { + val takenIds = mutableSetOf() + val entries = mutableMapOf() + val buffers = mutableMapOf>>() + + val metaReader = AvroParquetReader + .builder(LocalInputFile(File(traceFile, "meta.parquet"))) + .disableCompatibility() + .withFilter(filter) + .build() + + while (true) { + val record = metaReader.read() ?: break + val id = record["id"].toString() + entries[id] = record + } + + metaReader.close() + + val selection = selectedVms.ifEmpty { entries.keys } + + // Create the entry iterator + iterator = selection.asSequence() + .mapNotNull { entries[it] } + .mapIndexed { index, record -> + val id = record["id"].toString() + val submissionTime = record["submissionTime"] as Long + val endTime = record["endTime"] as Long + val maxCores = record["maxCores"] as Int + val requiredMemory = record["requiredMemory"] as Long + val uid = UUID.nameUUIDFromBytes("$id-$index".toByteArray()) + + assert(uid !in takenIds) + takenIds += uid + + logger.info("Processing VM $id") + + val internalBuffer = mutableListOf() + val externalBuffer = mutableListOf() + buffers.getOrPut(id) { mutableListOf() }.add(externalBuffer) + val fragments = sequence { + var time = submissionTime + repeat@ while (true) { + if (externalBuffer.isEmpty()) { + if (hasNext) { + pull(buffers) + continue + } else { + break + } + } + + internalBuffer.addAll(externalBuffer) + externalBuffer.clear() + + for (fragment in internalBuffer) { + yield(fragment) + + time += fragment.duration + if (time >= endTime) { + break@repeat + } + } + + internalBuffer.clear() + } + + buffers.remove(id) + } + val workload = SimTraceWorkload(fragments) + val meta = mapOf( + "cores" to maxCores, + "required-memory" to requiredMemory, + "workload" to workload + ) + + TraceEntry(uid, id, submissionTime, workload, meta) + } + .sortedBy { it.start } + .toList() + .iterator() + } + + override fun hasNext(): Boolean = iterator.hasNext() + + override fun next(): TraceEntry = iterator.next() + + override fun close() { + readerThread.interrupt() + } + + private class SelectedVmFilter(val selectedVms: SortedSet) : UserDefinedPredicate(), Serializable { + override fun keep(value: Binary?): Boolean = value != null && selectedVms.contains(value.toStringUsingUTF8()) + + override fun canDrop(statistics: Statistics): Boolean { + val min = statistics.min + val max = statistics.max + + return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isEmpty() + } + + override fun inverseCanDrop(statistics: Statistics): Boolean { + val min = statistics.min + val max = statistics.max + + return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isNotEmpty() + } + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt new file mode 100644 index 00000000..7cd1f159 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -0,0 +1,620 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.arguments.argument +import com.github.ajalt.clikt.parameters.groups.OptionGroup +import com.github.ajalt.clikt.parameters.groups.groupChoice +import com.github.ajalt.clikt.parameters.options.convert +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.defaultLazy +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.options.split +import com.github.ajalt.clikt.parameters.types.file +import com.github.ajalt.clikt.parameters.types.long +import me.tongfei.progressbar.ProgressBar +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.generic.GenericData +import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.hadoop.ParquetWriter +import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.opendc.format.util.LocalOutputFile +import java.io.BufferedReader +import java.io.File +import java.io.FileReader +import java.util.Random +import kotlin.math.max +import kotlin.math.min + +/** + * Represents the command for converting traces + */ +class TraceConverterCli : CliktCommand(name = "trace-converter") { + /** + * The directory where the trace should be stored. + */ + private val outputPath by option("-O", "--output", help = "path to store the trace") + .file(canBeFile = false, mustExist = false) + .defaultLazy { File("output") } + + /** + * The directory where the input trace is located. + */ + private val inputPath by argument("input", help = "path to the input trace") + .file(canBeFile = false) + + /** + * The input type of the trace. + */ + private val type by option("-t", "--type", help = "input type of trace").groupChoice( + "solvinity" to SolvinityConversion(), + "bitbrains" to BitbrainsConversion(), + "azure" to AzureConversion() + ) + + override fun run() { + val metaSchema = SchemaBuilder + .record("meta") + .namespace("org.opendc.format.sc20") + .fields() + .name("id").type().stringType().noDefault() + .name("submissionTime").type().longType().noDefault() + .name("endTime").type().longType().noDefault() + .name("maxCores").type().intType().noDefault() + .name("requiredMemory").type().longType().noDefault() + .endRecord() + val schema = SchemaBuilder + .record("trace") + .namespace("org.opendc.format.sc20") + .fields() + .name("id").type().stringType().noDefault() + .name("time").type().longType().noDefault() + .name("duration").type().longType().noDefault() + .name("cores").type().intType().noDefault() + .name("cpuUsage").type().doubleType().noDefault() + .name("flops").type().longType().noDefault() + .endRecord() + + val metaParquet = File(outputPath, "meta.parquet") + val traceParquet = File(outputPath, "trace.parquet") + + if (metaParquet.exists()) { + metaParquet.delete() + } + if (traceParquet.exists()) { + traceParquet.delete() + } + + val metaWriter = AvroParquetWriter.builder(LocalOutputFile(metaParquet)) + .withSchema(metaSchema) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withPageSize(4 * 1024 * 1024) // For compression + .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) + .build() + + val writer = AvroParquetWriter.builder(LocalOutputFile(traceParquet)) + .withSchema(schema) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withPageSize(4 * 1024 * 1024) // For compression + .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) + .build() + + try { + val type = type ?: throw IllegalArgumentException("Invalid trace conversion") + val allFragments = type.read(inputPath, metaSchema, metaWriter) + allFragments.sortWith(compareBy { it.tick }.thenBy { it.id }) + + for (fragment in allFragments) { + val record = GenericData.Record(schema) + record.put("id", fragment.id) + record.put("time", fragment.tick) + record.put("duration", fragment.duration) + record.put("cores", fragment.cores) + record.put("cpuUsage", fragment.usage) + record.put("flops", fragment.flops) + + writer.write(record) + } + } finally { + writer.close() + metaWriter.close() + } + } +} + +/** + * The supported trace conversions. + */ +sealed class TraceConversion(name: String) : OptionGroup(name) { + /** + * Read the fragments of the trace. + */ + abstract fun read( + traceDirectory: File, + metaSchema: Schema, + metaWriter: ParquetWriter + ): MutableList +} + +class SolvinityConversion : TraceConversion("Solvinity") { + private val clusters by option() + .split(",") + + private val vmPlacements by option("--vm-placements", help = "file containing the VM placements") + .file(canBeDir = false) + .convert { VmPlacementReader(it.inputStream()).use { reader -> reader.read() } } + .required() + + override fun read( + traceDirectory: File, + metaSchema: Schema, + metaWriter: ParquetWriter + ): MutableList { + val clusters = clusters?.toSet() ?: emptySet() + val timestampCol = 0 + val cpuUsageCol = 1 + val coreCol = 12 + val provisionedMemoryCol = 20 + val traceInterval = 5 * 60 * 1000L + + // Identify start time of the entire trace + var minTimestamp = Long.MAX_VALUE + traceDirectory.walk() + .filterNot { it.isDirectory } + .filter { it.extension == "csv" || it.extension == "txt" } + .toList() + .forEach file@{ vmFile -> + BufferedReader(FileReader(vmFile)).use { reader -> + reader.lineSequence() + .chunked(128) + .forEach { lines -> + for (line in lines) { + // Ignore comments in the trace + if (line.startsWith("#") || line.isBlank()) { + continue + } + + val vmId = vmFile.name + + // Check if VM in topology + val clusterName = vmPlacements[vmId] + if (clusterName == null || !clusters.contains(clusterName)) { + continue + } + + val values = line.split("\t") + val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L + + if (timestamp < minTimestamp) { + minTimestamp = timestamp + } + return@file + } + } + } + } + + println("Start of trace at $minTimestamp") + + val allFragments = mutableListOf() + + val begin = 15 * 24 * 60 * 60 * 1000L + val end = 45 * 24 * 60 * 60 * 1000L + + traceDirectory.walk() + .filterNot { it.isDirectory } + .filter { it.extension == "csv" || it.extension == "txt" } + .toList() + .forEach { vmFile -> + println(vmFile) + + var vmId = "" + var maxCores = -1 + var requiredMemory = -1L + var cores: Int + var minTime = Long.MAX_VALUE + + val flopsFragments = sequence { + var last: Fragment? = null + + BufferedReader(FileReader(vmFile)).use { reader -> + reader.lineSequence() + .chunked(128) + .forEach { lines -> + for (line in lines) { + // Ignore comments in the trace + if (line.startsWith("#") || line.isBlank()) { + continue + } + + val values = line.split("\t") + + vmId = vmFile.name + + // Check if VM in topology + val clusterName = vmPlacements[vmId] + if (clusterName == null || !clusters.contains(clusterName)) { + continue + } + + val timestamp = + (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - minTimestamp + if (begin > timestamp || timestamp > end) { + continue + } + + cores = values[coreCol].trim().toInt() + requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) + maxCores = max(maxCores, cores) + minTime = min(minTime, timestamp) + val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz + requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) + maxCores = max(maxCores, cores) + + val flops: Long = (cpuUsage * 5 * 60).toLong() + + last = if (last != null && last!!.flops == 0L && flops == 0L) { + val oldFragment = last!! + Fragment( + vmId, + oldFragment.tick, + oldFragment.flops + flops, + oldFragment.duration + traceInterval, + cpuUsage, + cores + ) + } else { + val fragment = + Fragment( + vmId, + timestamp, + flops, + traceInterval, + cpuUsage, + cores + ) + if (last != null) { + yield(last!!) + } + fragment + } + } + } + } + + if (last != null) { + yield(last!!) + } + } + + var maxTime = Long.MIN_VALUE + flopsFragments.filter { it.tick in begin until end }.forEach { fragment -> + allFragments.add(fragment) + maxTime = max(maxTime, fragment.tick) + } + + if (minTime in begin until end) { + val metaRecord = GenericData.Record(metaSchema) + metaRecord.put("id", vmId) + metaRecord.put("submissionTime", minTime) + metaRecord.put("endTime", maxTime) + metaRecord.put("maxCores", maxCores) + metaRecord.put("requiredMemory", requiredMemory) + metaWriter.write(metaRecord) + } + } + + return allFragments + } +} + +/** + * Conversion of the Bitbrains public trace. + */ +class BitbrainsConversion : TraceConversion("Bitbrains") { + override fun read( + traceDirectory: File, + metaSchema: Schema, + metaWriter: ParquetWriter + ): MutableList { + val timestampCol = 0 + val cpuUsageCol = 3 + val coreCol = 1 + val provisionedMemoryCol = 5 + val traceInterval = 5 * 60 * 1000L + + val allFragments = mutableListOf() + + traceDirectory.walk() + .filterNot { it.isDirectory } + .filter { it.extension == "csv" || it.extension == "txt" } + .toList() + .forEach { vmFile -> + println(vmFile) + + var vmId = "" + var maxCores = -1 + var requiredMemory = -1L + var cores: Int + var minTime = Long.MAX_VALUE + + val flopsFragments = sequence { + var last: Fragment? = null + + BufferedReader(FileReader(vmFile)).use { reader -> + reader.lineSequence() + .drop(1) + .chunked(128) + .forEach { lines -> + for (line in lines) { + // Ignore comments in the trace + if (line.startsWith("#") || line.isBlank()) { + continue + } + + val values = line.split(";\t") + + vmId = vmFile.name + + val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L + + cores = values[coreCol].trim().toInt() + val provisionedMemory = values[provisionedMemoryCol].trim().toDouble() // KB + requiredMemory = max(requiredMemory, (provisionedMemory / 1000).toLong()) + maxCores = max(maxCores, cores) + minTime = min(minTime, timestamp) + val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz + + val flops: Long = (cpuUsage * 5 * 60).toLong() + + last = if (last != null && last!!.flops == 0L && flops == 0L) { + val oldFragment = last!! + Fragment( + vmId, + oldFragment.tick, + oldFragment.flops + flops, + oldFragment.duration + traceInterval, + cpuUsage, + cores + ) + } else { + val fragment = + Fragment( + vmId, + timestamp, + flops, + traceInterval, + cpuUsage, + cores + ) + if (last != null) { + yield(last!!) + } + fragment + } + } + } + } + + if (last != null) { + yield(last!!) + } + } + + var maxTime = Long.MIN_VALUE + flopsFragments.forEach { fragment -> + allFragments.add(fragment) + maxTime = max(maxTime, fragment.tick) + } + + val metaRecord = GenericData.Record(metaSchema) + metaRecord.put("id", vmId) + metaRecord.put("submissionTime", minTime) + metaRecord.put("endTime", maxTime) + metaRecord.put("maxCores", maxCores) + metaRecord.put("requiredMemory", requiredMemory) + metaWriter.write(metaRecord) + } + + return allFragments + } +} + +/** + * Conversion of the Azure public VM trace. + */ +class AzureConversion : TraceConversion("Azure") { + private val seed by option(help = "seed for trace sampling") + .long() + .default(0) + + override fun read( + traceDirectory: File, + metaSchema: Schema, + metaWriter: ParquetWriter + ): MutableList { + val random = Random(seed) + val fraction = 0.01 + + // Read VM table + val vmIdTableCol = 0 + val coreTableCol = 9 + val provisionedMemoryTableCol = 10 + + var vmId: String + var cores: Int + var requiredMemory: Long + + val vmIds = mutableSetOf() + val vmIdToMetadata = mutableMapOf() + + BufferedReader(FileReader(File(traceDirectory, "vmtable.csv"))).use { reader -> + reader.lineSequence() + .chunked(1024) + .forEach { lines -> + for (line in lines) { + // Ignore comments in the trace + if (line.startsWith("#") || line.isBlank()) { + continue + } + // Sample only a fraction of the VMs + if (random.nextDouble() > fraction) { + continue + } + + val values = line.split(",") + + // Exclude VMs with a large number of cores (not specified exactly) + if (values[coreTableCol].contains(">")) { + continue + } + + vmId = values[vmIdTableCol].trim() + cores = values[coreTableCol].trim().toInt() + requiredMemory = values[provisionedMemoryTableCol].trim().toInt() * 1_000L // GB -> MB + + vmIds.add(vmId) + vmIdToMetadata[vmId] = VmInfo(cores, requiredMemory, Long.MAX_VALUE, -1L) + } + } + } + + // Read VM metric reading files + val timestampCol = 0 + val vmIdCol = 1 + val cpuUsageCol = 4 + val traceInterval = 5 * 60 * 1000L + + val vmIdToFragments = mutableMapOf>() + val vmIdToLastFragment = mutableMapOf() + val allFragments = mutableListOf() + + for (i in ProgressBar.wrap((1..195).toList(), "Reading Trace")) { + val readingsFile = File(File(traceDirectory, "readings"), "readings-$i.csv") + var timestamp: Long + var cpuUsage: Double + + BufferedReader(FileReader(readingsFile)).use { reader -> + reader.lineSequence() + .chunked(128) + .forEach { lines -> + for (line in lines) { + // Ignore comments in the trace + if (line.startsWith("#") || line.isBlank()) { + continue + } + + val values = line.split(",") + vmId = values[vmIdCol].trim() + + // Ignore readings for VMs not in the sample + if (!vmIds.contains(vmId)) { + continue + } + + timestamp = values[timestampCol].trim().toLong() * 1000L + vmIdToMetadata[vmId]!!.minTime = min(vmIdToMetadata[vmId]!!.minTime, timestamp) + cpuUsage = values[cpuUsageCol].trim().toDouble() * 3_000 // MHz + vmIdToMetadata[vmId]!!.maxTime = max(vmIdToMetadata[vmId]!!.maxTime, timestamp) + + val flops: Long = (cpuUsage * 5 * 60).toLong() + val lastFragment = vmIdToLastFragment[vmId] + + vmIdToLastFragment[vmId] = + if (lastFragment != null && lastFragment.flops == 0L && flops == 0L) { + Fragment( + vmId, + lastFragment.tick, + lastFragment.flops + flops, + lastFragment.duration + traceInterval, + cpuUsage, + vmIdToMetadata[vmId]!!.cores + ) + } else { + val fragment = + Fragment( + vmId, + timestamp, + flops, + traceInterval, + cpuUsage, + vmIdToMetadata[vmId]!!.cores + ) + if (lastFragment != null) { + if (vmIdToFragments[vmId] == null) { + vmIdToFragments[vmId] = mutableListOf() + } + vmIdToFragments[vmId]!!.add(lastFragment) + allFragments.add(lastFragment) + } + fragment + } + } + } + } + } + + for (entry in vmIdToLastFragment) { + if (entry.value != null) { + if (vmIdToFragments[entry.key] == null) { + vmIdToFragments[entry.key] = mutableListOf() + } + vmIdToFragments[entry.key]!!.add(entry.value!!) + } + } + + println("Read ${vmIdToLastFragment.size} VMs") + + for (entry in vmIdToMetadata) { + val metaRecord = GenericData.Record(metaSchema) + metaRecord.put("id", entry.key) + metaRecord.put("submissionTime", entry.value.minTime) + metaRecord.put("endTime", entry.value.maxTime) + println("${entry.value.minTime} - ${entry.value.maxTime}") + metaRecord.put("maxCores", entry.value.cores) + metaRecord.put("requiredMemory", entry.value.requiredMemory) + metaWriter.write(metaRecord) + } + + return allFragments + } +} + +data class Fragment( + val id: String, + val tick: Long, + val flops: Long, + val duration: Long, + val usage: Double, + val cores: Int +) + +class VmInfo(val cores: Int, val requiredMemory: Long, var minTime: Long, var maxTime: Long) + +/** + * A script to convert a trace in text format into a Parquet trace. + */ +fun main(args: Array): Unit = TraceConverterCli().main(args) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt new file mode 100644 index 00000000..fb641f1b --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import java.io.InputStream + +/** + * A parser for the JSON VM placement data files used for the SC20 paper. + * + * @param input The input stream to read from. + * @param mapper The Jackson object mapper to use. + */ +public class VmPlacementReader( + private val input: InputStream, + private val mapper: ObjectMapper = jacksonObjectMapper() +) : AutoCloseable { + /** + * Read the VM placements. + */ + fun read(): Map { + return mapper.readValue>(input) + .mapKeys { "vm__workload__${it.key}.txt" } + .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 + } + + override fun close() { + input.close() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 4b21b4f7..beaa798f 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -34,12 +34,12 @@ import org.opendc.compute.service.scheduler.FilterScheduler import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.weights.CoreMemoryWeigher +import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.experiments.capelin.trace.Sc20ParquetTraceReader -import org.opendc.experiments.capelin.trace.Sc20RawParquetTraceReader +import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation @@ -161,9 +161,8 @@ class CapelinIntegrationTest { * Obtain the trace reader for the test. */ private fun createTestTraceReader(fraction: Double = 1.0, seed: Int = 0): TraceReader { - return Sc20ParquetTraceReader( - listOf(Sc20RawParquetTraceReader(File("src/test/resources/trace"))), - emptyMap(), + return ParquetTraceReader( + listOf(RawParquetTraceReader(File("src/test/resources/trace"))), Workload("test", fraction), seed ) @@ -174,7 +173,7 @@ class CapelinIntegrationTest { */ private fun createTestEnvironmentReader(name: String = "topology"): EnvironmentReader { val stream = object {}.javaClass.getResourceAsStream("/env/$name.txt") - return Sc20ClusterEnvironmentReader(stream) + return ClusterEnvironmentReader(stream) } class TestExperimentReporter : ExperimentMonitor { diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 28928dcb..8fc4f6b8 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -38,7 +38,7 @@ import org.opendc.compute.service.scheduler.weights.RandomWeigher import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.* import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor -import org.opendc.experiments.capelin.trace.Sc20StreamingParquetTraceReader +import org.opendc.experiments.capelin.trace.StreamingParquetTraceReader import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider @@ -53,7 +53,6 @@ import org.opendc.simulator.resources.SimResourceInterpreter import java.io.File import java.time.Clock import java.util.* -import kotlin.random.asKotlinRandom /** * Experiments for the OpenDC project on Energy modeling. @@ -88,7 +87,7 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { val meterProvider: MeterProvider = createMeterProvider(clock) val monitor = ParquetExperimentMonitor(File(config.getString("output-path")), "power_model=$powerModel/run_id=$repeat", 4096) - val trace = Sc20StreamingParquetTraceReader(File(config.getString("trace-path"), trace), random = Random(1).asKotlinRandom()) + val trace = StreamingParquetTraceReader(File(config.getString("trace-path"), trace)) withComputeService(clock, meterProvider, allocationPolicy) { scheduler -> withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { -- cgit v1.2.3 From e56967a29ac2b2d26cc085b1f3e27096dad6a170 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 24 Jun 2021 12:54:52 +0200 Subject: simulator: Re-implement performance interference model This change updates reimplements the performance interference model to work on top of the universal resource model in `opendc-simulator-resources`. This enables us to model interference and performance variability of other resources such as disk or network in the future. --- .../experiments/capelin/ExperimentHelpers.kt | 7 ++- .../org/opendc/experiments/capelin/Portfolio.kt | 11 +++- .../capelin/trace/ParquetTraceReader.kt | 8 +-- .../capelin/trace/PerformanceInterferenceReader.kt | 65 ++++++++++++++++++++++ .../experiments/capelin/trace/VmPlacementReader.kt | 6 +- .../experiments/capelin/CapelinIntegrationTest.kt | 2 +- .../trace/PerformanceInterferenceReaderTest.kt | 47 ++++++++++++++++ .../src/test/resources/perf-interference.json | 22 ++++++++ 8 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 06251dd3..9548253d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -44,6 +44,8 @@ import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel +import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.failures.CorrelatedFaultInjector @@ -123,6 +125,7 @@ suspend fun withComputeService( meterProvider: MeterProvider, environmentReader: EnvironmentReader, scheduler: ComputeScheduler, + interferenceModel: VmInterferenceModel? = null, block: suspend CoroutineScope.(ComputeService) -> Unit ): Unit = coroutineScope { val interpreter = SimResourceInterpreter(coroutineContext, clock) @@ -138,7 +141,8 @@ suspend fun withComputeService( interpreter, meterProvider.get("opendc-compute-simulator"), SimFairShareHypervisorProvider(), - def.powerModel + powerDriver = SimplePowerDriver(def.powerModel), + interferenceDomain = interferenceModel?.newDomain() ) } @@ -161,7 +165,6 @@ suspend fun withComputeService( /** * Attach the specified monitor to the VM provisioner. */ -@OptIn(ExperimentalCoroutinesApi::class) suspend fun withMonitor( monitor: ExperimentMonitor, clock: Clock, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 460da303..cbb5bfd9 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -39,11 +39,14 @@ import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.core.runBlockingSimulation import java.io.File +import java.io.FileInputStream import java.util.* import java.util.concurrent.ConcurrentHashMap @@ -119,6 +122,12 @@ abstract class Portfolio(name: String) : Experiment(name) { } } + val performanceInterferenceModel = if (operationalPhenomena.hasInterference) + PerformanceInterferenceReader(FileInputStream(config.getString("interference-model"))) + .use { VmInterferenceModel(it.read(), Random(seeder.nextLong())) } + else + null + val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) val monitor = ParquetExperimentMonitor( @@ -127,7 +136,7 @@ abstract class Portfolio(name: String) : Experiment(name) { 4096 ) - withComputeService(clock, meterProvider, environment, allocationPolicy) { scheduler -> + withComputeService(clock, meterProvider, environment, allocationPolicy, performanceInterferenceModel) { scheduler -> val failureDomain = if (operationalPhenomena.failureFrequency > 0) { logger.debug("ENABLING failures") createFailureDomain( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt index 2ebe65ea..5ad75565 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt @@ -31,11 +31,11 @@ import org.opendc.simulator.compute.workload.SimWorkload /** * A [TraceReader] for the internal VM workload trace format. * - * @param rawReaders The raw trace readers to use.. - * @param workload The workload to use. - * @param seed The seed to use for workload sampling. + * @param rawReaders The internal raw trace readers to use. + * @param workload The workload to read. + * @param seed The seed to use for sampling. */ -class ParquetTraceReader( +public class ParquetTraceReader( rawReaders: List, workload: Workload, seed: Int diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt new file mode 100644 index 00000000..a19f5699 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import org.opendc.simulator.compute.kernel.interference.VmInterferenceGroup +import java.io.InputStream + +/** + * A parser for the JSON performance interference setup files used for the TPDS article on Capelin. + * + * @param input The input stream to read from. + * @param mapper The Jackson object mapper to use. + */ +class PerformanceInterferenceReader( + private val input: InputStream, + private val mapper: ObjectMapper = jacksonObjectMapper() +) : AutoCloseable { + init { + mapper.addMixIn(VmInterferenceGroup::class.java, GroupMixin::class.java) + } + + /** + * Read the performance interface model from the input. + */ + fun read(): List { + return mapper.readValue(input) + } + + override fun close() { + input.close() + } + + private data class GroupMixin( + @JsonProperty("minServerLoad") + val targetLoad: Double, + @JsonProperty("performanceScore") + val score: Double, + @JsonProperty("vms") + val members: Set, + ) +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt index fb641f1b..7a1683f0 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt @@ -28,7 +28,7 @@ import com.fasterxml.jackson.module.kotlin.readValue import java.io.InputStream /** - * A parser for the JSON VM placement data files used for the SC20 paper. + * A parser for the JSON VM placement data files used for the TPDS article on Capelin. * * @param input The input stream to read from. * @param mapper The Jackson object mapper to use. @@ -38,9 +38,9 @@ public class VmPlacementReader( private val mapper: ObjectMapper = jacksonObjectMapper() ) : AutoCloseable { /** - * Read the VM placements. + * Read the VM placements from the input. */ - fun read(): Map { + public fun read(): Map { return mapper.readValue>(input) .mapKeys { "vm__workload__${it.key}.txt" } .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index beaa798f..08e04ddf 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -172,7 +172,7 @@ class CapelinIntegrationTest { * Obtain the environment reader for the test. */ private fun createTestEnvironmentReader(name: String = "topology"): EnvironmentReader { - val stream = object {}.javaClass.getResourceAsStream("/env/$name.txt") + val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name.txt")) return ClusterEnvironmentReader(stream) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt new file mode 100644 index 00000000..9b1513dc --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertAll + +/** + * Test suite for the [PerformanceInterferenceReader] class. + */ +class PerformanceInterferenceReaderTest { + @Test + fun testSmoke() { + val input = checkNotNull(PerformanceInterferenceReader::class.java.getResourceAsStream("/perf-interference.json")) + val reader = PerformanceInterferenceReader(input) + + val result = reader.use { reader.read() } + + assertAll( + { assertEquals(2, result.size) }, + { assertEquals(setOf("vm_a", "vm_c", "vm_x", "vm_y"), result[0].members) }, + { assertEquals(0.0, result[0].targetLoad, 0.001) }, + { assertEquals(0.8830158730158756, result[0].score, 0.001) } + ) + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json b/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json new file mode 100644 index 00000000..1be5852b --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json @@ -0,0 +1,22 @@ +[ + { + "vms": [ + "vm_a", + "vm_c", + "vm_x", + "vm_y" + ], + "minServerLoad": 0.0, + "performanceScore": 0.8830158730158756 + }, + { + "vms": [ + "vm_a", + "vm_b", + "vm_c", + "vm_d" + ], + "minServerLoad": 0.0, + "performanceScore": 0.7133055555552751 + } +] -- cgit v1.2.3 From ce2d730159ae24c7cebb22ec42d094fe5fdc888f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 13 Aug 2021 17:14:53 +0200 Subject: build: Update Kotlin dependencies This change updates the Kotlin dependencies used by OpenDC to their latest version. --- .../kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt | 2 +- .../experiments/capelin/monitor/ExperimentMetricExporter.kt | 9 ++++++--- .../org/opendc/experiments/serverless/ServerlessExperiment.kt | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 9548253d..42d240dc 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -186,7 +186,7 @@ suspend fun withMonitor( this, listOf(metricProducer), ExperimentMetricExporter(monitor, clock, scheduler.hosts.associateBy { it.uid.toString() }), - exportInterval = 5 * 60 * 1000 /* Every 5 min (which is the granularity of the workload trace) */ + exportInterval = 5L * 60 * 1000 /* Every 5 min (which is the granularity of the workload trace) */ ) try { diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index 54ab3b5b..f520a28c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -22,6 +22,7 @@ package org.opendc.experiments.capelin.monitor +import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.sdk.common.CompletableResultCode import io.opentelemetry.sdk.metrics.data.MetricData import io.opentelemetry.sdk.metrics.export.MetricExporter @@ -36,6 +37,8 @@ public class ExperimentMetricExporter( private val clock: Clock, private val hosts: Map ) : MetricExporter { + private val hostKey = AttributeKey.stringKey("host") + override fun export(metrics: Collection): CompletableResultCode { val metricsByName = metrics.associateBy { it.name } reportHostMetrics(metricsByName) @@ -99,7 +102,7 @@ public class ExperimentMetricExporter( private fun mapDoubleSummary(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data?.doubleSummaryData?.points ?: emptyList() for (point in points) { - val uid = point.labels["host"] + val uid = point.attributes[hostKey] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -113,7 +116,7 @@ public class ExperimentMetricExporter( private fun mapDoubleGauge(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data?.doubleGaugeData?.points ?: emptyList() for (point in points) { - val uid = point.labels["host"] + val uid = point.attributes[hostKey] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -125,7 +128,7 @@ public class ExperimentMetricExporter( private fun mapLongSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Long) -> Unit) { val points = data?.longSumData?.points ?: emptyList() for (point in points) { - val uid = point.labels["host"] + val uid = point.attributes[hostKey] val hostMetric = hostMetrics[uid] if (hostMetric != null) { diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt index 231b491e..650416f5 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt @@ -85,7 +85,7 @@ public class ServerlessExperiment : Experiment("Serverless") { val delayInjector = StochasticDelayInjector(coldStartModel, Random()) val deployer = SimFunctionDeployer(clock, this, createMachineModel(), delayInjector) { FunctionTraceWorkload(traceById.getValue(it.name)) } val service = - FaaSService(coroutineContext, clock, meterProvider.get("opendc-serverless"), deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = 10 * 60 * 1000)) + FaaSService(coroutineContext, clock, meterProvider.get("opendc-serverless"), deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = 10L * 60 * 1000)) val client = service.newClient() coroutineScope { -- cgit v1.2.3 From b8f64c1d3df2c990df8941cd036222fab2def9fa Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 22 Aug 2021 13:23:53 +0200 Subject: refactor(compute): Update FilterScheduler to follow OpenStack's Nova This change updates the FilterScheduler implementation to follow more closely the scheduler implementation in OpenStack's Nova. We now normalize the weights, support many of the filters and weights in OpenStack and support overcommitting resources. --- .../experiments/capelin/ExperimentHelpers.kt | 59 ++++++++++++++++++++++ .../org/opendc/experiments/capelin/Portfolio.kt | 53 +------------------ .../experiments/capelin/CapelinIntegrationTest.kt | 19 +++---- .../experiments/energy21/EnergyExperiment.kt | 9 ++-- 4 files changed, 76 insertions(+), 64 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 42d240dc..2c443678 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -38,6 +38,15 @@ import org.opendc.compute.service.driver.Host import org.opendc.compute.service.driver.HostListener import org.opendc.compute.service.driver.HostState import org.opendc.compute.service.scheduler.ComputeScheduler +import org.opendc.compute.service.scheduler.FilterScheduler +import org.opendc.compute.service.scheduler.ReplayScheduler +import org.opendc.compute.service.scheduler.filters.ComputeFilter +import org.opendc.compute.service.scheduler.filters.RamFilter +import org.opendc.compute.service.scheduler.filters.VCpuFilter +import org.opendc.compute.service.scheduler.weights.CoreRamWeigher +import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher +import org.opendc.compute.service.scheduler.weights.RamWeigher +import org.opendc.compute.service.scheduler.weights.VCpuWeigher import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter import org.opendc.experiments.capelin.monitor.ExperimentMonitor @@ -301,3 +310,53 @@ fun createMeterProvider(clock: Clock): MeterProvider { .registerView(powerSelector, powerView) .build() } + +/** + * Create a [ComputeScheduler] for the experiment. + */ +fun createComputeScheduler(allocationPolicy: String, seeder: Random, vmPlacements: Map = emptyMap()): ComputeScheduler { + val cpuAllocationRatio = 16.0 + val ramAllocationRatio = 1.5 + return when (allocationPolicy) { + "mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = 1.0)) + ) + "mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = -1.0)) + ) + "core-mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) + "core-mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = -1.0)) + ) + "active-servers" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = -1.0)) + ) + "active-servers-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = 1.0)) + ) + "provisioned-cores" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)) + ) + "provisioned-cores-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)) + ) + "random" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = emptyList(), + subsetSize = Int.MAX_VALUE, + random = java.util.Random(seeder.nextLong()) + ) + "replay" -> ReplayScheduler(vmPlacements) + else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index cbb5bfd9..ee832af8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -28,10 +28,6 @@ import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import mu.KotlinLogging -import org.opendc.compute.service.scheduler.* -import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.weights.* import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena @@ -49,6 +45,7 @@ import java.io.File import java.io.FileInputStream import java.util.* import java.util.concurrent.ConcurrentHashMap +import kotlin.random.asKotlinRandom /** * A portfolio represents a collection of scenarios are tested for the work. @@ -105,7 +102,7 @@ abstract class Portfolio(name: String) : Experiment(name) { val environment = ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) val chan = Channel(Channel.CONFLATED) - val allocationPolicy = createComputeScheduler(seeder) + val allocationPolicy = createComputeScheduler(allocationPolicy, seeder.asKotlinRandom(), vmPlacements) val meterProvider = createMeterProvider(clock) val workload = workload @@ -167,50 +164,4 @@ abstract class Portfolio(name: String) : Experiment(name) { val monitorResults = collectMetrics(meterProvider as MetricProducer) logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } } - - /** - * Create the [ComputeScheduler] instance to use for the trial. - */ - private fun createComputeScheduler(seeder: Random): ComputeScheduler { - return when (allocationPolicy) { - "mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to -1.0) - ) - "mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to -1.0) - ) - "core-mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) - ) - "core-mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) - ) - "active-servers" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "active-servers-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(InstanceCountWeigher() to 1.0) - ) - "provisioned-cores" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "provisioned-cores-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to 1.0) - ) - "random" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(RandomWeigher(Random(seeder.nextLong())) to 1.0) - ) - "replay" -> ReplayScheduler(vmPlacements) - else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") - } - } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 08e04ddf..75428011 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -31,9 +31,10 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll import org.opendc.compute.service.driver.Host import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.weights.CoreMemoryWeigher +import org.opendc.compute.service.scheduler.filters.RamFilter +import org.opendc.compute.service.scheduler.filters.VCpuFilter +import org.opendc.compute.service.scheduler.weights.CoreRamWeigher import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ExperimentMonitor @@ -68,8 +69,8 @@ class CapelinIntegrationTest { val seed = 0 val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) ) val traceReader = createTestTraceReader() val environmentReader = createTestEnvironmentReader() @@ -113,9 +114,9 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(207380244590, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(207112418950, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(267825640, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(207380204679, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(207371815929, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(8388750, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } @@ -125,8 +126,8 @@ class CapelinIntegrationTest { val seed = 1 val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) ) val traceReader = createTestTraceReader(0.5, seed) val environmentReader = createTestEnvironmentReader("single") diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 8fc4f6b8..e64e20a2 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -32,9 +32,9 @@ import mu.KotlinLogging import org.opendc.compute.service.ComputeService import org.opendc.compute.service.scheduler.ComputeScheduler import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.weights.RandomWeigher +import org.opendc.compute.service.scheduler.filters.RamFilter +import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.* import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor @@ -81,8 +81,9 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { override fun doRun(repeat: Int): Unit = runBlockingSimulation { val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(RandomWeigher(Random(0)) to 1.0) + filters = listOf(ComputeFilter(), VCpuFilter(1.0), RamFilter(1.0)), + weighers = listOf(), + subsetSize = Int.MAX_VALUE ) val meterProvider: MeterProvider = createMeterProvider(clock) -- cgit v1.2.3 From 31a1f298c71cd3203fdcd57bd39ba8813009dd5b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 17 Aug 2021 19:25:11 +0200 Subject: refactor(simulator): Execute traces based on timestamps This change refactors the trace workload in the OpenDC simulator to track execute a fragment based on the fragment's timestamp. This makes sure that the trace is replayed identically to the original execution. --- .../org/opendc/experiments/capelin/ExperimentHelpers.kt | 2 +- .../experiments/capelin/trace/RawParquetTraceReader.kt | 2 ++ .../experiments/capelin/trace/StreamingParquetTraceReader.kt | 4 +++- .../org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 12 ++++++------ .../experiments/serverless/trace/FunctionTraceWorkload.kt | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 2c443678..fa9fa2fc 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -258,7 +258,7 @@ suspend fun processTrace( delay(max(0, (entry.start - offset) - clock.millis())) launch { chan.send(Unit) - val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace) + val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, offset = -offset + 300001) val server = client.newServer( entry.name, image, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt index 94193780..16ad6816 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -50,11 +50,13 @@ class RawParquetTraceReader(private val path: File) { val record = reader.read() ?: break val id = record["id"].toString() + val time = record["time"] as Long val duration = record["duration"] as Long val cores = record["cores"] as Int val cpuUsage = record["cpuUsage"] as Double val fragment = SimTraceWorkload.Fragment( + time, duration, cpuUsage, cores diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt index a3b45f47..35f4c5b8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -81,7 +81,7 @@ class StreamingParquetTraceReader(traceFile: File, selectedVms: List = e /** * A poisonous fragment. */ - private val poison = Pair("\u0000", SimTraceWorkload.Fragment(0, 0.0, 0)) + private val poison = Pair("\u0000", SimTraceWorkload.Fragment(0L, 0, 0.0, 0)) /** * The thread to read the records in. @@ -103,11 +103,13 @@ class StreamingParquetTraceReader(traceFile: File, selectedVms: List = e } val id = record["id"].toString() + val time = record["time"] as Long val duration = record["duration"] as Long val cores = record["cores"] as Int val cpuUsage = record["cpuUsage"] as Double val fragment = SimTraceWorkload.Fragment( + time, duration, cpuUsage, cores diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 75428011..00ab9190 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -114,9 +114,9 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(207380204679, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(207371815929, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(8388750, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(155252275350, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(155086837649, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(165488283, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } @@ -151,9 +151,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(96344616902, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(96324879442, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(19737460, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(29454904468, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(29355293349, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(99627123, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt index 9a93092e..a119a219 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt @@ -29,6 +29,6 @@ import org.opendc.simulator.compute.workload.SimWorkload /** * A [SimFaaSWorkload] for a [FunctionTrace]. */ -public class FunctionTraceWorkload(trace: FunctionTrace) : SimFaaSWorkload, SimWorkload by SimTraceWorkload(trace.samples.asSequence().map { SimTraceWorkload.Fragment(it.duration, it.cpuUsage, 1) }) { +public class FunctionTraceWorkload(trace: FunctionTrace) : SimFaaSWorkload, SimWorkload by SimTraceWorkload(trace.samples.asSequence().map { SimTraceWorkload.Fragment(it.timestamp, it.duration, it.cpuUsage, 1) }) { override suspend fun invoke() {} } -- cgit v1.2.3 From 484848e2e0bfdaf46f10112e358d3475dbf8e725 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 12:13:07 +0200 Subject: fix(simulator): Record overcommit only after deadline This change fixes an issue with the simulator where it would record overcomitted work if the output was updated before the deadline was reached. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 00ab9190..3dca1e09 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -116,7 +116,7 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, { assertEquals(155252275350, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, { assertEquals(155086837649, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(165488283, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(155088144, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } -- cgit v1.2.3 From c46ff4c5cc18ba8a82ee0135f087c4d7aed1e804 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 14:39:03 +0200 Subject: fix(simulator): Support unaligned trace fragments --- .../org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 3dca1e09..393fb88d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -114,9 +114,9 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(155252275350, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(155086837649, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(155088144, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(155252275351, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(155086837645, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(725049, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } @@ -153,7 +153,7 @@ class CapelinIntegrationTest { assertAll( { assertEquals(29454904468, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, { assertEquals(29355293349, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(99627123, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(0, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } -- cgit v1.2.3 From a23ad09d5a1c4033781bd5403ad766cae83a2beb Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 12:11:22 +0200 Subject: refactor(format): Clean up Bitbrains trace reader to enable re-use This change updates the code for the Bitbrains trace reader and upgrades the TraceConverter to re-use existing code of the Bitbrains trace reader. --- .../experiments/capelin/trace/TraceConverter.kt | 127 +++++---------------- 1 file changed, 27 insertions(+), 100 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt index 7cd1f159..d7daa35b 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -26,12 +26,7 @@ import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.groups.OptionGroup import com.github.ajalt.clikt.parameters.groups.groupChoice -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.defaultLazy -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.options.split +import com.github.ajalt.clikt.parameters.options.* import com.github.ajalt.clikt.parameters.types.file import com.github.ajalt.clikt.parameters.types.long import me.tongfei.progressbar.ProgressBar @@ -41,11 +36,13 @@ import org.apache.avro.generic.GenericData import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.opendc.format.trace.bitbrains.BitbrainsTraceReader import org.opendc.format.util.LocalOutputFile +import org.opendc.simulator.compute.workload.SimTraceWorkload import java.io.BufferedReader import java.io.File import java.io.FileReader -import java.util.Random +import java.util.* import kotlin.math.max import kotlin.math.min @@ -340,106 +337,36 @@ class BitbrainsConversion : TraceConversion("Bitbrains") { metaSchema: Schema, metaWriter: ParquetWriter ): MutableList { - val timestampCol = 0 - val cpuUsageCol = 3 - val coreCol = 1 - val provisionedMemoryCol = 5 - val traceInterval = 5 * 60 * 1000L - - val allFragments = mutableListOf() - - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach { vmFile -> - println(vmFile) - - var vmId = "" - var maxCores = -1 - var requiredMemory = -1L - var cores: Int - var minTime = Long.MAX_VALUE - - val flopsFragments = sequence { - var last: Fragment? = null - - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .drop(1) - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split(";\t") - - vmId = vmFile.name - - val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - - cores = values[coreCol].trim().toInt() - val provisionedMemory = values[provisionedMemoryCol].trim().toDouble() // KB - requiredMemory = max(requiredMemory, (provisionedMemory / 1000).toLong()) - maxCores = max(maxCores, cores) - minTime = min(minTime, timestamp) - val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz - - val flops: Long = (cpuUsage * 5 * 60).toLong() - - last = if (last != null && last!!.flops == 0L && flops == 0L) { - val oldFragment = last!! - Fragment( - vmId, - oldFragment.tick, - oldFragment.flops + flops, - oldFragment.duration + traceInterval, - cpuUsage, - cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - cores - ) - if (last != null) { - yield(last!!) - } - fragment - } - } - } - } - - if (last != null) { - yield(last!!) - } - } - + val fragments = mutableListOf() + BitbrainsTraceReader(traceDirectory).use { reader -> + reader.forEach { entry -> + val trace = (entry.workload as SimTraceWorkload).trace var maxTime = Long.MIN_VALUE - flopsFragments.forEach { fragment -> - allFragments.add(fragment) - maxTime = max(maxTime, fragment.tick) + trace.forEach { fragment -> + val flops: Long = (fragment.usage * fragment.duration / 1000).toLong() + fragments.add( + Fragment( + entry.name, + fragment.timestamp, + flops, + fragment.duration, + fragment.usage, + fragment.cores + ) + ) + maxTime = max(maxTime, fragment.timestamp + fragment.duration) } val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", vmId) - metaRecord.put("submissionTime", minTime) + metaRecord.put("id", entry.name) + metaRecord.put("submissionTime", entry.start) metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", maxCores) - metaRecord.put("requiredMemory", requiredMemory) + metaRecord.put("maxCores", entry.meta["cores"]) + metaRecord.put("requiredMemory", entry.meta["required-memory"]) metaWriter.write(metaRecord) } - - return allFragments + } + return fragments } } -- cgit v1.2.3 From 1af7b83695d997381163f2b72c67ed26d5b4891f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 12:16:02 +0200 Subject: fix(capelin): Keep trace order after sampling This change fixes an issue with the workload sampler where the resulting workload entries would not be ordered properly according to their submission time. --- .../main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt | 2 ++ .../kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index fa9fa2fc..b5090119 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -255,6 +255,8 @@ suspend fun processTrace( offset = entry.start - clock.millis() } + // Make sure the trace entries are ordered by submission time + assert(entry.start - offset >= 0) { "Invalid trace order" } delay(max(0, (entry.start - offset) - clock.millis())) launch { chan.send(Unit) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt index 5ad75565..0f49ecd2 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt @@ -53,8 +53,7 @@ public class ParquetTraceReader( this.zip(listOf(workload)) } } - .map { sampleWorkload(it.first, workload, it.second, seed) } - .flatten() + .flatMap { sampleWorkload(it.first, workload, it.second, seed).sortedBy(TraceEntry::start) } .iterator() override fun hasNext(): Boolean = iterator.hasNext() -- cgit v1.2.3 From 97a28129e4638f601864a08f483908907b9026e6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 14:40:28 +0200 Subject: fix(capelin): Update Bitbrains trace tests This change updates the Bitbrains trace tests with the updated trace that does not hardcode the duration of the trace fragments. --- .../experiments/capelin/CapelinIntegrationTest.kt | 12 +++--- .../src/test/resources/trace/meta.parquet | Bin 2148 -> 2081 bytes .../src/test/resources/trace/trace.parquet | Bin 1672463 -> 1647189 bytes .../opendc-experiments-radice/build.gradle.kts | 48 +++++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-radice/build.gradle.kts (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 393fb88d..0437a022 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -114,9 +114,9 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(155252275351, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(155086837645, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(725049, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(219751355711, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(206351165081, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(1148906334, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } @@ -151,9 +151,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(29454904468, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(29355293349, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(0, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(37954956986, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(34840774250, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(971076806, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet index ce7a812c..ee76d38f 100644 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet differ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet index 1d7ce882..9b1cde13 100644 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet differ diff --git a/opendc-experiments/opendc-experiments-radice/build.gradle.kts b/opendc-experiments/opendc-experiments-radice/build.gradle.kts new file mode 100644 index 00000000..c1515165 --- /dev/null +++ b/opendc-experiments/opendc-experiments-radice/build.gradle.kts @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "Experiments for the Risk Analysis work" + +/* Build configuration */ +plugins { + `experiment-conventions` + `testing-conventions` +} + +dependencies { + api(platform(projects.opendcPlatform)) + api(projects.opendcHarness.opendcHarnessApi) + implementation(projects.opendcFormat) + implementation(projects.opendcSimulator.opendcSimulatorCore) + implementation(projects.opendcSimulator.opendcSimulatorCompute) + implementation(projects.opendcSimulator.opendcSimulatorFailures) + implementation(projects.opendcCompute.opendcComputeSimulator) + implementation(projects.opendcTelemetry.opendcTelemetrySdk) + + implementation(libs.kotlin.logging) + implementation(libs.config) + implementation(libs.progressbar) + implementation(libs.clikt) + + implementation(libs.parquet) + testImplementation(libs.log4j.slf4j) +} -- cgit v1.2.3 From 5266ecd476a18f601cb4eb6166f4c8338c440210 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 18 Aug 2021 21:54:55 +0200 Subject: test(capelin): Add tests for interference and failures This change adds tests to the Capelin integration suite for performance interference as well as failures. These test more accurately the experiment setup. --- .../experiments/capelin/ExperimentHelpers.kt | 2 +- .../capelin/monitor/ExperimentMetricExporter.kt | 2 +- .../experiments/capelin/CapelinIntegrationTest.kt | 105 +++++++++++++++++++++ .../resources/bitbrains-perf-interference.json | 21 +++++ 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/bitbrains-perf-interference.json (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index b5090119..d7df4454 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -277,7 +277,7 @@ suspend fun processTrace( override fun onStateChanged(server: Server, newState: ServerState) { monitor.reportVmStateChange(clock.millis(), server, newState) - if (newState == ServerState.TERMINATED || newState == ServerState.ERROR) { + if (newState == ServerState.TERMINATED) { cont.resume(Unit) } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index f520a28c..7fb2f83c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -74,7 +74,7 @@ public class ExperimentMetricExporter( m.overcommissionedBurst = v.toLong() } - mapDoubleSummary(metrics["cpu.work.interfered"], hostMetrics) { m, v -> + mapDoubleSummary(metrics["cpu.work.interference"], hostMetrics) { m, v -> m.interferedBurst = v.toLong() } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 0437a022..a3300b71 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -39,12 +39,15 @@ import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation import java.io.File +import java.util.* /** * An integration test suite for the SC20 experiments. @@ -63,6 +66,9 @@ class CapelinIntegrationTest { monitor = TestExperimentReporter() } + /** + * Test a large simulation setup. + */ @Test fun testLarge() = runBlockingSimulation { val failures = false @@ -121,6 +127,9 @@ class CapelinIntegrationTest { ) } + /** + * Test a small simulation setup. + */ @Test fun testSmall() = runBlockingSimulation { val seed = 1 @@ -158,6 +167,102 @@ class CapelinIntegrationTest { ) } + /** + * Test a small simulation setup with interference. + */ + @Test + fun testInterference() = runBlockingSimulation { + val seed = 1 + val chan = Channel(Channel.CONFLATED) + val allocationPolicy = FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) + val traceReader = createTestTraceReader(0.25, seed) + val environmentReader = createTestEnvironmentReader("single") + + val perfInterferenceInput = checkNotNull(CapelinIntegrationTest::class.java.getResourceAsStream("/bitbrains-perf-interference.json")) + val performanceInterferenceModel = + PerformanceInterferenceReader(perfInterferenceInput).use { VmInterferenceModel(it.read(), Random(seed.toLong())) } + + val meterProvider = createMeterProvider(clock) + + withComputeService(clock, meterProvider, environmentReader, allocationPolicy, performanceInterferenceModel) { scheduler -> + withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + processTrace( + clock, + traceReader, + scheduler, + chan, + monitor + ) + } + } + + val metrics = collectMetrics(meterProvider as MetricProducer) + println("Finish SUBMIT=${metrics.submittedVms} FAIL=${metrics.unscheduledVms} QUEUE=${metrics.queuedVms} RUNNING=${metrics.runningVms}") + + // Note that these values have been verified beforehand + assertAll( + { assertEquals(37954956986, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(34840774250, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(971076806, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(13885404, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + ) + } + + /** + * Test a small simulation setup with failures. + */ + @Test + fun testFailures() = runBlockingSimulation { + val seed = 1 + val chan = Channel(Channel.CONFLATED) + val allocationPolicy = FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) + val traceReader = createTestTraceReader(0.25, seed) + val environmentReader = createTestEnvironmentReader("single") + + val meterProvider = createMeterProvider(clock) + + withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> + val failureDomain = + createFailureDomain( + this, + clock, + seed, + 24.0 * 7, + scheduler, + chan + ) + + withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + processTrace( + clock, + traceReader, + scheduler, + chan, + monitor + ) + } + + failureDomain.cancel() + } + + val metrics = collectMetrics(meterProvider as MetricProducer) + println("Finish SUBMIT=${metrics.submittedVms} FAIL=${metrics.unscheduledVms} QUEUE=${metrics.queuedVms} RUNNING=${metrics.runningVms}") + + // Note that these values have been verified beforehand + assertAll( + { assertEquals(25336984869, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(23668547517, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(368151656, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + ) + } + /** * Obtain the trace reader for the test. */ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/bitbrains-perf-interference.json b/opendc-experiments/opendc-experiments-capelin/src/test/resources/bitbrains-perf-interference.json new file mode 100644 index 00000000..51fc6366 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/test/resources/bitbrains-perf-interference.json @@ -0,0 +1,21 @@ +[ + { + "vms": [ + "141", + "379", + "851", + "116" + ], + "minServerLoad": 0.0, + "performanceScore": 0.8830158730158756 + }, + { + "vms": [ + "205", + "116", + "463" + ], + "minServerLoad": 0.0, + "performanceScore": 0.7133055555552751 + } +] -- cgit v1.2.3 From f111081627280d4e7e1d7147c56cdce708e32433 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 14:06:39 +0200 Subject: build: Upgrade to OpenTelemetry 1.5 This change upgrades the OpenTelemetry dependency to version 1.5, which contains various breaking changes in the metrics API. --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../experiments/capelin/ExperimentHelpers.kt | 13 ---- .../capelin/monitor/ExperimentMetricExporter.kt | 77 ++++++++++------------ .../capelin/monitor/ExperimentMonitor.kt | 3 +- .../capelin/monitor/ParquetExperimentMonitor.kt | 8 +-- .../experiments/capelin/CapelinIntegrationTest.kt | 34 +++++----- .../opendc/experiments/tf20/core/SimTFDevice.kt | 16 +++-- 7 files changed, 70 insertions(+), 82 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 324cae3e..53643aba 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) + implementation(libs.opentelemetry.semconv) implementation(libs.kotlin.logging) implementation(libs.config) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index d7df4454..4cffb8d3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -24,11 +24,7 @@ package org.opendc.experiments.capelin import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.aggregator.AggregatorFactory -import io.opentelemetry.sdk.metrics.common.InstrumentType import io.opentelemetry.sdk.metrics.export.MetricProducer -import io.opentelemetry.sdk.metrics.view.InstrumentSelector -import io.opentelemetry.sdk.metrics.view.View import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel import mu.KotlinLogging @@ -298,18 +294,9 @@ suspend fun processTrace( * Create a [MeterProvider] instance for the experiment. */ fun createMeterProvider(clock: Clock): MeterProvider { - val powerSelector = InstrumentSelector.builder() - .setInstrumentNameRegex("power\\.usage") - .setInstrumentType(InstrumentType.VALUE_RECORDER) - .build() - val powerView = View.builder() - .setAggregatorFactory(AggregatorFactory.lastValue()) - .build() - return SdkMeterProvider .builder() .setClock(clock.toOtelClock()) - .registerView(powerSelector, powerView) .build() } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index 7fb2f83c..16358817 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -22,10 +22,10 @@ package org.opendc.experiments.capelin.monitor -import io.opentelemetry.api.common.AttributeKey import io.opentelemetry.sdk.common.CompletableResultCode import io.opentelemetry.sdk.metrics.data.MetricData import io.opentelemetry.sdk.metrics.export.MetricExporter +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes import org.opendc.compute.service.driver.Host import java.time.Clock @@ -37,7 +37,7 @@ public class ExperimentMetricExporter( private val clock: Clock, private val hosts: Map ) : MetricExporter { - private val hostKey = AttributeKey.stringKey("host") + private val hostKey = ResourceAttributes.HOST_ID override fun export(metrics: Collection): CompletableResultCode { val metricsByName = metrics.associateBy { it.name } @@ -46,50 +46,31 @@ public class ExperimentMetricExporter( return CompletableResultCode.ofSuccess() } + private var lastHostMetrics: Map = emptyMap() + private val hostMetricsSingleton = HostMetrics() + private fun reportHostMetrics(metrics: Map) { val hostMetrics = mutableMapOf() hosts.mapValuesTo(hostMetrics) { HostMetrics() } - mapDoubleSummary(metrics["cpu.demand"], hostMetrics) { m, v -> - m.cpuDemand = v - } - - mapDoubleSummary(metrics["cpu.usage"], hostMetrics) { m, v -> - m.cpuUsage = v - } - - mapDoubleGauge(metrics["power.usage"], hostMetrics) { m, v -> - m.powerDraw = v - } - - mapDoubleSummary(metrics["cpu.work.total"], hostMetrics) { m, v -> - m.requestedBurst = v.toLong() - } - - mapDoubleSummary(metrics["cpu.work.granted"], hostMetrics) { m, v -> - m.grantedBurst = v.toLong() - } - - mapDoubleSummary(metrics["cpu.work.overcommit"], hostMetrics) { m, v -> - m.overcommissionedBurst = v.toLong() - } - - mapDoubleSummary(metrics["cpu.work.interference"], hostMetrics) { m, v -> - m.interferedBurst = v.toLong() - } - - mapLongSum(metrics["guests.active"], hostMetrics) { m, v -> - m.numberOfDeployedImages = v.toInt() - } + mapDoubleSummary(metrics["cpu.demand"], hostMetrics) { m, v -> m.cpuDemand = v } + mapDoubleSummary(metrics["cpu.usage"], hostMetrics) { m, v -> m.cpuUsage = v } + mapDoubleGauge(metrics["power.usage"], hostMetrics) { m, v -> m.powerDraw = v } + mapDoubleSum(metrics["cpu.work.total"], hostMetrics) { m, v -> m.requestedBurst = v } + mapDoubleSum(metrics["cpu.work.granted"], hostMetrics) { m, v -> m.grantedBurst = v } + mapDoubleSum(metrics["cpu.work.overcommit"], hostMetrics) { m, v -> m.overcommissionedBurst = v } + mapDoubleSum(metrics["cpu.work.interference"], hostMetrics) { m, v -> m.interferedBurst = v } + mapLongSum(metrics["guests.active"], hostMetrics) { m, v -> m.numberOfDeployedImages = v.toInt() } for ((id, hostMetric) in hostMetrics) { + val lastHostMetric = lastHostMetrics.getOrDefault(id, hostMetricsSingleton) val host = hosts.getValue(id) monitor.reportHostSlice( clock.millis(), - hostMetric.requestedBurst, - hostMetric.grantedBurst, - hostMetric.overcommissionedBurst, - hostMetric.interferedBurst, + (hostMetric.requestedBurst - lastHostMetric.requestedBurst).toLong(), + (hostMetric.grantedBurst - lastHostMetric.grantedBurst).toLong(), + (hostMetric.overcommissionedBurst - lastHostMetric.overcommissionedBurst).toLong(), + (hostMetric.interferedBurst - lastHostMetric.interferedBurst).toLong(), hostMetric.cpuUsage, hostMetric.cpuDemand, hostMetric.powerDraw, @@ -97,6 +78,8 @@ public class ExperimentMetricExporter( host ) } + + lastHostMetrics = hostMetrics } private fun mapDoubleSummary(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { @@ -137,6 +120,18 @@ public class ExperimentMetricExporter( } } + private fun mapDoubleSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { + val points = data?.doubleSumData?.points ?: emptyList() + for (point in points) { + val uid = point.attributes[hostKey] + val hostMetric = hostMetrics[uid] + + if (hostMetric != null) { + block(hostMetric, point.value) + } + } + } + private fun reportProvisionerMetrics(metrics: Map) { val submittedVms = metrics["servers.submitted"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 val queuedVms = metrics["servers.waiting"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 @@ -159,10 +154,10 @@ public class ExperimentMetricExporter( } private class HostMetrics { - var requestedBurst: Long = 0 - var grantedBurst: Long = 0 - var overcommissionedBurst: Long = 0 - var interferedBurst: Long = 0 + var requestedBurst: Double = 0.0 + var grantedBurst: Double = 0.0 + var overcommissionedBurst: Double = 0.0 + var interferedBurst: Double = 0.0 var cpuUsage: Double = 0.0 var cpuDemand: Double = 0.0 var numberOfDeployedImages: Int = 0 diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt index 68631dee..79af88fe 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt @@ -55,8 +55,7 @@ public interface ExperimentMonitor : AutoCloseable { powerDraw: Double, numberOfDeployedImages: Int, host: Host - ) { - } + ) {} /** * This method is invoked for a provisioner event. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt index bfdf5f3e..d314c6f5 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt @@ -75,10 +75,10 @@ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: 5 * 60 * 1000L, host, numberOfDeployedImages, - requestedBurst, - grantedBurst, - overcommissionedBurst, - interferedBurst, + requestedBurst.toLong(), + grantedBurst.toLong(), + overcommissionedBurst.toLong(), + interferedBurst.toLong(), cpuUsage, cpuDemand, powerDraw, diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index a3300b71..9b98b329 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -120,9 +120,9 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(219751355711, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(206351165081, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(1148906334, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, + { assertEquals(220346369672, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, + { assertEquals(206667809431, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, + { assertEquals(1151611104, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } ) } @@ -160,9 +160,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(37954956986, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(34840774250, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(971076806, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(38051879542, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(34888186396, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(971668973, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } @@ -204,10 +204,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(37954956986, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(34840774250, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(971076806, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, - { assertEquals(13885404, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + { assertEquals(38051879542, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(34888186396, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(971668973, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(13910799, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } @@ -256,9 +256,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(25336984869, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(23668547517, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(368151656, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, + { assertEquals(25412073100, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, + { assertEquals(23695061847, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, + { assertEquals(368502468, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } ) } @@ -300,10 +300,10 @@ class CapelinIntegrationTest { numberOfDeployedImages: Int, host: Host, ) { - totalRequestedBurst += requestedBurst - totalGrantedBurst += grantedBurst - totalOvercommissionedBurst += overcommissionedBurst - totalInterferedBurst += interferedBurst + totalRequestedBurst += requestedBurst.toLong() + totalGrantedBurst += grantedBurst.toLong() + totalOvercommissionedBurst += overcommissionedBurst.toLong() + totalInterferedBurst += interferedBurst.toLong() } override fun close() {} diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index d8f92155..0873aac9 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -22,8 +22,9 @@ package org.opendc.experiments.tf20.core +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.metrics.Meter -import io.opentelemetry.api.metrics.common.Labels import kotlinx.coroutines.* import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine @@ -67,23 +68,28 @@ public class SimTFDevice( SimplePowerDriver(powerModel) ) + /** + * The identifier of a device. + */ + private val deviceId = AttributeKey.stringKey("device.id") + /** * The usage of the device. */ - private val _usage = meter.doubleValueRecorderBuilder("device.usage") + private val _usage = meter.histogramBuilder("device.usage") .setDescription("The amount of device resources used") .setUnit("MHz") .build() - .bind(Labels.of("device", uid.toString())) + .bind(Attributes.of(deviceId, uid.toString())) /** * The power draw of the device. */ - private val _power = meter.doubleValueRecorderBuilder("device.power") + private val _power = meter.histogramBuilder("device.power") .setDescription("The power draw of the device") .setUnit("W") .build() - .bind(Labels.of("device", uid.toString())) + .bind(Attributes.of(deviceId, uid.toString())) /** * The workload that will be run by the device. -- cgit v1.2.3 From bb6066e1cecc55a50ac29da200bf3beba1ddd80b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 18:16:20 +0200 Subject: fix(capelin): Eliminate unnecessary double to long conversions This change eliminates the unnecessary conversions from double to long in the Capelin metric processing code. --- .../capelin/monitor/ExperimentMetricExporter.kt | 12 ++--- .../capelin/monitor/ExperimentMonitor.kt | 16 +++--- .../capelin/monitor/ParquetExperimentMonitor.kt | 24 ++++----- .../experiments/capelin/CapelinIntegrationTest.kt | 60 +++++++++++----------- 4 files changed, 56 insertions(+), 56 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index 16358817..be94593c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -65,12 +65,12 @@ public class ExperimentMetricExporter( for ((id, hostMetric) in hostMetrics) { val lastHostMetric = lastHostMetrics.getOrDefault(id, hostMetricsSingleton) val host = hosts.getValue(id) - monitor.reportHostSlice( + monitor.reportHostData( clock.millis(), - (hostMetric.requestedBurst - lastHostMetric.requestedBurst).toLong(), - (hostMetric.grantedBurst - lastHostMetric.grantedBurst).toLong(), - (hostMetric.overcommissionedBurst - lastHostMetric.overcommissionedBurst).toLong(), - (hostMetric.interferedBurst - lastHostMetric.interferedBurst).toLong(), + hostMetric.requestedBurst - lastHostMetric.requestedBurst, + hostMetric.grantedBurst - lastHostMetric.grantedBurst, + hostMetric.overcommissionedBurst - lastHostMetric.overcommissionedBurst, + hostMetric.interferedBurst - lastHostMetric.interferedBurst, hostMetric.cpuUsage, hostMetric.cpuDemand, hostMetric.powerDraw, @@ -141,7 +141,7 @@ public class ExperimentMetricExporter( val hosts = metrics["hosts.total"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 val availableHosts = metrics["hosts.available"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - monitor.reportProvisionerMetrics( + monitor.reportServiceData( clock.millis(), hosts, availableHosts, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt index 79af88fe..9a4aec35 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt @@ -44,23 +44,23 @@ public interface ExperimentMonitor : AutoCloseable { /** * This method is invoked for a host for each slice that is finishes. */ - public fun reportHostSlice( + public fun reportHostData( time: Long, - requestedBurst: Long, - grantedBurst: Long, - overcommissionedBurst: Long, - interferedBurst: Long, + totalWork: Double, + grantedWork: Double, + overcommittedWork: Double, + interferedWork: Double, cpuUsage: Double, cpuDemand: Double, powerDraw: Double, - numberOfDeployedImages: Int, + instanceCount: Int, host: Host ) {} /** - * This method is invoked for a provisioner event. + * This method is invoked for reporting service data. */ - public fun reportProvisionerMetrics( + public fun reportServiceData( time: Long, totalHostCount: Int, availableHostCount: Int, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt index d314c6f5..83351c41 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt @@ -57,16 +57,16 @@ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: logger.debug { "Host ${host.uid} changed state $newState [$time]" } } - override fun reportHostSlice( + override fun reportHostData( time: Long, - requestedBurst: Long, - grantedBurst: Long, - overcommissionedBurst: Long, - interferedBurst: Long, + totalWork: Double, + grantedWork: Double, + overcommittedWork: Double, + interferedWork: Double, cpuUsage: Double, cpuDemand: Double, powerDraw: Double, - numberOfDeployedImages: Int, + instanceCount: Int, host: Host ) { hostWriter.write( @@ -74,11 +74,11 @@ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: time, 5 * 60 * 1000L, host, - numberOfDeployedImages, - requestedBurst.toLong(), - grantedBurst.toLong(), - overcommissionedBurst.toLong(), - interferedBurst.toLong(), + instanceCount, + totalWork.toLong(), + grantedWork.toLong(), + overcommittedWork.toLong(), + interferedWork.toLong(), cpuUsage, cpuDemand, powerDraw, @@ -87,7 +87,7 @@ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: ) } - override fun reportProvisionerMetrics( + override fun reportServiceData( time: Long, totalHostCount: Int, availableHostCount: Int, diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 9b98b329..8008c944 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -120,10 +120,10 @@ class CapelinIntegrationTest { { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, - { assertEquals(220346369672, monitor.totalRequestedBurst) { "Incorrect requested burst" } }, - { assertEquals(206667809431, monitor.totalGrantedBurst) { "Incorrect granted burst" } }, - { assertEquals(1151611104, monitor.totalOvercommissionedBurst) { "Incorrect overcommitted burst" } }, - { assertEquals(0, monitor.totalInterferedBurst) { "Incorrect interfered burst" } } + { assertEquals(220346369753, monitor.totalWork) { "Incorrect requested burst" } }, + { assertEquals(206667809529, monitor.totalGrantedWork) { "Incorrect granted burst" } }, + { assertEquals(1151611104, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, + { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } } ) } @@ -160,10 +160,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38051879542, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(34888186396, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(971668973, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, - { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + { assertEquals(38051879552, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(34888186408, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(971668973, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -204,10 +204,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38051879542, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(34888186396, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(971668973, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, - { assertEquals(13910799, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + { assertEquals(38051879552, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(34888186408, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(971668973, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(13910814, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -256,10 +256,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(25412073100, monitor.totalRequestedBurst) { "Total requested work incorrect" } }, - { assertEquals(23695061847, monitor.totalGrantedBurst) { "Total granted work incorrect" } }, - { assertEquals(368502468, monitor.totalOvercommissionedBurst) { "Total overcommitted work incorrect" } }, - { assertEquals(0, monitor.totalInterferedBurst) { "Total interfered work incorrect" } } + { assertEquals(25412073109, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(23695061858, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(368502468, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -283,27 +283,27 @@ class CapelinIntegrationTest { } class TestExperimentReporter : ExperimentMonitor { - var totalRequestedBurst = 0L - var totalGrantedBurst = 0L - var totalOvercommissionedBurst = 0L - var totalInterferedBurst = 0L + var totalWork = 0L + var totalGrantedWork = 0L + var totalOvercommittedWork = 0L + var totalInterferedWork = 0L - override fun reportHostSlice( + override fun reportHostData( time: Long, - requestedBurst: Long, - grantedBurst: Long, - overcommissionedBurst: Long, - interferedBurst: Long, + totalWork: Double, + grantedWork: Double, + overcommittedWork: Double, + interferedWork: Double, cpuUsage: Double, cpuDemand: Double, powerDraw: Double, - numberOfDeployedImages: Int, + instanceCount: Int, host: Host, ) { - totalRequestedBurst += requestedBurst.toLong() - totalGrantedBurst += grantedBurst.toLong() - totalOvercommissionedBurst += overcommissionedBurst.toLong() - totalInterferedBurst += interferedBurst.toLong() + this.totalWork += totalWork.toLong() + totalGrantedWork += grantedWork.toLong() + totalOvercommittedWork += overcommittedWork.toLong() + totalInterferedWork += interferedWork.toLong() } override fun close() {} -- cgit v1.2.3 From e5b79b18dab4f2874f3c5730b7e599dc74573c8d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 18:24:55 +0200 Subject: refactor(capelin): Simplify metric extraction for monitor This change updates the metric exporter logic in the Capelin module and reduces the number of allocations by looping directly over the collection of metrics. --- .../experiments/capelin/ExperimentHelpers.kt | 38 ++++++--- .../capelin/monitor/ExperimentMetricExporter.kt | 91 +++++++++++----------- 2 files changed, 74 insertions(+), 55 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 4cffb8d3..7f428b2a 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -24,6 +24,7 @@ package org.opendc.experiments.capelin import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel @@ -208,24 +209,41 @@ class ComputeMetrics { var runningVms: Int = 0 var unscheduledVms: Int = 0 var finishedVms: Int = 0 + var hosts: Int = 0 + var availableHosts = 0 } /** * Collect the metrics of the compute service. */ fun collectMetrics(metricProducer: MetricProducer): ComputeMetrics { - val metrics = metricProducer.collectAllMetrics().associateBy { it.name } + return extractComputeMetrics(metricProducer.collectAllMetrics()) +} + +/** + * Extract an [ComputeMetrics] object from the specified list of metric data. + */ +internal fun extractComputeMetrics(metrics: Collection): ComputeMetrics { val res = ComputeMetrics() - try { - // Hack to extract metrics from OpenTelemetry SDK - res.submittedVms = metrics["servers.submitted"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - res.queuedVms = metrics["servers.waiting"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - res.unscheduledVms = metrics["servers.unscheduled"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - res.runningVms = metrics["servers.active"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - res.finishedVms = metrics["servers.finished"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - } catch (cause: Throwable) { - logger.warn(cause) { "Failed to collect metrics" } + for (metric in metrics) { + val points = metric.longSumData.points + + if (points.isEmpty()) { + continue + } + + val value = points.first().value.toInt() + when (metric.name) { + "servers.submitted" -> res.submittedVms = value + "servers.waiting" -> res.queuedVms = value + "servers.unscheduled" -> res.unscheduledVms = value + "servers.active" -> res.runningVms = value + "servers.finished" -> res.finishedVms = value + "hosts.total" -> res.hosts = value + "hosts.available" -> res.availableHosts = value + } } + return res } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index be94593c..e9c817de 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.data.MetricData import io.opentelemetry.sdk.metrics.export.MetricExporter import io.opentelemetry.semconv.resource.attributes.ResourceAttributes import org.opendc.compute.service.driver.Host +import org.opendc.experiments.capelin.extractComputeMetrics import java.time.Clock /** @@ -37,44 +38,50 @@ public class ExperimentMetricExporter( private val clock: Clock, private val hosts: Map ) : MetricExporter { - private val hostKey = ResourceAttributes.HOST_ID override fun export(metrics: Collection): CompletableResultCode { - val metricsByName = metrics.associateBy { it.name } - reportHostMetrics(metricsByName) - reportProvisionerMetrics(metricsByName) - return CompletableResultCode.ofSuccess() + return try { + reportHostMetrics(metrics) + reportProvisionerMetrics(metrics) + CompletableResultCode.ofSuccess() + } catch (e: Throwable) { + CompletableResultCode.ofFailure() + } } private var lastHostMetrics: Map = emptyMap() private val hostMetricsSingleton = HostMetrics() - private fun reportHostMetrics(metrics: Map) { + private fun reportHostMetrics(metrics: Collection) { val hostMetrics = mutableMapOf() hosts.mapValuesTo(hostMetrics) { HostMetrics() } - mapDoubleSummary(metrics["cpu.demand"], hostMetrics) { m, v -> m.cpuDemand = v } - mapDoubleSummary(metrics["cpu.usage"], hostMetrics) { m, v -> m.cpuUsage = v } - mapDoubleGauge(metrics["power.usage"], hostMetrics) { m, v -> m.powerDraw = v } - mapDoubleSum(metrics["cpu.work.total"], hostMetrics) { m, v -> m.requestedBurst = v } - mapDoubleSum(metrics["cpu.work.granted"], hostMetrics) { m, v -> m.grantedBurst = v } - mapDoubleSum(metrics["cpu.work.overcommit"], hostMetrics) { m, v -> m.overcommissionedBurst = v } - mapDoubleSum(metrics["cpu.work.interference"], hostMetrics) { m, v -> m.interferedBurst = v } - mapLongSum(metrics["guests.active"], hostMetrics) { m, v -> m.numberOfDeployedImages = v.toInt() } + for (metric in metrics) { + when (metric.name) { + "cpu.demand" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuDemand = v } + "cpu.usage" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuUsage = v } + "power.usage" -> mapDoubleGauge(metric, hostMetrics) { m, v -> m.powerDraw = v } + "cpu.work.total" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.totalWork = v } + "cpu.work.granted" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.grantedWork = v } + "cpu.work.overcommit" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.overcommittedWork = v } + "cpu.work.interference" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.interferedWork = v } + "guests.active" -> mapLongSum(metric, hostMetrics) { m, v -> m.instanceCount = v.toInt() } + } + } for ((id, hostMetric) in hostMetrics) { val lastHostMetric = lastHostMetrics.getOrDefault(id, hostMetricsSingleton) val host = hosts.getValue(id) monitor.reportHostData( clock.millis(), - hostMetric.requestedBurst - lastHostMetric.requestedBurst, - hostMetric.grantedBurst - lastHostMetric.grantedBurst, - hostMetric.overcommissionedBurst - lastHostMetric.overcommissionedBurst, - hostMetric.interferedBurst - lastHostMetric.interferedBurst, + hostMetric.totalWork - lastHostMetric.totalWork, + hostMetric.grantedWork - lastHostMetric.grantedWork, + hostMetric.overcommittedWork - lastHostMetric.overcommittedWork, + hostMetric.interferedWork - lastHostMetric.interferedWork, hostMetric.cpuUsage, hostMetric.cpuDemand, hostMetric.powerDraw, - hostMetric.numberOfDeployedImages, + hostMetric.instanceCount, host ) } @@ -82,10 +89,10 @@ public class ExperimentMetricExporter( lastHostMetrics = hostMetrics } - private fun mapDoubleSummary(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { - val points = data?.doubleSummaryData?.points ?: emptyList() + private fun mapDoubleSummary(data: MetricData, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { + val points = data.doubleSummaryData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[hostKey] + val uid = point.attributes[ResourceAttributes.HOST_ID] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -99,7 +106,7 @@ public class ExperimentMetricExporter( private fun mapDoubleGauge(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data?.doubleGaugeData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[hostKey] + val uid = point.attributes[ResourceAttributes.HOST_ID] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -111,7 +118,7 @@ public class ExperimentMetricExporter( private fun mapLongSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Long) -> Unit) { val points = data?.longSumData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[hostKey] + val uid = point.attributes[ResourceAttributes.HOST_ID] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -123,7 +130,7 @@ public class ExperimentMetricExporter( private fun mapDoubleSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data?.doubleSumData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[hostKey] + val uid = point.attributes[ResourceAttributes.HOST_ID] val hostMetric = hostMetrics[uid] if (hostMetric != null) { @@ -132,35 +139,29 @@ public class ExperimentMetricExporter( } } - private fun reportProvisionerMetrics(metrics: Map) { - val submittedVms = metrics["servers.submitted"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val queuedVms = metrics["servers.waiting"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val unscheduledVms = metrics["servers.unscheduled"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val runningVms = metrics["servers.active"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val finishedVms = metrics["servers.finished"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val hosts = metrics["hosts.total"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 - val availableHosts = metrics["hosts.available"]?.longSumData?.points?.last()?.value?.toInt() ?: 0 + private fun reportProvisionerMetrics(metrics: Collection) { + val res = extractComputeMetrics(metrics) monitor.reportServiceData( clock.millis(), - hosts, - availableHosts, - submittedVms, - runningVms, - finishedVms, - queuedVms, - unscheduledVms + res.hosts, + res.availableHosts, + res.submittedVms, + res.runningVms, + res.finishedVms, + res.queuedVms, + res.unscheduledVms ) } private class HostMetrics { - var requestedBurst: Double = 0.0 - var grantedBurst: Double = 0.0 - var overcommissionedBurst: Double = 0.0 - var interferedBurst: Double = 0.0 + var totalWork: Double = 0.0 + var grantedWork: Double = 0.0 + var overcommittedWork: Double = 0.0 + var interferedWork: Double = 0.0 var cpuUsage: Double = 0.0 var cpuDemand: Double = 0.0 - var numberOfDeployedImages: Int = 0 + var instanceCount: Int = 0 var powerDraw: Double = 0.0 } -- cgit v1.2.3 From b0f6402f60ddbba1aad7e198fe6757792337f4d4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 20:46:36 +0200 Subject: refactor(compute): Measure power draw without PSU overhead This change updates the SimHost implementation to measure the power draw of the machine without PSU overhead to make the results more realistic. --- .../capelin/monitor/ExperimentMetricExporter.kt | 14 +------------- .../opendc/experiments/capelin/CapelinIntegrationTest.kt | 5 ++++- 2 files changed, 5 insertions(+), 14 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index e9c817de..42b7cbb8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -60,7 +60,7 @@ public class ExperimentMetricExporter( when (metric.name) { "cpu.demand" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuDemand = v } "cpu.usage" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuUsage = v } - "power.usage" -> mapDoubleGauge(metric, hostMetrics) { m, v -> m.powerDraw = v } + "power.usage" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.powerDraw = v } "cpu.work.total" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.totalWork = v } "cpu.work.granted" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.grantedWork = v } "cpu.work.overcommit" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.overcommittedWork = v } @@ -103,18 +103,6 @@ public class ExperimentMetricExporter( } } - private fun mapDoubleGauge(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { - val points = data?.doubleGaugeData?.points ?: emptyList() - for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] - val hostMetric = hostMetrics[uid] - - if (hostMetric != null) { - block(hostMetric, point.value) - } - } - } - private fun mapLongSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Long) -> Unit) { val points = data?.longSumData?.points ?: emptyList() for (point in points) { diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 8008c944..e4d3fed3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -123,7 +123,8 @@ class CapelinIntegrationTest { { assertEquals(220346369753, monitor.totalWork) { "Incorrect requested burst" } }, { assertEquals(206667809529, monitor.totalGrantedWork) { "Incorrect granted burst" } }, { assertEquals(1151611104, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, - { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } } + { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } }, + { assertEquals(1.7671768767192196E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, ) } @@ -287,6 +288,7 @@ class CapelinIntegrationTest { var totalGrantedWork = 0L var totalOvercommittedWork = 0L var totalInterferedWork = 0L + var totalPowerDraw = 0.0 override fun reportHostData( time: Long, @@ -304,6 +306,7 @@ class CapelinIntegrationTest { totalGrantedWork += grantedWork.toLong() totalOvercommittedWork += overcommittedWork.toLong() totalInterferedWork += interferedWork.toLong() + totalPowerDraw += powerDraw } override fun close() {} -- cgit v1.2.3 From e8cdfbcec3f75b3f303ce52bac5f5595a94555e4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:14:46 +0200 Subject: refactor(trace): Extract Parquet helpers into separate module This change extracts the Parquet helpers outside format module into a new module, in order to improve re-usability of these helpers. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + .../opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt | 2 +- .../org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt | 2 +- .../org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt | 2 +- .../main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 53643aba..e597c5ad 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcFormat) + implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcSimulator.opendcSimulatorFailures) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt index d8f7ff75..897a6692 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt @@ -28,7 +28,7 @@ import org.apache.avro.generic.GenericData import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.opendc.experiments.capelin.telemetry.Event -import org.opendc.format.util.LocalOutputFile +import org.opendc.trace.util.parquet.LocalOutputFile import java.io.Closeable import java.io.File import java.util.concurrent.ArrayBlockingQueue diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt index 16ad6816..2630784b 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -25,9 +25,9 @@ package org.opendc.experiments.capelin.trace import org.apache.avro.generic.GenericData import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader -import org.opendc.format.util.LocalParquetReader import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload +import org.opendc.trace.util.parquet.LocalParquetReader import java.io.File import java.util.UUID diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt index 35f4c5b8..9b5d0f47 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -32,9 +32,9 @@ import org.apache.parquet.filter2.predicate.UserDefinedPredicate import org.apache.parquet.io.api.Binary import org.opendc.format.trace.TraceEntry import org.opendc.format.trace.TraceReader -import org.opendc.format.util.LocalInputFile import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload +import org.opendc.trace.util.parquet.LocalInputFile import java.io.File import java.io.Serializable import java.util.SortedSet diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt index d7daa35b..e64f997f 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -37,8 +37,8 @@ import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.opendc.format.trace.bitbrains.BitbrainsTraceReader -import org.opendc.format.util.LocalOutputFile import org.opendc.simulator.compute.workload.SimTraceWorkload +import org.opendc.trace.util.parquet.LocalOutputFile import java.io.BufferedReader import java.io.File import java.io.FileReader -- cgit v1.2.3 From 9fcce6ade8714f7f0a9073fe5b7ddd3f0b35c375 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:44:47 +0200 Subject: refactor(format): Remove environment reader from format library This change removes the environment reader from the format library since they are highly specific for the particular experiment. In the future, we hope to have a single format to setup the entire datacenter (perhaps similar to the format used by the web runner). --- .../experiments/capelin/ExperimentHelpers.kt | 2 +- .../capelin/env/ClusterEnvironmentReader.kt | 2 -- .../experiments/capelin/env/EnvironmentReader.kt | 35 ++++++++++++++++++++ .../opendc/experiments/capelin/env/MachineDef.kt | 38 ++++++++++++++++++++++ .../capelin/trace/StreamingParquetTraceReader.kt | 2 +- .../experiments/capelin/CapelinIntegrationTest.kt | 2 +- .../opendc-experiments-tf20/build.gradle.kts | 5 ++- .../experiments/tf20/TensorFlowExperiment.kt | 3 +- .../experiments/tf20/util/MLEnvironmentReader.kt | 17 +++++----- .../org/opendc/experiments/tf20/util/MachineDef.kt | 38 ++++++++++++++++++++++ 10 files changed, 128 insertions(+), 16 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 7f428b2a..20dd603f 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -45,9 +45,9 @@ import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher import org.opendc.compute.service.scheduler.weights.RamWeigher import org.opendc.compute.service.scheduler.weights.VCpuWeigher import org.opendc.compute.simulator.SimHost +import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt index d73d14f5..babd8ada 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt @@ -22,8 +22,6 @@ package org.opendc.experiments.capelin.env -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt new file mode 100644 index 00000000..a968b043 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.env + +import java.io.Closeable + +/** + * An interface for reading descriptions of topology environments into memory. + */ +public interface EnvironmentReader : Closeable { + /** + * Read the environment into a list. + */ + public fun read(): List +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt new file mode 100644 index 00000000..b0c0318f --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.env + +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.power.PowerModel +import java.util.* + +/** + * A definition of a machine in a cluster. + */ +public data class MachineDef( + val uid: UUID, + val name: String, + val meta: Map, + val model: MachineModel, + val powerModel: PowerModel +) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt index 9b5d0f47..9bcbdc75 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -187,7 +187,7 @@ class StreamingParquetTraceReader(traceFile: File, selectedVms: List = e assert(uid !in takenIds) takenIds += uid - logger.info("Processing VM $id") + logger.info { "Processing VM $id" } val internalBuffer = mutableListOf() val externalBuffer = mutableListOf() diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index e4d3fed3..9d6329d1 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -36,12 +36,12 @@ import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher import org.opendc.experiments.capelin.env.ClusterEnvironmentReader +import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.format.environment.EnvironmentReader import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload diff --git a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts index b088045b..882c4894 100644 --- a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts @@ -34,8 +34,11 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcTelemetry.opendcTelemetrySdk) - implementation(projects.opendcFormat) implementation(projects.opendcUtils) implementation(libs.kotlin.logging) + implementation(libs.jackson.module.kotlin) { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") + } + implementation("org.jetbrains.kotlin:kotlin-reflect:1.5.30") } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt index 9a48aced..2153a862 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt @@ -55,7 +55,8 @@ public class TensorFlowExperiment : Experiment(name = "tf20") { .build() val meter = meterProvider.get("opendc-tf20") - val def = MLEnvironmentReader(TensorFlowExperiment::class.java.getResourceAsStream(environmentFile)).read().first() + val envInput = checkNotNull(TensorFlowExperiment::class.java.getResourceAsStream(environmentFile)) + val def = MLEnvironmentReader().readEnvironment(envInput).first() val device = SimTFDevice( def.uid, def.meta["gpu"] as Boolean, coroutineContext, clock, meter, def.model.cpus[0], def.model.memory[0], LinearPowerModel(250.0, 60.0) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt index 3e61f508..3cdf28fd 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt @@ -25,8 +25,6 @@ package org.opendc.experiments.tf20.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode @@ -36,13 +34,16 @@ import java.io.InputStream import java.util.* /** - * An [EnvironmentReader] for the TensorFlow experiments. + * An environment reader for the TensorFlow experiments. */ -public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jacksonObjectMapper()) : EnvironmentReader { +public class MLEnvironmentReader { + /** + * The [ObjectMapper] to convert the format. + */ + private val mapper = jacksonObjectMapper() - private val setup: Setup = mapper.readValue(input) - - override fun read(): List { + public fun readEnvironment(input: InputStream): List { + val setup: Setup = mapper.readValue(input) var counter = 0 return setup.rooms.flatMap { room -> room.objects.flatMap { roomObject -> @@ -109,6 +110,4 @@ public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jack } } } - - override fun close() {} } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt new file mode 100644 index 00000000..271f5923 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.util + +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.power.PowerModel +import java.util.* + +/** + * A definition of a machine in a cluster. + */ +public data class MachineDef( + val uid: UUID, + val name: String, + val meta: Map, + val model: MachineModel, + val powerModel: PowerModel +) -- cgit v1.2.3 From 214480d154771f0b783829b6e5ec82b837304ad2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 16:18:56 +0200 Subject: refactor(trace): Move Bitbrains format into separate module This change moves Bitbrains trace support into a separate module and adds support for the new trace api. --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../experiments/capelin/trace/TraceConverter.kt | 84 ++++++++++++++++------ 2 files changed, 63 insertions(+), 22 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index e597c5ad..97ca97ec 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcFormat) implementation(projects.opendcTrace.opendcTraceParquet) + implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcSimulator.opendcSimulatorFailures) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt index e64f997f..0ded32f3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -36,8 +36,8 @@ import org.apache.avro.generic.GenericData import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName -import org.opendc.format.trace.bitbrains.BitbrainsTraceReader -import org.opendc.simulator.compute.workload.SimTraceWorkload +import org.opendc.trace.* +import org.opendc.trace.bitbrains.BitbrainsTraceFormat import org.opendc.trace.util.parquet.LocalOutputFile import java.io.BufferedReader import java.io.File @@ -338,33 +338,73 @@ class BitbrainsConversion : TraceConversion("Bitbrains") { metaWriter: ParquetWriter ): MutableList { val fragments = mutableListOf() - BitbrainsTraceReader(traceDirectory).use { reader -> - reader.forEach { entry -> - val trace = (entry.workload as SimTraceWorkload).trace - var maxTime = Long.MIN_VALUE - trace.forEach { fragment -> - val flops: Long = (fragment.usage * fragment.duration / 1000).toLong() + val trace = BitbrainsTraceFormat().open(traceDirectory.toURI().toURL()) + val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() + + var lastId: String? = null + var maxCores = Int.MIN_VALUE + var requiredMemory = Long.MIN_VALUE + var minTime = Long.MAX_VALUE + var maxTime = Long.MIN_VALUE + var lastTimestamp = Long.MIN_VALUE + + while (reader.nextRow()) { + val id = reader.get(RESOURCE_STATE_ID) + + if (lastId != null && lastId != id) { + val metaRecord = GenericData.Record(metaSchema) + metaRecord.put("id", lastId) + metaRecord.put("submissionTime", minTime) + metaRecord.put("endTime", maxTime) + metaRecord.put("maxCores", maxCores) + metaRecord.put("requiredMemory", requiredMemory) + metaWriter.write(metaRecord) + } + lastId = id + + val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP) + val timestampMs = timestamp.toEpochMilli() + val cpuUsage = reader.getDouble(RESOURCE_STATE_MEM_USAGE) + val cores = reader.getInt(RESOURCE_STATE_NCPUS) + val memCapacity = reader.getDouble(RESOURCE_STATE_MEM_CAPACITY) + + maxCores = max(maxCores, cores) + requiredMemory = max(requiredMemory, (memCapacity / 1000).toLong()) + + if (lastTimestamp < 0) { + lastTimestamp = timestampMs - 5 * 60 * 1000L + minTime = min(minTime, lastTimestamp) + } + + if (fragments.isEmpty()) { + val duration = 5 * 60 * 1000L + val flops: Long = (cpuUsage * duration / 1000).toLong() + fragments.add(Fragment(id, lastTimestamp, flops, duration, cpuUsage, cores)) + } else { + val last = fragments.last() + val duration = timestampMs - lastTimestamp + val flops: Long = (cpuUsage * duration / 1000).toLong() + + // Perform run-length encoding + if (last.id == id && (duration == 0L || last.usage == cpuUsage)) { + fragments[fragments.size - 1] = last.copy(duration = last.duration + duration) + } else { fragments.add( Fragment( - entry.name, - fragment.timestamp, + id, + lastTimestamp, flops, - fragment.duration, - fragment.usage, - fragment.cores + duration, + cpuUsage, + cores ) ) - maxTime = max(maxTime, fragment.timestamp + fragment.duration) } - - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", entry.name) - metaRecord.put("submissionTime", entry.start) - metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", entry.meta["cores"]) - metaRecord.put("requiredMemory", entry.meta["required-memory"]) - metaWriter.write(metaRecord) } + + val last = fragments.last() + maxTime = max(maxTime, last.tick + last.duration) + lastTimestamp = timestampMs } return fragments } -- cgit v1.2.3 From b2308e1077dc60ec6a4dc646613a4be5b59695a6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 11:29:27 +0200 Subject: refactor(trace): Implement trace API for WTF reader This change updates the WTF trace reader to support the new streaming trace API. --- .../opendc-experiments-capelin/build.gradle.kts | 4 +- .../experiments/capelin/ExperimentHelpers.kt | 2 +- .../capelin/trace/ParquetTraceReader.kt | 2 - .../capelin/trace/RawParquetTraceReader.kt | 2 - .../capelin/trace/StreamingParquetTraceReader.kt | 2 - .../opendc/experiments/capelin/trace/TraceEntry.kt | 44 ++++++++++++++++++++++ .../experiments/capelin/trace/TraceReader.kt | 32 ++++++++++++++++ .../experiments/capelin/trace/WorkloadSampler.kt | 1 - .../experiments/capelin/CapelinIntegrationTest.kt | 2 +- .../opendc-experiments-energy21/build.gradle.kts | 1 - 10 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 97ca97ec..65cebe1b 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -31,7 +31,6 @@ plugins { dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) - implementation(projects.opendcFormat) implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) @@ -45,6 +44,9 @@ dependencies { implementation(libs.config) implementation(libs.progressbar) implementation(libs.clikt) + implementation(libs.jackson.module.kotlin) { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") + } implementation(libs.parquet) testImplementation(libs.log4j.slf4j) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 20dd603f..46e11056 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -48,7 +48,7 @@ import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.format.trace.TraceReader +import org.opendc.experiments.capelin.trace.TraceReader import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.power.SimplePowerDriver diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt index 0f49ecd2..0bf4ada6 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt @@ -24,8 +24,6 @@ package org.opendc.experiments.capelin.trace import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.Workload -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.workload.SimWorkload /** diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt index 2630784b..24ff0ba1 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -23,8 +23,6 @@ package org.opendc.experiments.capelin.trace import org.apache.avro.generic.GenericData -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.trace.util.parquet.LocalParquetReader diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt index 9bcbdc75..61e4cab5 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -30,8 +30,6 @@ import org.apache.parquet.filter2.predicate.FilterApi import org.apache.parquet.filter2.predicate.Statistics import org.apache.parquet.filter2.predicate.UserDefinedPredicate import org.apache.parquet.io.api.Binary -import org.opendc.format.trace.TraceEntry -import org.opendc.format.trace.TraceReader import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.trace.util.parquet.LocalInputFile diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt new file mode 100644 index 00000000..303a6a8c --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2019 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +import java.util.UUID + +/** + * An entry in a workload trace. + * + * @param uid The unique identifier of the entry. + * @param name The name of the entry. + * @param start The start time of the workload. + * @param workload The workload of the entry. + * @param meta The meta-data associated with the workload. + */ +public data class TraceEntry( + val uid: UUID, + val name: String, + val start: Long, + val workload: T, + val meta: Map +) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt new file mode 100644 index 00000000..08304edc --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace + +/** + * An interface for reading workloads into memory. + * + * This interface must guarantee that the entries are delivered in order of submission time. + * + * @param T The shape of the workloads supported by this reader. + */ +public interface TraceReader : Iterator>, AutoCloseable diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt index 6de3f265..cb32ce88 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt @@ -26,7 +26,6 @@ import mu.KotlinLogging import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.SamplingStrategy import org.opendc.experiments.capelin.model.Workload -import org.opendc.format.trace.TraceEntry import org.opendc.simulator.compute.workload.SimWorkload import java.util.* import kotlin.random.Random diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 9d6329d1..2934bbe6 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -42,7 +42,7 @@ import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.format.trace.TraceReader +import org.opendc.experiments.capelin.trace.TraceReader import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation diff --git a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts index bc05f09b..7d34d098 100644 --- a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts @@ -31,7 +31,6 @@ plugins { dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) - implementation(projects.opendcFormat) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcSimulator.opendcSimulatorFailures) -- cgit v1.2.3 From 8bae0f3053a53aac9d483ae97d99f2e7e80b42ef Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 22:30:39 +0200 Subject: refactor(capelin): Migrate trace reader to new trace API This change updates the trace reading classes in the Capelin experiment to use the new trace API in order to re-use many of the trace reading parts. --- .../capelin/trace/RawParquetTraceReader.kt | 62 +++--- .../experiments/capelin/trace/TraceConverter.kt | 195 ++--------------- .../capelin/trace/bp/BPResourceStateTable.kt | 56 +++++ .../capelin/trace/bp/BPResourceStateTableReader.kt | 103 +++++++++ .../capelin/trace/bp/BPResourceTable.kt | 56 +++++ .../capelin/trace/bp/BPResourceTableReader.kt | 103 +++++++++ .../opendc/experiments/capelin/trace/bp/BPTrace.kt | 49 +++++ .../experiments/capelin/trace/bp/BPTraceFormat.kt | 47 +++++ .../capelin/trace/sv/SvResourceStateTable.kt | 140 +++++++++++++ .../capelin/trace/sv/SvResourceStateTableReader.kt | 230 +++++++++++++++++++++ .../opendc/experiments/capelin/trace/sv/SvTrace.kt | 45 ++++ .../experiments/capelin/trace/sv/SvTraceFormat.kt | 47 +++++ 12 files changed, 927 insertions(+), 206 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt index 24ff0ba1..fa4e9ed8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -22,10 +22,10 @@ package org.opendc.experiments.capelin.trace -import org.apache.avro.generic.GenericData +import org.opendc.experiments.capelin.trace.bp.BPTraceFormat import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.trace.util.parquet.LocalParquetReader +import org.opendc.trace.* import java.io.File import java.util.UUID @@ -35,27 +35,30 @@ import java.util.UUID * @param path The directory of the traces. */ class RawParquetTraceReader(private val path: File) { + /** + * The [Trace] that represents this trace. + */ + private val trace = BPTraceFormat().open(path.toURI().toURL()) + /** * Read the fragments into memory. */ - private fun parseFragments(path: File): Map> { - val reader = LocalParquetReader(File(path, "trace.parquet")) + private fun parseFragments(): Map> { + val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() val fragments = mutableMapOf>() return try { - while (true) { - val record = reader.read() ?: break - - val id = record["id"].toString() - val time = record["time"] as Long - val duration = record["duration"] as Long - val cores = record["cores"] as Int - val cpuUsage = record["cpuUsage"] as Double + while (reader.nextRow()) { + val id = reader.get(RESOURCE_STATE_ID) + val time = reader.get(RESOURCE_STATE_TIMESTAMP) + val duration = reader.get(RESOURCE_STATE_DURATION) + val cores = reader.getInt(RESOURCE_STATE_NCPUS) + val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) val fragment = SimTraceWorkload.Fragment( - time, - duration, + time.toEpochMilli(), + duration.toMillis(), cpuUsage, cores ) @@ -72,25 +75,24 @@ class RawParquetTraceReader(private val path: File) { /** * Read the metadata into a workload. */ - private fun parseMeta(path: File, fragments: Map>): List> { - val metaReader = LocalParquetReader(File(path, "meta.parquet")) + private fun parseMeta(fragments: Map>): List> { + val reader = checkNotNull(trace.getTable(TABLE_RESOURCES)).newReader() var counter = 0 val entries = mutableListOf>() return try { - while (true) { - val record = metaReader.read() ?: break + while (reader.nextRow()) { - val id = record["id"].toString() + val id = reader.get(RESOURCE_ID) if (!fragments.containsKey(id)) { continue } - val submissionTime = record["submissionTime"] as Long - val endTime = record["endTime"] as Long - val maxCores = record["maxCores"] as Int - val requiredMemory = record["requiredMemory"] as Long + val submissionTime = reader.get(RESOURCE_START_TIME) + val endTime = reader.get(RESOURCE_END_TIME) + val maxCores = reader.getInt(RESOURCE_NCPUS) + val requiredMemory = reader.getDouble(RESOURCE_MEM_CAPACITY) val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) val vmFragments = fragments.getValue(id).asSequence() @@ -98,13 +100,13 @@ class RawParquetTraceReader(private val path: File) { val workload = SimTraceWorkload(vmFragments) entries.add( TraceEntry( - uid, id, submissionTime, workload, + uid, id, submissionTime.toEpochMilli(), workload, mapOf( - "submit-time" to submissionTime, - "end-time" to endTime, + "submit-time" to submissionTime.toEpochMilli(), + "end-time" to endTime.toEpochMilli(), "total-load" to totalLoad, "cores" to maxCores, - "required-memory" to requiredMemory, + "required-memory" to requiredMemory.toLong(), "workload" to workload ) ) @@ -116,7 +118,7 @@ class RawParquetTraceReader(private val path: File) { e.printStackTrace() throw e } finally { - metaReader.close() + reader.close() } } @@ -126,8 +128,8 @@ class RawParquetTraceReader(private val path: File) { private val entries: List> init { - val fragments = parseFragments(path) - entries = parseMeta(path, fragments) + val fragments = parseFragments() + entries = parseMeta(fragments) } /** diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt index 0ded32f3..a021de8d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -36,8 +36,10 @@ import org.apache.avro.generic.GenericData import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.opendc.experiments.capelin.trace.sv.SvTraceFormat import org.opendc.trace.* import org.opendc.trace.bitbrains.BitbrainsTraceFormat +import org.opendc.trace.spi.TraceFormat import org.opendc.trace.util.parquet.LocalOutputFile import java.io.BufferedReader import java.io.File @@ -156,189 +158,19 @@ sealed class TraceConversion(name: String) : OptionGroup(name) { ): MutableList } -class SolvinityConversion : TraceConversion("Solvinity") { - private val clusters by option() - .split(",") - - private val vmPlacements by option("--vm-placements", help = "file containing the VM placements") - .file(canBeDir = false) - .convert { VmPlacementReader(it.inputStream()).use { reader -> reader.read() } } - .required() - - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val clusters = clusters?.toSet() ?: emptySet() - val timestampCol = 0 - val cpuUsageCol = 1 - val coreCol = 12 - val provisionedMemoryCol = 20 - val traceInterval = 5 * 60 * 1000L - - // Identify start time of the entire trace - var minTimestamp = Long.MAX_VALUE - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach file@{ vmFile -> - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val vmId = vmFile.name - - // Check if VM in topology - val clusterName = vmPlacements[vmId] - if (clusterName == null || !clusters.contains(clusterName)) { - continue - } - - val values = line.split("\t") - val timestamp = (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - - if (timestamp < minTimestamp) { - minTimestamp = timestamp - } - return@file - } - } - } - } - - println("Start of trace at $minTimestamp") - - val allFragments = mutableListOf() - - val begin = 15 * 24 * 60 * 60 * 1000L - val end = 45 * 24 * 60 * 60 * 1000L - - traceDirectory.walk() - .filterNot { it.isDirectory } - .filter { it.extension == "csv" || it.extension == "txt" } - .toList() - .forEach { vmFile -> - println(vmFile) - - var vmId = "" - var maxCores = -1 - var requiredMemory = -1L - var cores: Int - var minTime = Long.MAX_VALUE - - val flopsFragments = sequence { - var last: Fragment? = null - - BufferedReader(FileReader(vmFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split("\t") - - vmId = vmFile.name - - // Check if VM in topology - val clusterName = vmPlacements[vmId] - if (clusterName == null || !clusters.contains(clusterName)) { - continue - } - - val timestamp = - (values[timestampCol].trim().toLong() - 5 * 60) * 1000L - minTimestamp - if (begin > timestamp || timestamp > end) { - continue - } - - cores = values[coreCol].trim().toInt() - requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) - maxCores = max(maxCores, cores) - minTime = min(minTime, timestamp) - val cpuUsage = values[cpuUsageCol].trim().toDouble() // MHz - requiredMemory = max(requiredMemory, values[provisionedMemoryCol].trim().toLong()) - maxCores = max(maxCores, cores) - - val flops: Long = (cpuUsage * 5 * 60).toLong() - - last = if (last != null && last!!.flops == 0L && flops == 0L) { - val oldFragment = last!! - Fragment( - vmId, - oldFragment.tick, - oldFragment.flops + flops, - oldFragment.duration + traceInterval, - cpuUsage, - cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - cores - ) - if (last != null) { - yield(last!!) - } - fragment - } - } - } - } - - if (last != null) { - yield(last!!) - } - } - - var maxTime = Long.MIN_VALUE - flopsFragments.filter { it.tick in begin until end }.forEach { fragment -> - allFragments.add(fragment) - maxTime = max(maxTime, fragment.tick) - } - - if (minTime in begin until end) { - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", vmId) - metaRecord.put("submissionTime", minTime) - metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", maxCores) - metaRecord.put("requiredMemory", requiredMemory) - metaWriter.write(metaRecord) - } - } - - return allFragments - } -} - /** - * Conversion of the Bitbrains public trace. + * A [TraceConversion] that uses the Trace API to perform the conversion. */ -class BitbrainsConversion : TraceConversion("Bitbrains") { +abstract class AbstractConversion(name: String) : TraceConversion(name) { + abstract val format: TraceFormat + override fun read( traceDirectory: File, metaSchema: Schema, metaWriter: ParquetWriter ): MutableList { val fragments = mutableListOf() - val trace = BitbrainsTraceFormat().open(traceDirectory.toURI().toURL()) + val trace = format.open(traceDirectory.toURI().toURL()) val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() var lastId: String? = null @@ -364,7 +196,7 @@ class BitbrainsConversion : TraceConversion("Bitbrains") { val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP) val timestampMs = timestamp.toEpochMilli() - val cpuUsage = reader.getDouble(RESOURCE_STATE_MEM_USAGE) + val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) val cores = reader.getInt(RESOURCE_STATE_NCPUS) val memCapacity = reader.getDouble(RESOURCE_STATE_MEM_CAPACITY) @@ -410,6 +242,17 @@ class BitbrainsConversion : TraceConversion("Bitbrains") { } } +class SolvinityConversion : AbstractConversion("Solvinity") { + override val format: TraceFormat = SvTraceFormat() +} + +/** + * Conversion of the Bitbrains public trace. + */ +class BitbrainsConversion : AbstractConversion("Bitbrains") { + override val format: TraceFormat = BitbrainsTraceFormat() +} + /** * Conversion of the Azure public VM trace. */ diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt new file mode 100644 index 00000000..35bfd5ef --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.apache.avro.generic.GenericRecord +import org.opendc.trace.* +import org.opendc.trace.util.parquet.LocalParquetReader +import java.nio.file.Path + +/** + * The resource state [Table] in the Bitbrains Parquet format. + */ +internal class BPResourceStateTable(private val path: Path) : Table { + override val name: String = TABLE_RESOURCE_STATES + override val isSynthetic: Boolean = false + + override fun isSupported(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_DURATION -> true + RESOURCE_STATE_NCPUS -> true + RESOURCE_STATE_CPU_USAGE -> true + else -> false + } + } + + override fun newReader(): TableReader { + val reader = LocalParquetReader(path.resolve("trace.parquet")) + return BPResourceStateTableReader(reader) + } + + override fun newReader(partition: String): TableReader { + throw IllegalArgumentException("Unknown partition $partition") + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt new file mode 100644 index 00000000..0e7ee555 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.apache.avro.generic.GenericRecord +import org.opendc.trace.* +import org.opendc.trace.util.parquet.LocalParquetReader +import java.time.Duration +import java.time.Instant + +/** + * A [TableReader] implementation for the Bitbrains Parquet format. + */ +internal class BPResourceStateTableReader(private val reader: LocalParquetReader) : TableReader { + /** + * The current record. + */ + private var record: GenericRecord? = null + + override fun nextRow(): Boolean { + record = reader.read() + return record != null + } + + override fun hasColumn(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_DURATION -> true + RESOURCE_STATE_NCPUS -> true + RESOURCE_STATE_CPU_USAGE -> true + else -> false + } + } + + override fun get(column: TableColumn): T { + val record = checkNotNull(record) { "Reader in invalid state" } + + @Suppress("UNCHECKED_CAST") + val res: Any = when (column) { + RESOURCE_STATE_ID -> record["id"].toString() + RESOURCE_STATE_TIMESTAMP -> Instant.ofEpochMilli(record["time"] as Long) + RESOURCE_STATE_DURATION -> Duration.ofMillis(record["duration"] as Long) + RESOURCE_STATE_NCPUS -> record["cores"] + RESOURCE_STATE_CPU_USAGE -> (record["cpuUsage"] as Number).toDouble() + else -> throw IllegalArgumentException("Invalid column") + } + + @Suppress("UNCHECKED_CAST") + return res as T + } + + override fun getBoolean(column: TableColumn): Boolean { + throw IllegalArgumentException("Invalid column") + } + + override fun getInt(column: TableColumn): Int { + val record = checkNotNull(record) { "Reader in invalid state" } + + return when (column) { + RESOURCE_STATE_NCPUS -> record["cores"] as Int + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getLong(column: TableColumn): Long { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(column: TableColumn): Double { + val record = checkNotNull(record) { "Reader in invalid state" } + return when (column) { + RESOURCE_STATE_CPU_USAGE -> (record["cpuUsage"] as Number).toDouble() + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun close() { + reader.close() + } + + override fun toString(): String = "BPResourceStateTableReader" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt new file mode 100644 index 00000000..74d1e574 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.apache.avro.generic.GenericRecord +import org.opendc.trace.* +import org.opendc.trace.util.parquet.LocalParquetReader +import java.nio.file.Path + +/** + * The resource [Table] in the Bitbrains Parquet format. + */ +internal class BPResourceTable(private val path: Path) : Table { + override val name: String = TABLE_RESOURCES + override val isSynthetic: Boolean = false + + override fun isSupported(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_ID -> true + RESOURCE_START_TIME -> true + RESOURCE_END_TIME -> true + RESOURCE_NCPUS -> true + RESOURCE_MEM_CAPACITY -> true + else -> false + } + } + + override fun newReader(): TableReader { + val reader = LocalParquetReader(path.resolve("meta.parquet")) + return BPResourceTableReader(reader) + } + + override fun newReader(partition: String): TableReader { + throw IllegalArgumentException("Unknown partition $partition") + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt new file mode 100644 index 00000000..0a105783 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.apache.avro.generic.GenericRecord +import org.opendc.trace.* +import org.opendc.trace.util.parquet.LocalParquetReader +import java.time.Instant + +/** + * A [TableReader] implementation for the Bitbrains Parquet format. + */ +internal class BPResourceTableReader(private val reader: LocalParquetReader) : TableReader { + /** + * The current record. + */ + private var record: GenericRecord? = null + + override fun nextRow(): Boolean { + record = reader.read() + return record != null + } + + override fun hasColumn(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_ID -> true + RESOURCE_START_TIME -> true + RESOURCE_END_TIME -> true + RESOURCE_NCPUS -> true + RESOURCE_MEM_CAPACITY -> true + else -> false + } + } + + override fun get(column: TableColumn): T { + val record = checkNotNull(record) { "Reader in invalid state" } + + @Suppress("UNCHECKED_CAST") + val res: Any = when (column) { + RESOURCE_ID -> record["id"].toString() + RESOURCE_START_TIME -> Instant.ofEpochMilli(record["submissionTime"] as Long) + RESOURCE_END_TIME -> Instant.ofEpochMilli(record["endTime"] as Long) + RESOURCE_NCPUS -> record["maxCores"] + RESOURCE_MEM_CAPACITY -> (record["requiredMemory"] as Number).toDouble() + else -> throw IllegalArgumentException("Invalid column") + } + + @Suppress("UNCHECKED_CAST") + return res as T + } + + override fun getBoolean(column: TableColumn): Boolean { + throw IllegalArgumentException("Invalid column") + } + + override fun getInt(column: TableColumn): Int { + val record = checkNotNull(record) { "Reader in invalid state" } + + return when (column) { + RESOURCE_NCPUS -> record["maxCores"] as Int + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getLong(column: TableColumn): Long { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(column: TableColumn): Double { + val record = checkNotNull(record) { "Reader in invalid state" } + + return when (column) { + RESOURCE_MEM_CAPACITY -> (record["requiredMemory"] as Number).toDouble() + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun close() { + reader.close() + } + + override fun toString(): String = "BPResourceTableReader" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt new file mode 100644 index 00000000..486587b1 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.opendc.trace.TABLE_RESOURCES +import org.opendc.trace.TABLE_RESOURCE_STATES +import org.opendc.trace.Table +import org.opendc.trace.Trace +import java.nio.file.Path + +/** + * A [Trace] in the Bitbrains Parquet format. + */ +public class BPTrace internal constructor(private val path: Path) : Trace { + override val tables: List = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES) + + override fun containsTable(name: String): Boolean = + name == TABLE_RESOURCES || name == TABLE_RESOURCE_STATES + + override fun getTable(name: String): Table? { + return when (name) { + TABLE_RESOURCES -> BPResourceTable(path) + TABLE_RESOURCE_STATES -> BPResourceStateTable(path) + else -> null + } + } + + override fun toString(): String = "BPTrace[$path]" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt new file mode 100644 index 00000000..49d5b4c5 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.opendc.trace.spi.TraceFormat +import java.net.URL +import java.nio.file.Paths +import kotlin.io.path.exists + +/** + * A format implementation for the GWF trace format. + */ +public class BPTraceFormat : TraceFormat { + /** + * The name of this trace format. + */ + override val name: String = "bitbrains-parquet" + + /** + * Open a Bitbrains Parquet trace. + */ + override fun open(url: URL): BPTrace { + val path = Paths.get(url.toURI()) + require(path.exists()) { "URL $url does not exist" } + return BPTrace(path) + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt new file mode 100644 index 00000000..71c9d52e --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.sv + +import org.opendc.trace.* +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import kotlin.io.path.bufferedReader +import kotlin.io.path.extension +import kotlin.io.path.nameWithoutExtension + +/** + * The resource state [Table] in the Bitbrains format. + */ +internal class SvResourceStateTable(path: Path) : Table { + /** + * The partitions that belong to the table. + */ + private val partitions = Files.walk(path, 1) + .filter { !Files.isDirectory(it) && it.extension == "txt" } + .collect(Collectors.toMap({ it.nameWithoutExtension }, { it })) + + override val name: String = TABLE_RESOURCE_STATES + + override val isSynthetic: Boolean = false + + override fun isSupported(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_CLUSTER_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_NCPUS -> true + RESOURCE_STATE_CPU_CAPACITY -> true + RESOURCE_STATE_CPU_USAGE -> true + RESOURCE_STATE_CPU_USAGE_PCT -> true + RESOURCE_STATE_CPU_DEMAND -> true + RESOURCE_STATE_CPU_READY_PCT -> true + RESOURCE_STATE_MEM_CAPACITY -> true + RESOURCE_STATE_DISK_READ -> true + RESOURCE_STATE_DISK_WRITE -> true + else -> false + } + } + + override fun newReader(): TableReader { + val it = partitions.iterator() + + return object : TableReader { + var delegate: TableReader? = nextDelegate() + + override fun nextRow(): Boolean { + var delegate = delegate + + while (delegate != null) { + if (delegate.nextRow()) { + break + } + + delegate.close() + delegate = nextDelegate() + } + + this.delegate = delegate + return delegate != null + } + + override fun hasColumn(column: TableColumn<*>): Boolean = delegate?.hasColumn(column) ?: false + + override fun get(column: TableColumn): T { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.get(column) + } + + override fun getBoolean(column: TableColumn): Boolean { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getBoolean(column) + } + + override fun getInt(column: TableColumn): Int { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getInt(column) + } + + override fun getLong(column: TableColumn): Long { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getLong(column) + } + + override fun getDouble(column: TableColumn): Double { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getDouble(column) + } + + override fun close() { + delegate?.close() + } + + private fun nextDelegate(): TableReader? { + return if (it.hasNext()) { + val (partition, path) = it.next() + val reader = path.bufferedReader() + return SvResourceStateTableReader(partition, reader) + } else { + null + } + } + + override fun toString(): String = "BitbrainsCompositeTableReader" + } + } + + override fun newReader(partition: String): TableReader { + val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" } + val reader = path.bufferedReader() + return SvResourceStateTableReader(partition, reader) + } + + override fun toString(): String = "SvResourceStateTable" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt new file mode 100644 index 00000000..adcdb2ea --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.sv + +import org.opendc.trace.* +import java.io.BufferedReader +import java.time.Instant + +/** + * A [TableReader] for the Bitbrains resource state table. + */ +internal class SvResourceStateTableReader(partition: String, private val reader: BufferedReader) : TableReader { + /** + * The current parser state. + */ + private val state = RowState() + + override fun nextRow(): Boolean { + state.reset() + + var line: String + var num = 0 + + while (true) { + line = reader.readLine() ?: return false + num++ + + if (line[0] == '#' || line.isBlank()) { + // Ignore empty lines or comments + continue + } + + break + } + + line = line.trim() + + val length = line.length + var col = 0 + var start = 0 + var end = 0 + + while (end < length) { + // Trim all whitespace before the field + start = end + while (start < length && line[start].isWhitespace()) { + start++ + } + + end = line.indexOf(' ', start) + + if (end < 0) { + break + } + + val field = line.subSequence(start, end) as String + when (col++) { + COL_TIMESTAMP -> state.timestamp = Instant.ofEpochSecond(field.toLong(10)) + COL_CPU_USAGE -> state.cpuUsage = field.toDouble() + COL_CPU_DEMAND -> state.cpuDemand = field.toDouble() + COL_DISK_READ -> state.diskRead = field.toDouble() + COL_DISK_WRITE -> state.diskWrite = field.toDouble() + COL_CLUSTER_ID -> state.cluster = field.trim() + COL_NCPUS -> state.cpuCores = field.toInt(10) + COL_CPU_READY_PCT -> state.cpuReadyPct = field.toDouble() + COL_POWERED_ON -> state.poweredOn = field.toInt(10) == 1 + COL_CPU_CAPACITY -> state.cpuCapacity = field.toDouble() + COL_ID -> state.id = field.trim() + COL_MEM_CAPACITY -> state.memCapacity = field.toDouble() + } + } + + return true + } + + override fun hasColumn(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_CLUSTER_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_NCPUS -> true + RESOURCE_STATE_CPU_CAPACITY -> true + RESOURCE_STATE_CPU_USAGE -> true + RESOURCE_STATE_CPU_USAGE_PCT -> true + RESOURCE_STATE_CPU_DEMAND -> true + RESOURCE_STATE_CPU_READY_PCT -> true + RESOURCE_STATE_MEM_CAPACITY -> true + RESOURCE_STATE_DISK_READ -> true + RESOURCE_STATE_DISK_WRITE -> true + else -> false + } + } + + override fun get(column: TableColumn): T { + val res: Any? = when (column) { + RESOURCE_STATE_ID -> state.id + RESOURCE_STATE_CLUSTER_ID -> state.cluster + RESOURCE_STATE_TIMESTAMP -> state.timestamp + RESOURCE_STATE_NCPUS -> state.cpuCores + RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity + RESOURCE_STATE_CPU_USAGE -> state.cpuUsage + RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsage / state.cpuCapacity + RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity + RESOURCE_STATE_DISK_READ -> state.diskRead + RESOURCE_STATE_DISK_WRITE -> state.diskWrite + else -> throw IllegalArgumentException("Invalid column") + } + + @Suppress("UNCHECKED_CAST") + return res as T + } + + override fun getBoolean(column: TableColumn): Boolean { + return when (column) { + RESOURCE_STATE_POWERED_ON -> state.poweredOn + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getInt(column: TableColumn): Int { + return when (column) { + RESOURCE_STATE_NCPUS -> state.cpuCores + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getLong(column: TableColumn): Long { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(column: TableColumn): Double { + return when (column) { + RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity + RESOURCE_STATE_CPU_USAGE -> state.cpuUsage + RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsage / state.cpuCapacity + RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity + RESOURCE_STATE_DISK_READ -> state.diskRead + RESOURCE_STATE_DISK_WRITE -> state.diskWrite + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun close() { + reader.close() + } + + /** + * The current row state. + */ + private class RowState { + @JvmField + var id: String? = null + @JvmField + var cluster: String? = null + @JvmField + var timestamp: Instant? = null + @JvmField + var cpuCores = -1 + @JvmField + var cpuCapacity = Double.NaN + @JvmField + var cpuUsage = Double.NaN + @JvmField + var cpuDemand = Double.NaN + @JvmField + var cpuReadyPct = Double.NaN + @JvmField + var memCapacity = Double.NaN + @JvmField + var diskRead = Double.NaN + @JvmField + var diskWrite = Double.NaN + @JvmField + var poweredOn: Boolean = false + + /** + * Reset the state. + */ + fun reset() { + id = null + timestamp = null + cluster = null + cpuCores = -1 + cpuCapacity = Double.NaN + cpuUsage = Double.NaN + cpuDemand = Double.NaN + cpuReadyPct = Double.NaN + memCapacity = Double.NaN + diskRead = Double.NaN + diskWrite = Double.NaN + poweredOn = false + } + } + + /** + * Default column indices for the extended Bitbrains format. + */ + private val COL_TIMESTAMP = 0 + private val COL_CPU_USAGE = 1 + private val COL_CPU_DEMAND = 2 + private val COL_DISK_READ = 4 + private val COL_DISK_WRITE = 6 + private val COL_CLUSTER_ID = 10 + private val COL_NCPUS = 12 + private val COL_CPU_READY_PCT = 13 + private val COL_POWERED_ON = 14 + private val COL_CPU_CAPACITY = 18 + private val COL_ID = 19 + private val COL_MEM_CAPACITY = 20 +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt new file mode 100644 index 00000000..dbd63de5 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.sv + +import org.opendc.trace.* +import java.nio.file.Path + +/** + * [Trace] implementation for the extended Bitbrains format. + */ +public class SvTrace internal constructor(private val path: Path) : Trace { + override val tables: List = listOf(TABLE_RESOURCE_STATES) + + override fun containsTable(name: String): Boolean = TABLE_RESOURCE_STATES == name + + override fun getTable(name: String): Table? { + if (!containsTable(name)) { + return null + } + + return SvResourceStateTable(path) + } + + override fun toString(): String = "SvTrace[$path]" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt new file mode 100644 index 00000000..0cce8559 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.sv + +import org.opendc.trace.spi.TraceFormat +import java.net.URL +import java.nio.file.Paths +import kotlin.io.path.exists + +/** + * A format implementation for the extended Bitbrains trace format. + */ +public class SvTraceFormat : TraceFormat { + /** + * The name of this trace format. + */ + override val name: String = "sv" + + /** + * Open the trace file. + */ + override fun open(url: URL): SvTrace { + val path = Paths.get(url.toURI()) + require(path.exists()) { "URL $url does not exist" } + return SvTrace(path) + } +} -- cgit v1.2.3 From 289cd3b6bc9d86b017dbb1ce6c50d346841a0ee6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 26 Aug 2021 10:33:42 +0200 Subject: refactor(capelin): Make ExperimentMonitor optional for trace processing --- .../main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 46e11056..9d23a5dd 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -255,7 +255,7 @@ suspend fun processTrace( reader: TraceReader, scheduler: ComputeService, chan: Channel, - monitor: ExperimentMonitor + monitor: ExperimentMonitor? = null, ) { val client = scheduler.newClient() val image = client.newImage("vm-image") @@ -289,7 +289,7 @@ suspend fun processTrace( suspendCancellableCoroutine { cont -> server.watch(object : ServerWatcher { override fun onStateChanged(server: Server, newState: ServerState) { - monitor.reportVmStateChange(clock.millis(), server, newState) + monitor?.reportVmStateChange(clock.millis(), server, newState) if (newState == ServerState.TERMINATED) { cont.resume(Unit) -- cgit v1.2.3 From befec2f1ddf3a6e6d15d9d1b9fd1ecbbc4f38960 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 26 Aug 2021 10:34:18 +0200 Subject: feat(capelin): Report up/downtime metrics in experiment monitor --- .../capelin/monitor/ExperimentMetricExporter.kt | 40 ++++++++++------------ .../capelin/monitor/ExperimentMonitor.kt | 2 ++ .../capelin/monitor/ParquetExperimentMonitor.kt | 2 ++ .../experiments/capelin/CapelinIntegrationTest.kt | 2 ++ 4 files changed, 24 insertions(+), 22 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt index 42b7cbb8..79be9ac4 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt @@ -54,7 +54,6 @@ public class ExperimentMetricExporter( private fun reportHostMetrics(metrics: Collection) { val hostMetrics = mutableMapOf() - hosts.mapValuesTo(hostMetrics) { HostMetrics() } for (metric in metrics) { when (metric.name) { @@ -66,12 +65,15 @@ public class ExperimentMetricExporter( "cpu.work.overcommit" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.overcommittedWork = v } "cpu.work.interference" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.interferedWork = v } "guests.active" -> mapLongSum(metric, hostMetrics) { m, v -> m.instanceCount = v.toInt() } + "host.time.up" -> mapLongSum(metric, hostMetrics) { m, v -> m.uptime = v } + "host.time.down" -> mapLongSum(metric, hostMetrics) { m, v -> m.downtime = v } } } for ((id, hostMetric) in hostMetrics) { val lastHostMetric = lastHostMetrics.getOrDefault(id, hostMetricsSingleton) - val host = hosts.getValue(id) + val host = hosts[id] ?: continue + monitor.reportHostData( clock.millis(), hostMetric.totalWork - lastHostMetric.totalWork, @@ -82,6 +84,8 @@ public class ExperimentMetricExporter( hostMetric.cpuDemand, hostMetric.powerDraw, hostMetric.instanceCount, + hostMetric.uptime - lastHostMetric.uptime, + hostMetric.downtime - lastHostMetric.downtime, host ) } @@ -92,38 +96,28 @@ public class ExperimentMetricExporter( private fun mapDoubleSummary(data: MetricData, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data.doubleSummaryData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] - val hostMetric = hostMetrics[uid] - - if (hostMetric != null) { - // Take the average of the summary - val avg = (point.percentileValues[0].value + point.percentileValues[1].value) / 2 - block(hostMetric, avg) - } + val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue + val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } + val avg = (point.percentileValues[0].value + point.percentileValues[1].value) / 2 + block(hostMetric, avg) } } private fun mapLongSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Long) -> Unit) { val points = data?.longSumData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] - val hostMetric = hostMetrics[uid] - - if (hostMetric != null) { - block(hostMetric, point.value) - } + val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue + val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } + block(hostMetric, point.value) } } private fun mapDoubleSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { val points = data?.doubleSumData?.points ?: emptyList() for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] - val hostMetric = hostMetrics[uid] - - if (hostMetric != null) { - block(hostMetric, point.value) - } + val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue + val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } + block(hostMetric, point.value) } } @@ -151,6 +145,8 @@ public class ExperimentMetricExporter( var cpuDemand: Double = 0.0 var instanceCount: Int = 0 var powerDraw: Double = 0.0 + var uptime: Long = 0 + var downtime: Long = 0 } override fun flush(): CompletableResultCode = CompletableResultCode.ofSuccess() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt index 9a4aec35..dc28b816 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt @@ -54,6 +54,8 @@ public interface ExperimentMonitor : AutoCloseable { cpuDemand: Double, powerDraw: Double, instanceCount: Int, + uptime: Long, + downtime: Long, host: Host ) {} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt index 83351c41..c49499da 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt @@ -67,6 +67,8 @@ public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: cpuDemand: Double, powerDraw: Double, instanceCount: Int, + uptime: Long, + downtime: Long, host: Host ) { hostWriter.write( diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 2934bbe6..24d8f768 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -300,6 +300,8 @@ class CapelinIntegrationTest { cpuDemand: Double, powerDraw: Double, instanceCount: Int, + uptime: Long, + downtime: Long, host: Host, ) { this.totalWork += totalWork.toLong() -- cgit v1.2.3 From aaedd4f3eed83d0c3ebc829fec08a1749a2bfba4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 27 Aug 2021 16:41:55 +0200 Subject: refactor(capelin): Move metric collection outside Capelin code This change moves the metric collection outside the Capelin codebase in a separate module so other modules can also benefit from the compute metric collection code. --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../experiments/capelin/ExperimentHelpers.kt | 98 +------------ .../org/opendc/experiments/capelin/Portfolio.kt | 19 ++- .../capelin/export/parquet/ParquetDataWriter.kt | 127 +++++++++++++++++ .../capelin/export/parquet/ParquetExportMonitor.kt | 55 ++++++++ .../export/parquet/ParquetHostDataWriter.kt | 73 ++++++++++ .../export/parquet/ParquetServiceDataWriter.kt | 65 +++++++++ .../capelin/monitor/ExperimentMetricExporter.kt | 155 --------------------- .../capelin/monitor/ExperimentMonitor.kt | 75 ---------- .../capelin/monitor/ParquetExperimentMonitor.kt | 120 ---------------- .../opendc/experiments/capelin/telemetry/Event.kt | 35 ----- .../experiments/capelin/telemetry/HostEvent.kt | 43 ------ .../capelin/telemetry/ProvisionerEvent.kt | 39 ------ .../experiments/capelin/telemetry/RunEvent.kt | 34 ----- .../experiments/capelin/telemetry/VmEvent.kt | 41 ------ .../telemetry/parquet/ParquetEventWriter.kt | 126 ----------------- .../telemetry/parquet/ParquetHostEventWriter.kt | 81 ----------- .../parquet/ParquetProvisionerEventWriter.kt | 65 --------- .../telemetry/parquet/ParquetRunEventWriter.kt | 72 ---------- .../experiments/capelin/CapelinIntegrationTest.kt | 92 ++++++------ .../opendc-experiments-energy21/build.gradle.kts | 1 + .../experiments/energy21/EnergyExperiment.kt | 18 +-- 22 files changed, 400 insertions(+), 1035 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/Event.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/HostEvent.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/ProvisionerEvent.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/RunEvent.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/VmEvent.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetHostEventWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetProvisionerEventWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetRunEventWriter.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 65cebe1b..1a4caf91 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -38,6 +38,7 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) + implementation(projects.opendcTelemetry.opendcTelemetryCompute) implementation(libs.opentelemetry.semconv) implementation(libs.kotlin.logging) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 9d23a5dd..0230409e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -24,16 +24,10 @@ package org.opendc.experiments.capelin import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.data.MetricData -import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel -import mu.KotlinLogging import org.opendc.compute.api.* import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostListener -import org.opendc.compute.service.driver.HostState import org.opendc.compute.service.scheduler.ComputeScheduler import org.opendc.compute.service.scheduler.FilterScheduler import org.opendc.compute.service.scheduler.ReplayScheduler @@ -46,8 +40,6 @@ import org.opendc.compute.service.scheduler.weights.RamWeigher import org.opendc.compute.service.scheduler.weights.VCpuWeigher import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.EnvironmentReader -import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter -import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.TraceReader import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel @@ -57,7 +49,7 @@ import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.failures.CorrelatedFaultInjector import org.opendc.simulator.failures.FaultInjector import org.opendc.simulator.resources.SimResourceInterpreter -import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader +import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.sdk.toOtelClock import java.time.Clock import kotlin.coroutines.resume @@ -65,11 +57,6 @@ import kotlin.math.ln import kotlin.math.max import kotlin.random.Random -/** - * The logger for this experiment. - */ -private val logger = KotlinLogging.logger {} - /** * Construct the failure domain for the experiments. */ @@ -168,85 +155,6 @@ suspend fun withComputeService( } } -/** - * Attach the specified monitor to the VM provisioner. - */ -suspend fun withMonitor( - monitor: ExperimentMonitor, - clock: Clock, - metricProducer: MetricProducer, - scheduler: ComputeService, - block: suspend CoroutineScope.() -> Unit -): Unit = coroutineScope { - // Monitor host events - for (host in scheduler.hosts) { - monitor.reportHostStateChange(clock.millis(), host, HostState.UP) - host.addListener(object : HostListener { - override fun onStateChanged(host: Host, newState: HostState) { - monitor.reportHostStateChange(clock.millis(), host, newState) - } - }) - } - - val reader = CoroutineMetricReader( - this, - listOf(metricProducer), - ExperimentMetricExporter(monitor, clock, scheduler.hosts.associateBy { it.uid.toString() }), - exportInterval = 5L * 60 * 1000 /* Every 5 min (which is the granularity of the workload trace) */ - ) - - try { - block(this) - } finally { - reader.close() - monitor.close() - } -} - -class ComputeMetrics { - var submittedVms: Int = 0 - var queuedVms: Int = 0 - var runningVms: Int = 0 - var unscheduledVms: Int = 0 - var finishedVms: Int = 0 - var hosts: Int = 0 - var availableHosts = 0 -} - -/** - * Collect the metrics of the compute service. - */ -fun collectMetrics(metricProducer: MetricProducer): ComputeMetrics { - return extractComputeMetrics(metricProducer.collectAllMetrics()) -} - -/** - * Extract an [ComputeMetrics] object from the specified list of metric data. - */ -internal fun extractComputeMetrics(metrics: Collection): ComputeMetrics { - val res = ComputeMetrics() - for (metric in metrics) { - val points = metric.longSumData.points - - if (points.isEmpty()) { - continue - } - - val value = points.first().value.toInt() - when (metric.name) { - "servers.submitted" -> res.submittedVms = value - "servers.waiting" -> res.queuedVms = value - "servers.unscheduled" -> res.unscheduledVms = value - "servers.active" -> res.runningVms = value - "servers.finished" -> res.finishedVms = value - "hosts.total" -> res.hosts = value - "hosts.available" -> res.availableHosts = value - } - } - - return res -} - /** * Process the trace. */ @@ -255,7 +163,7 @@ suspend fun processTrace( reader: TraceReader, scheduler: ComputeService, chan: Channel, - monitor: ExperimentMonitor? = null, + monitor: ComputeMonitor? = null, ) { val client = scheduler.newClient() val image = client.newImage("vm-image") @@ -289,7 +197,7 @@ suspend fun processTrace( suspendCancellableCoroutine { cont -> server.watch(object : ServerWatcher { override fun onStateChanged(server: Server, newState: ServerState) { - monitor?.reportVmStateChange(clock.millis(), server, newState) + monitor?.onStateChange(clock.millis(), server, newState) if (newState == ServerState.TERMINATED) { cont.resume(Unit) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index ee832af8..d3bba182 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -29,11 +29,11 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import mu.KotlinLogging import org.opendc.experiments.capelin.env.ClusterEnvironmentReader +import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload -import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader @@ -41,6 +41,8 @@ import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.compute.collectServiceMetrics +import org.opendc.telemetry.compute.withMonitor import java.io.File import java.io.FileInputStream import java.util.* @@ -127,7 +129,7 @@ abstract class Portfolio(name: String) : Experiment(name) { val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) - val monitor = ParquetExperimentMonitor( + val monitor = ParquetExportMonitor( File(config.getString("output-path")), "portfolio_id=$name/scenario_id=$id/run_id=$repeat", 4096 @@ -148,7 +150,7 @@ abstract class Portfolio(name: String) : Experiment(name) { null } - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, trace, @@ -159,9 +161,16 @@ abstract class Portfolio(name: String) : Experiment(name) { } failureDomain?.cancel() + monitor.close() } - val monitorResults = collectMetrics(meterProvider as MetricProducer) - logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } + val monitorResults = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + logger.debug { + "Finish " + + "SUBMIT=${monitorResults.instanceCount} " + + "FAIL=${monitorResults.failedInstanceCount} " + + "QUEUE=${monitorResults.queuedInstanceCount} " + + "RUNNING=${monitorResults.activeHostCount}" + } } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt new file mode 100644 index 00000000..c5cb80e2 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.export.parquet + +import mu.KotlinLogging +import org.apache.avro.Schema +import org.apache.avro.generic.GenericData +import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.hadoop.ParquetFileWriter +import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.opendc.trace.util.parquet.LocalOutputFile +import java.io.Closeable +import java.io.File +import java.util.concurrent.ArrayBlockingQueue +import java.util.concurrent.BlockingQueue +import kotlin.concurrent.thread + +/** + * A writer that writes data in Parquet format. + */ +public open class ParquetDataWriter( + private val path: File, + private val schema: Schema, + private val converter: (T, GenericData.Record) -> Unit, + private val bufferSize: Int = 4096 +) : Runnable, Closeable { + /** + * The logging instance to use. + */ + private val logger = KotlinLogging.logger {} + + /** + * The writer to write the Parquet file. + */ + private val writer = AvroParquetWriter.builder(LocalOutputFile(path)) + .withSchema(schema) + .withCompressionCodec(CompressionCodecName.SNAPPY) + .withPageSize(4 * 1024 * 1024) // For compression + .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + .build() + + /** + * The queue of commands to process. + */ + private val queue: BlockingQueue = ArrayBlockingQueue(bufferSize) + + /** + * The thread that is responsible for writing the Parquet records. + */ + private val writerThread = thread(start = false, name = this.toString()) { run() } + + /** + * Write the specified metrics to the database. + */ + public fun write(event: T) { + queue.put(Action.Write(event)) + } + + /** + * Signal the writer to stop. + */ + public override fun close() { + queue.put(Action.Stop) + writerThread.join() + } + + init { + writerThread.start() + } + + /** + * Start the writer thread. + */ + override fun run() { + try { + loop@ while (true) { + val action = queue.take() + when (action) { + is Action.Stop -> break@loop + is Action.Write<*> -> { + val record = GenericData.Record(schema) + @Suppress("UNCHECKED_CAST") + converter(action.data as T, record) + writer.write(record) + } + } + } + } catch (e: Throwable) { + logger.error("Writer failed", e) + } finally { + writer.close() + } + } + + public sealed class Action { + /** + * A poison pill that will stop the writer thread. + */ + public object Stop : Action() + + /** + * Write the specified metrics to the database. + */ + public data class Write(val data: T) : Action() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt new file mode 100644 index 00000000..79b84e9d --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.export.parquet + +import org.opendc.telemetry.compute.ComputeMonitor +import org.opendc.telemetry.compute.table.HostData +import org.opendc.telemetry.compute.table.ServiceData +import java.io.File + +/** + * A [ComputeMonitor] that logs the events to a Parquet file. + */ +public class ParquetExportMonitor(base: File, partition: String, bufferSize: Int) : ComputeMonitor, AutoCloseable { + private val hostWriter = ParquetHostDataWriter( + File(base, "host/$partition/data.parquet").also { it.parentFile.mkdirs() }, + bufferSize + ) + private val serviceWriter = ParquetServiceDataWriter( + File(base, "service/$partition/data.parquet").also { it.parentFile.mkdirs() }, + bufferSize + ) + + override fun record(data: HostData) { + hostWriter.write(data) + } + + override fun record(data: ServiceData) { + serviceWriter.write(data) + } + + override fun close() { + hostWriter.close() + serviceWriter.close() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt new file mode 100644 index 00000000..8912c12e --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.export.parquet + +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.generic.GenericData +import org.opendc.telemetry.compute.table.HostData +import java.io.File + +/** + * A Parquet event writer for [HostData]s. + */ +public class ParquetHostDataWriter(path: File, bufferSize: Int) : + ParquetDataWriter(path, schema, convert, bufferSize) { + + override fun toString(): String = "host-writer" + + public companion object { + private val convert: (HostData, GenericData.Record) -> Unit = { data, record -> + record.put("host_id", data.host.name) + record.put("state", data.host.state.name) + record.put("timestamp", data.timestamp) + record.put("total_work", data.totalWork) + record.put("granted_work", data.grantedWork) + record.put("overcommitted_work", data.overcommittedWork) + record.put("interfered_work", data.interferedWork) + record.put("cpu_usage", data.cpuUsage) + record.put("cpu_demand", data.cpuDemand) + record.put("power_draw", data.powerDraw) + record.put("instance_count", data.instanceCount) + record.put("cores", data.host.model.cpuCount) + } + + private val schema: Schema = SchemaBuilder + .record("host") + .namespace("org.opendc.telemetry.compute") + .fields() + .name("timestamp").type().longType().noDefault() + .name("host_id").type().stringType().noDefault() + .name("state").type().stringType().noDefault() + .name("requested_work").type().longType().noDefault() + .name("granted_work").type().longType().noDefault() + .name("overcommitted_work").type().longType().noDefault() + .name("interfered_work").type().longType().noDefault() + .name("cpu_usage").type().doubleType().noDefault() + .name("cpu_demand").type().doubleType().noDefault() + .name("power_draw").type().doubleType().noDefault() + .name("instance_count").type().intType().noDefault() + .name("cores").type().intType().noDefault() + .endRecord() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt new file mode 100644 index 00000000..36d630f3 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.export.parquet + +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.generic.GenericData +import org.opendc.telemetry.compute.table.ServiceData +import java.io.File + +/** + * A Parquet event writer for [ServiceData]s. + */ +public class ParquetServiceDataWriter(path: File, bufferSize: Int) : + ParquetDataWriter(path, schema, convert, bufferSize) { + + override fun toString(): String = "service-writer" + + public companion object { + private val convert: (ServiceData, GenericData.Record) -> Unit = { data, record -> + record.put("timestamp", data.timestamp) + record.put("host_total_count", data.hostCount) + record.put("host_available_count", data.activeHostCount) + record.put("instance_total_count", data.instanceCount) + record.put("instance_active_count", data.runningInstanceCount) + record.put("instance_inactive_count", data.finishedInstanceCount) + record.put("instance_waiting_count", data.queuedInstanceCount) + record.put("instance_failed_count", data.failedInstanceCount) + } + + private val schema: Schema = SchemaBuilder + .record("service") + .namespace("org.opendc.telemetry.compute") + .fields() + .name("timestamp").type().longType().noDefault() + .name("host_total_count").type().intType().noDefault() + .name("host_available_count").type().intType().noDefault() + .name("instance_total_count").type().intType().noDefault() + .name("instance_active_count").type().intType().noDefault() + .name("instance_inactive_count").type().intType().noDefault() + .name("instance_waiting_count").type().intType().noDefault() + .name("instance_failed_count").type().intType().noDefault() + .endRecord() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt deleted file mode 100644 index 79be9ac4..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMetricExporter.kt +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.monitor - -import io.opentelemetry.sdk.common.CompletableResultCode -import io.opentelemetry.sdk.metrics.data.MetricData -import io.opentelemetry.sdk.metrics.export.MetricExporter -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes -import org.opendc.compute.service.driver.Host -import org.opendc.experiments.capelin.extractComputeMetrics -import java.time.Clock - -/** - * A [MetricExporter] that exports the metrics to the [ExperimentMonitor]. - */ -public class ExperimentMetricExporter( - private val monitor: ExperimentMonitor, - private val clock: Clock, - private val hosts: Map -) : MetricExporter { - - override fun export(metrics: Collection): CompletableResultCode { - return try { - reportHostMetrics(metrics) - reportProvisionerMetrics(metrics) - CompletableResultCode.ofSuccess() - } catch (e: Throwable) { - CompletableResultCode.ofFailure() - } - } - - private var lastHostMetrics: Map = emptyMap() - private val hostMetricsSingleton = HostMetrics() - - private fun reportHostMetrics(metrics: Collection) { - val hostMetrics = mutableMapOf() - - for (metric in metrics) { - when (metric.name) { - "cpu.demand" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuDemand = v } - "cpu.usage" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.cpuUsage = v } - "power.usage" -> mapDoubleSummary(metric, hostMetrics) { m, v -> m.powerDraw = v } - "cpu.work.total" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.totalWork = v } - "cpu.work.granted" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.grantedWork = v } - "cpu.work.overcommit" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.overcommittedWork = v } - "cpu.work.interference" -> mapDoubleSum(metric, hostMetrics) { m, v -> m.interferedWork = v } - "guests.active" -> mapLongSum(metric, hostMetrics) { m, v -> m.instanceCount = v.toInt() } - "host.time.up" -> mapLongSum(metric, hostMetrics) { m, v -> m.uptime = v } - "host.time.down" -> mapLongSum(metric, hostMetrics) { m, v -> m.downtime = v } - } - } - - for ((id, hostMetric) in hostMetrics) { - val lastHostMetric = lastHostMetrics.getOrDefault(id, hostMetricsSingleton) - val host = hosts[id] ?: continue - - monitor.reportHostData( - clock.millis(), - hostMetric.totalWork - lastHostMetric.totalWork, - hostMetric.grantedWork - lastHostMetric.grantedWork, - hostMetric.overcommittedWork - lastHostMetric.overcommittedWork, - hostMetric.interferedWork - lastHostMetric.interferedWork, - hostMetric.cpuUsage, - hostMetric.cpuDemand, - hostMetric.powerDraw, - hostMetric.instanceCount, - hostMetric.uptime - lastHostMetric.uptime, - hostMetric.downtime - lastHostMetric.downtime, - host - ) - } - - lastHostMetrics = hostMetrics - } - - private fun mapDoubleSummary(data: MetricData, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { - val points = data.doubleSummaryData?.points ?: emptyList() - for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue - val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } - val avg = (point.percentileValues[0].value + point.percentileValues[1].value) / 2 - block(hostMetric, avg) - } - } - - private fun mapLongSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Long) -> Unit) { - val points = data?.longSumData?.points ?: emptyList() - for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue - val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } - block(hostMetric, point.value) - } - } - - private fun mapDoubleSum(data: MetricData?, hostMetrics: MutableMap, block: (HostMetrics, Double) -> Unit) { - val points = data?.doubleSumData?.points ?: emptyList() - for (point in points) { - val uid = point.attributes[ResourceAttributes.HOST_ID] ?: continue - val hostMetric = hostMetrics.computeIfAbsent(uid) { HostMetrics() } - block(hostMetric, point.value) - } - } - - private fun reportProvisionerMetrics(metrics: Collection) { - val res = extractComputeMetrics(metrics) - - monitor.reportServiceData( - clock.millis(), - res.hosts, - res.availableHosts, - res.submittedVms, - res.runningVms, - res.finishedVms, - res.queuedVms, - res.unscheduledVms - ) - } - - private class HostMetrics { - var totalWork: Double = 0.0 - var grantedWork: Double = 0.0 - var overcommittedWork: Double = 0.0 - var interferedWork: Double = 0.0 - var cpuUsage: Double = 0.0 - var cpuDemand: Double = 0.0 - var instanceCount: Int = 0 - var powerDraw: Double = 0.0 - var uptime: Long = 0 - var downtime: Long = 0 - } - - override fun flush(): CompletableResultCode = CompletableResultCode.ofSuccess() - - override fun shutdown(): CompletableResultCode = CompletableResultCode.ofSuccess() -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt deleted file mode 100644 index dc28b816..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ExperimentMonitor.kt +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.monitor - -import org.opendc.compute.api.Server -import org.opendc.compute.api.ServerState -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostState - -/** - * A monitor watches the events of an experiment. - */ -public interface ExperimentMonitor : AutoCloseable { - /** - * This method is invoked when the state of a VM changes. - */ - public fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} - - /** - * This method is invoked when the state of a host changes. - */ - public fun reportHostStateChange(time: Long, host: Host, newState: HostState) {} - - /** - * This method is invoked for a host for each slice that is finishes. - */ - public fun reportHostData( - time: Long, - totalWork: Double, - grantedWork: Double, - overcommittedWork: Double, - interferedWork: Double, - cpuUsage: Double, - cpuDemand: Double, - powerDraw: Double, - instanceCount: Int, - uptime: Long, - downtime: Long, - host: Host - ) {} - - /** - * This method is invoked for reporting service data. - */ - public fun reportServiceData( - time: Long, - totalHostCount: Int, - availableHostCount: Int, - totalVmCount: Int, - activeVmCount: Int, - inactiveVmCount: Int, - waitingVmCount: Int, - failedVmCount: Int - ) {} -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt deleted file mode 100644 index c49499da..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/monitor/ParquetExperimentMonitor.kt +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.monitor - -import mu.KotlinLogging -import org.opendc.compute.api.Server -import org.opendc.compute.api.ServerState -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostState -import org.opendc.experiments.capelin.telemetry.HostEvent -import org.opendc.experiments.capelin.telemetry.ProvisionerEvent -import org.opendc.experiments.capelin.telemetry.parquet.ParquetHostEventWriter -import org.opendc.experiments.capelin.telemetry.parquet.ParquetProvisionerEventWriter -import java.io.File - -/** - * The logger instance to use. - */ -private val logger = KotlinLogging.logger {} - -/** - * An [ExperimentMonitor] that logs the events to a Parquet file. - */ -public class ParquetExperimentMonitor(base: File, partition: String, bufferSize: Int) : ExperimentMonitor { - private val hostWriter = ParquetHostEventWriter( - File(base, "host-metrics/$partition/data.parquet").also { it.parentFile.mkdirs() }, - bufferSize - ) - private val provisionerWriter = ParquetProvisionerEventWriter( - File(base, "provisioner-metrics/$partition/data.parquet").also { it.parentFile.mkdirs() }, - bufferSize - ) - - override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} - - override fun reportHostStateChange(time: Long, host: Host, newState: HostState) { - logger.debug { "Host ${host.uid} changed state $newState [$time]" } - } - - override fun reportHostData( - time: Long, - totalWork: Double, - grantedWork: Double, - overcommittedWork: Double, - interferedWork: Double, - cpuUsage: Double, - cpuDemand: Double, - powerDraw: Double, - instanceCount: Int, - uptime: Long, - downtime: Long, - host: Host - ) { - hostWriter.write( - HostEvent( - time, - 5 * 60 * 1000L, - host, - instanceCount, - totalWork.toLong(), - grantedWork.toLong(), - overcommittedWork.toLong(), - interferedWork.toLong(), - cpuUsage, - cpuDemand, - powerDraw, - host.model.cpuCount - ) - ) - } - - override fun reportServiceData( - time: Long, - totalHostCount: Int, - availableHostCount: Int, - totalVmCount: Int, - activeVmCount: Int, - inactiveVmCount: Int, - waitingVmCount: Int, - failedVmCount: Int - ) { - provisionerWriter.write( - ProvisionerEvent( - time, - totalHostCount, - availableHostCount, - totalVmCount, - activeVmCount, - inactiveVmCount, - waitingVmCount, - failedVmCount - ) - ) - } - - override fun close() { - hostWriter.close() - provisionerWriter.close() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/Event.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/Event.kt deleted file mode 100644 index c29e116e..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/Event.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry - -/** - * An event that occurs within the system. - */ -public abstract class Event(public val name: String) { - /** - * The time of occurrence of this event. - */ - public abstract val timestamp: Long -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/HostEvent.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/HostEvent.kt deleted file mode 100644 index 899fc9b1..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/HostEvent.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry - -import org.opendc.compute.service.driver.Host - -/** - * A periodic report of the host machine metrics. - */ -public data class HostEvent( - override val timestamp: Long, - public val duration: Long, - public val host: Host, - public val vmCount: Int, - public val requestedBurst: Long, - public val grantedBurst: Long, - public val overcommissionedBurst: Long, - public val interferedBurst: Long, - public val cpuUsage: Double, - public val cpuDemand: Double, - public val powerDraw: Double, - public val cores: Int -) : Event("host-metrics") diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/ProvisionerEvent.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/ProvisionerEvent.kt deleted file mode 100644 index 539c9bc9..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/ProvisionerEvent.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry - -/** - * A periodic report of the provisioner's metrics. - */ -public data class ProvisionerEvent( - override val timestamp: Long, - public val totalHostCount: Int, - public val availableHostCount: Int, - public val totalVmCount: Int, - public val activeVmCount: Int, - public val inactiveVmCount: Int, - public val waitingVmCount: Int, - public val failedVmCount: Int -) : Event("provisioner-metrics") diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/RunEvent.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/RunEvent.kt deleted file mode 100644 index 6c8fc941..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/RunEvent.kt +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry - -import org.opendc.experiments.capelin.Portfolio - -/** - * A periodic report of the host machine metrics. - */ -public data class RunEvent( - val portfolio: Portfolio, - val repeat: Int, - override val timestamp: Long -) : Event("run") diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/VmEvent.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/VmEvent.kt deleted file mode 100644 index 7631f55f..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/VmEvent.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry - -import org.opendc.compute.api.Server - -/** - * A periodic report of a virtual machine's metrics. - */ -public data class VmEvent( - override val timestamp: Long, - public val duration: Long, - public val vm: Server, - public val host: Server, - public val requestedBurst: Long, - public val grantedBurst: Long, - public val overcommissionedBurst: Long, - public val interferedBurst: Long, - public val cpuUsage: Double, - public val cpuDemand: Double -) : Event("vm-metrics") diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt deleted file mode 100644 index 897a6692..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetEventWriter.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry.parquet - -import mu.KotlinLogging -import org.apache.avro.Schema -import org.apache.avro.generic.GenericData -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.metadata.CompressionCodecName -import org.opendc.experiments.capelin.telemetry.Event -import org.opendc.trace.util.parquet.LocalOutputFile -import java.io.Closeable -import java.io.File -import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.BlockingQueue -import kotlin.concurrent.thread - -/** - * The logging instance to use. - */ -private val logger = KotlinLogging.logger {} - -/** - * A writer that writes events in Parquet format. - */ -public open class ParquetEventWriter( - private val path: File, - private val schema: Schema, - private val converter: (T, GenericData.Record) -> Unit, - private val bufferSize: Int = 4096 -) : Runnable, Closeable { - /** - * The writer to write the Parquet file. - */ - private val writer = AvroParquetWriter.builder(LocalOutputFile(path)) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) - .build() - - /** - * The queue of commands to process. - */ - private val queue: BlockingQueue = ArrayBlockingQueue(bufferSize) - - /** - * The thread that is responsible for writing the Parquet records. - */ - private val writerThread = thread(start = false, name = "parquet-writer") { run() } - - /** - * Write the specified metrics to the database. - */ - public fun write(event: T) { - queue.put(Action.Write(event)) - } - - /** - * Signal the writer to stop. - */ - public override fun close() { - queue.put(Action.Stop) - writerThread.join() - } - - init { - writerThread.start() - } - - /** - * Start the writer thread. - */ - override fun run() { - try { - loop@ while (true) { - val action = queue.take() - when (action) { - is Action.Stop -> break@loop - is Action.Write<*> -> { - val record = GenericData.Record(schema) - @Suppress("UNCHECKED_CAST") - converter(action.event as T, record) - writer.write(record) - } - } - } - } catch (e: Throwable) { - logger.error("Writer failed", e) - } finally { - writer.close() - } - } - - public sealed class Action { - /** - * A poison pill that will stop the writer thread. - */ - public object Stop : Action() - - /** - * Write the specified metrics to the database. - */ - public data class Write(val event: T) : Action() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetHostEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetHostEventWriter.kt deleted file mode 100644 index c8fe1cb2..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetHostEventWriter.kt +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.opendc.experiments.capelin.telemetry.HostEvent -import java.io.File - -/** - * A Parquet event writer for [HostEvent]s. - */ -public class ParquetHostEventWriter(path: File, bufferSize: Int) : - ParquetEventWriter(path, schema, convert, bufferSize) { - - override fun toString(): String = "host-writer" - - public companion object { - private val convert: (HostEvent, GenericData.Record) -> Unit = { event, record -> - // record.put("portfolio_id", event.run.parent.parent.id) - // record.put("scenario_id", event.run.parent.id) - // record.put("run_id", event.run.id) - record.put("host_id", event.host.name) - record.put("state", event.host.state.name) - record.put("timestamp", event.timestamp) - record.put("duration", event.duration) - record.put("vm_count", event.vmCount) - record.put("requested_burst", event.requestedBurst) - record.put("granted_burst", event.grantedBurst) - record.put("overcommissioned_burst", event.overcommissionedBurst) - record.put("interfered_burst", event.interferedBurst) - record.put("cpu_usage", event.cpuUsage) - record.put("cpu_demand", event.cpuDemand) - record.put("power_draw", event.powerDraw) - record.put("cores", event.cores) - } - - private val schema: Schema = SchemaBuilder - .record("host_metrics") - .namespace("org.opendc.experiments.sc20") - .fields() - // .name("portfolio_id").type().intType().noDefault() - // .name("scenario_id").type().intType().noDefault() - // .name("run_id").type().intType().noDefault() - .name("timestamp").type().longType().noDefault() - .name("duration").type().longType().noDefault() - .name("host_id").type().stringType().noDefault() - .name("state").type().stringType().noDefault() - .name("vm_count").type().intType().noDefault() - .name("requested_burst").type().longType().noDefault() - .name("granted_burst").type().longType().noDefault() - .name("overcommissioned_burst").type().longType().noDefault() - .name("interfered_burst").type().longType().noDefault() - .name("cpu_usage").type().doubleType().noDefault() - .name("cpu_demand").type().doubleType().noDefault() - .name("power_draw").type().doubleType().noDefault() - .name("cores").type().intType().noDefault() - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetProvisionerEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetProvisionerEventWriter.kt deleted file mode 100644 index 8feff8d9..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetProvisionerEventWriter.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.opendc.experiments.capelin.telemetry.ProvisionerEvent -import java.io.File - -/** - * A Parquet event writer for [ProvisionerEvent]s. - */ -public class ParquetProvisionerEventWriter(path: File, bufferSize: Int) : - ParquetEventWriter(path, schema, convert, bufferSize) { - - override fun toString(): String = "provisioner-writer" - - public companion object { - private val convert: (ProvisionerEvent, GenericData.Record) -> Unit = { event, record -> - record.put("timestamp", event.timestamp) - record.put("host_total_count", event.totalHostCount) - record.put("host_available_count", event.availableHostCount) - record.put("vm_total_count", event.totalVmCount) - record.put("vm_active_count", event.activeVmCount) - record.put("vm_inactive_count", event.inactiveVmCount) - record.put("vm_waiting_count", event.waitingVmCount) - record.put("vm_failed_count", event.failedVmCount) - } - - private val schema: Schema = SchemaBuilder - .record("provisioner_metrics") - .namespace("org.opendc.experiments.sc20") - .fields() - .name("timestamp").type().longType().noDefault() - .name("host_total_count").type().intType().noDefault() - .name("host_available_count").type().intType().noDefault() - .name("vm_total_count").type().intType().noDefault() - .name("vm_active_count").type().intType().noDefault() - .name("vm_inactive_count").type().intType().noDefault() - .name("vm_waiting_count").type().intType().noDefault() - .name("vm_failed_count").type().intType().noDefault() - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetRunEventWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetRunEventWriter.kt deleted file mode 100644 index 946410eb..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/telemetry/parquet/ParquetRunEventWriter.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.telemetry.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.opendc.experiments.capelin.telemetry.RunEvent -import java.io.File - -/** - * A Parquet event writer for [RunEvent]s. - */ -public class ParquetRunEventWriter(path: File, bufferSize: Int) : - ParquetEventWriter(path, schema, convert, bufferSize) { - - override fun toString(): String = "run-writer" - - public companion object { - private val convert: (RunEvent, GenericData.Record) -> Unit = { event, record -> - val portfolio = event.portfolio - record.put("portfolio_name", portfolio.name) - record.put("scenario_id", portfolio.id) - record.put("run_id", event.repeat) - record.put("topology", portfolio.topology.name) - record.put("workload_name", portfolio.workload.name) - record.put("workload_fraction", portfolio.workload.fraction) - record.put("workload_sampler", portfolio.workload.samplingStrategy) - record.put("allocation_policy", portfolio.allocationPolicy) - record.put("failure_frequency", portfolio.operationalPhenomena.failureFrequency) - record.put("interference", portfolio.operationalPhenomena.hasInterference) - record.put("seed", event.repeat) - } - - private val schema: Schema = SchemaBuilder - .record("runs") - .namespace("org.opendc.experiments.sc20") - .fields() - .name("portfolio_name").type().stringType().noDefault() - .name("scenario_id").type().intType().noDefault() - .name("run_id").type().intType().noDefault() - .name("topology").type().stringType().noDefault() - .name("workload_name").type().stringType().noDefault() - .name("workload_fraction").type().doubleType().noDefault() - .name("workload_sampler").type().stringType().noDefault() - .name("allocation_policy").type().stringType().noDefault() - .name("failure_frequency").type().doubleType().noDefault() - .name("interference").type().booleanType().noDefault() - .name("seed").type().intType().noDefault() - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 24d8f768..abd8efeb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -29,7 +29,6 @@ import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertAll -import org.opendc.compute.service.driver.Host import org.opendc.compute.service.scheduler.FilterScheduler import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.filters.RamFilter @@ -38,7 +37,6 @@ import org.opendc.compute.service.scheduler.weights.CoreRamWeigher import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload -import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader @@ -46,6 +44,10 @@ import org.opendc.experiments.capelin.trace.TraceReader import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.compute.ComputeMonitor +import org.opendc.telemetry.compute.collectServiceMetrics +import org.opendc.telemetry.compute.table.HostData +import org.opendc.telemetry.compute.withMonitor import java.io.File import java.util.* @@ -80,7 +82,6 @@ class CapelinIntegrationTest { ) val traceReader = createTestTraceReader() val environmentReader = createTestEnvironmentReader() - lateinit var monitorResults: ComputeMetrics val meterProvider = createMeterProvider(clock) withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> @@ -98,7 +99,7 @@ class CapelinIntegrationTest { null } - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, traceReader, @@ -111,15 +112,21 @@ class CapelinIntegrationTest { failureDomain?.cancel() } - monitorResults = collectMetrics(meterProvider as MetricProducer) - println("Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}") + val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + println( + "Finish " + + "SUBMIT=${serviceMetrics.instanceCount} " + + "FAIL=${serviceMetrics.failedInstanceCount} " + + "QUEUE=${serviceMetrics.queuedInstanceCount} " + + "RUNNING=${serviceMetrics.runningInstanceCount}" + ) // Note that these values have been verified beforehand assertAll( - { assertEquals(50, monitorResults.submittedVms, "The trace contains 50 VMs") }, - { assertEquals(0, monitorResults.runningVms, "All VMs should finish after a run") }, - { assertEquals(0, monitorResults.unscheduledVms, "No VM should not be unscheduled") }, - { assertEquals(0, monitorResults.queuedVms, "No VM should not be in the queue") }, + { assertEquals(50, serviceMetrics.instanceCount, "The trace contains 50 VMs") }, + { assertEquals(0, serviceMetrics.runningInstanceCount, "All VMs should finish after a run") }, + { assertEquals(0, serviceMetrics.failedInstanceCount, "No VM should not be unscheduled") }, + { assertEquals(0, serviceMetrics.queuedInstanceCount, "No VM should not be in the queue") }, { assertEquals(220346369753, monitor.totalWork) { "Incorrect requested burst" } }, { assertEquals(206667809529, monitor.totalGrantedWork) { "Incorrect granted burst" } }, { assertEquals(1151611104, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, @@ -145,7 +152,7 @@ class CapelinIntegrationTest { val meterProvider = createMeterProvider(clock) withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, traceReader, @@ -156,8 +163,14 @@ class CapelinIntegrationTest { } } - val metrics = collectMetrics(meterProvider as MetricProducer) - println("Finish SUBMIT=${metrics.submittedVms} FAIL=${metrics.unscheduledVms} QUEUE=${metrics.queuedVms} RUNNING=${metrics.runningVms}") + val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + println( + "Finish " + + "SUBMIT=${serviceMetrics.instanceCount} " + + "FAIL=${serviceMetrics.failedInstanceCount} " + + "QUEUE=${serviceMetrics.queuedInstanceCount} " + + "RUNNING=${serviceMetrics.runningInstanceCount}" + ) // Note that these values have been verified beforehand assertAll( @@ -189,7 +202,7 @@ class CapelinIntegrationTest { val meterProvider = createMeterProvider(clock) withComputeService(clock, meterProvider, environmentReader, allocationPolicy, performanceInterferenceModel) { scheduler -> - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, traceReader, @@ -200,8 +213,14 @@ class CapelinIntegrationTest { } } - val metrics = collectMetrics(meterProvider as MetricProducer) - println("Finish SUBMIT=${metrics.submittedVms} FAIL=${metrics.unscheduledVms} QUEUE=${metrics.queuedVms} RUNNING=${metrics.runningVms}") + val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + println( + "Finish " + + "SUBMIT=${serviceMetrics.instanceCount} " + + "FAIL=${serviceMetrics.failedInstanceCount} " + + "QUEUE=${serviceMetrics.queuedInstanceCount} " + + "RUNNING=${serviceMetrics.runningInstanceCount}" + ) // Note that these values have been verified beforehand assertAll( @@ -239,7 +258,7 @@ class CapelinIntegrationTest { chan ) - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, traceReader, @@ -252,8 +271,14 @@ class CapelinIntegrationTest { failureDomain.cancel() } - val metrics = collectMetrics(meterProvider as MetricProducer) - println("Finish SUBMIT=${metrics.submittedVms} FAIL=${metrics.unscheduledVms} QUEUE=${metrics.queuedVms} RUNNING=${metrics.runningVms}") + val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + println( + "Finish " + + "SUBMIT=${serviceMetrics.instanceCount} " + + "FAIL=${serviceMetrics.failedInstanceCount} " + + "QUEUE=${serviceMetrics.queuedInstanceCount} " + + "RUNNING=${serviceMetrics.runningInstanceCount}" + ) // Note that these values have been verified beforehand assertAll( @@ -283,34 +308,19 @@ class CapelinIntegrationTest { return ClusterEnvironmentReader(stream) } - class TestExperimentReporter : ExperimentMonitor { + class TestExperimentReporter : ComputeMonitor { var totalWork = 0L var totalGrantedWork = 0L var totalOvercommittedWork = 0L var totalInterferedWork = 0L var totalPowerDraw = 0.0 - override fun reportHostData( - time: Long, - totalWork: Double, - grantedWork: Double, - overcommittedWork: Double, - interferedWork: Double, - cpuUsage: Double, - cpuDemand: Double, - powerDraw: Double, - instanceCount: Int, - uptime: Long, - downtime: Long, - host: Host, - ) { - this.totalWork += totalWork.toLong() - totalGrantedWork += grantedWork.toLong() - totalOvercommittedWork += overcommittedWork.toLong() - totalInterferedWork += interferedWork.toLong() - totalPowerDraw += powerDraw + override fun record(data: HostData) { + this.totalWork += data.totalWork.toLong() + totalGrantedWork += data.grantedWork.toLong() + totalOvercommittedWork += data.overcommittedWork.toLong() + totalInterferedWork += data.interferedWork.toLong() + totalPowerDraw += data.powerDraw } - - override fun close() {} } } diff --git a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts index 7d34d098..40ac2967 100644 --- a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcExperiments.opendcExperimentsCapelin) implementation(projects.opendcTelemetry.opendcTelemetrySdk) + implementation(projects.opendcTelemetry.opendcTelemetryCompute) implementation(libs.kotlin.logging) implementation(libs.config) diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index e64e20a2..02aaab3c 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -37,7 +37,7 @@ import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.* -import org.opendc.experiments.capelin.monitor.ParquetExperimentMonitor +import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor import org.opendc.experiments.capelin.trace.StreamingParquetTraceReader import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf @@ -50,6 +50,8 @@ import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.* import org.opendc.simulator.core.runBlockingSimulation import org.opendc.simulator.resources.SimResourceInterpreter +import org.opendc.telemetry.compute.collectServiceMetrics +import org.opendc.telemetry.compute.withMonitor import java.io.File import java.time.Clock import java.util.* @@ -87,11 +89,11 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { ) val meterProvider: MeterProvider = createMeterProvider(clock) - val monitor = ParquetExperimentMonitor(File(config.getString("output-path")), "power_model=$powerModel/run_id=$repeat", 4096) + val monitor = ParquetExportMonitor(File(config.getString("output-path")), "power_model=$powerModel/run_id=$repeat", 4096) val trace = StreamingParquetTraceReader(File(config.getString("trace-path"), trace)) withComputeService(clock, meterProvider, allocationPolicy) { scheduler -> - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, trace, @@ -102,12 +104,12 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { } } - val monitorResults = collectMetrics(meterProvider as MetricProducer) + val monitorResults = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) logger.debug { - "Finish SUBMIT=${monitorResults.submittedVms} " + - "FAIL=${monitorResults.unscheduledVms} " + - "QUEUE=${monitorResults.queuedVms} " + - "RUNNING=${monitorResults.runningVms}" + "Finish SUBMIT=${monitorResults.instanceCount} " + + "FAIL=${monitorResults.failedInstanceCount} " + + "QUEUE=${monitorResults.queuedInstanceCount} " + + "RUNNING=${monitorResults.runningInstanceCount}" } } -- cgit v1.2.3 From 18ff316a6b6ab984ebf8283ea48ed98ec69d8295 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 2 Sep 2021 13:20:05 +0200 Subject: refactor(capelin): Restructure input reading classes --- .../opendc-experiments-capelin/build.gradle.kts | 5 ++--- .../org/opendc/experiments/capelin/Portfolio.kt | 5 +++-- .../capelin/trace/PerformanceInterferenceReader.kt | 21 ++++++++------------- .../capelin/trace/StreamingParquetTraceReader.kt | 4 ++-- .../experiments/capelin/trace/VmPlacementReader.kt | 19 +++++++------------ .../capelin/trace/sv/SvResourceStateTable.kt | 6 +++--- .../capelin/trace/sv/SvResourceStateTableReader.kt | 4 ++-- .../experiments/capelin/CapelinIntegrationTest.kt | 4 +++- .../trace/PerformanceInterferenceReaderTest.kt | 4 +--- 9 files changed, 31 insertions(+), 41 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 1a4caf91..b2330af0 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,9 +45,8 @@ dependencies { implementation(libs.config) implementation(libs.progressbar) implementation(libs.clikt) - implementation(libs.jackson.module.kotlin) { - exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") - } + implementation(libs.jackson.module.kotlin) + implementation(kotlin("reflect")) implementation(libs.parquet) testImplementation(libs.log4j.slf4j) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index d3bba182..4db04591 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -122,8 +122,9 @@ abstract class Portfolio(name: String) : Experiment(name) { } val performanceInterferenceModel = if (operationalPhenomena.hasInterference) - PerformanceInterferenceReader(FileInputStream(config.getString("interference-model"))) - .use { VmInterferenceModel(it.read(), Random(seeder.nextLong())) } + PerformanceInterferenceReader() + .read(FileInputStream(config.getString("interference-model"))) + .let { VmInterferenceModel(it, Random(seeder.nextLong())) } else null diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt index a19f5699..9549af42 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt @@ -31,14 +31,13 @@ import java.io.InputStream /** * A parser for the JSON performance interference setup files used for the TPDS article on Capelin. - * - * @param input The input stream to read from. - * @param mapper The Jackson object mapper to use. */ -class PerformanceInterferenceReader( - private val input: InputStream, - private val mapper: ObjectMapper = jacksonObjectMapper() -) : AutoCloseable { +class PerformanceInterferenceReader { + /** + * The [ObjectMapper] to use. + */ + private val mapper = jacksonObjectMapper() + init { mapper.addMixIn(VmInterferenceGroup::class.java, GroupMixin::class.java) } @@ -46,12 +45,8 @@ class PerformanceInterferenceReader( /** * Read the performance interface model from the input. */ - fun read(): List { - return mapper.readValue(input) - } - - override fun close() { - input.close() + fun read(input: InputStream): List { + return input.use { mapper.readValue(input) } } private data class GroupMixin( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt index 61e4cab5..ed82217d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt @@ -41,8 +41,6 @@ import java.util.UUID import java.util.concurrent.ArrayBlockingQueue import kotlin.concurrent.thread -private val logger = KotlinLogging.logger {} - /** * A [TraceReader] for the internal VM workload trace format that streams workloads on the fly. * @@ -50,6 +48,8 @@ private val logger = KotlinLogging.logger {} * @param selectedVms The list of VMs to read from the trace. */ class StreamingParquetTraceReader(traceFile: File, selectedVms: List = emptyList()) : TraceReader { + private val logger = KotlinLogging.logger {} + /** * The internal iterator to use for this reader. */ diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt index 7a1683f0..b55bd577 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt @@ -29,24 +29,19 @@ import java.io.InputStream /** * A parser for the JSON VM placement data files used for the TPDS article on Capelin. - * - * @param input The input stream to read from. - * @param mapper The Jackson object mapper to use. */ -public class VmPlacementReader( - private val input: InputStream, - private val mapper: ObjectMapper = jacksonObjectMapper() -) : AutoCloseable { +class VmPlacementReader { + /** + * The [ObjectMapper] to parse the placement. + */ + private val mapper = jacksonObjectMapper() + /** * Read the VM placements from the input. */ - public fun read(): Map { + fun read(input: InputStream): Map { return mapper.readValue>(input) .mapKeys { "vm__workload__${it.key}.txt" } .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 } - - override fun close() { - input.close() - } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt index 71c9d52e..24abb109 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt @@ -118,9 +118,9 @@ internal class SvResourceStateTable(path: Path) : Table { private fun nextDelegate(): TableReader? { return if (it.hasNext()) { - val (partition, path) = it.next() + val (_, path) = it.next() val reader = path.bufferedReader() - return SvResourceStateTableReader(partition, reader) + return SvResourceStateTableReader(reader) } else { null } @@ -133,7 +133,7 @@ internal class SvResourceStateTable(path: Path) : Table { override fun newReader(partition: String): TableReader { val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" } val reader = path.bufferedReader() - return SvResourceStateTableReader(partition, reader) + return SvResourceStateTableReader(reader) } override fun toString(): String = "SvResourceStateTable" diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt index adcdb2ea..1a556f8d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt @@ -29,7 +29,7 @@ import java.time.Instant /** * A [TableReader] for the Bitbrains resource state table. */ -internal class SvResourceStateTableReader(partition: String, private val reader: BufferedReader) : TableReader { +internal class SvResourceStateTableReader(private val reader: BufferedReader) : TableReader { /** * The current parser state. */ @@ -57,7 +57,7 @@ internal class SvResourceStateTableReader(partition: String, private val reader: val length = line.length var col = 0 - var start = 0 + var start: Int var end = 0 while (end < length) { diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index abd8efeb..aed9a4bb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -197,7 +197,9 @@ class CapelinIntegrationTest { val perfInterferenceInput = checkNotNull(CapelinIntegrationTest::class.java.getResourceAsStream("/bitbrains-perf-interference.json")) val performanceInterferenceModel = - PerformanceInterferenceReader(perfInterferenceInput).use { VmInterferenceModel(it.read(), Random(seed.toLong())) } + PerformanceInterferenceReader() + .read(perfInterferenceInput) + .let { VmInterferenceModel(it, Random(seed.toLong())) } val meterProvider = createMeterProvider(clock) diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt index 9b1513dc..fbc39b87 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt @@ -33,9 +33,7 @@ class PerformanceInterferenceReaderTest { @Test fun testSmoke() { val input = checkNotNull(PerformanceInterferenceReader::class.java.getResourceAsStream("/perf-interference.json")) - val reader = PerformanceInterferenceReader(input) - - val result = reader.use { reader.read() } + val result = PerformanceInterferenceReader().read(input) assertAll( { assertEquals(2, result.size) }, -- cgit v1.2.3 From f20d615e3f6e5b9d02526ac033778fb0419fed4e Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 9 Sep 2021 16:21:41 +0200 Subject: feat(simulator): Support generic distribution in fault injector This change adds support for specifying the distribution of the failures, group size and duration for the fault injector. --- .../org/opendc/experiments/capelin/ExperimentHelpers.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 0230409e..3d605300 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -26,6 +26,8 @@ import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider import kotlinx.coroutines.* import kotlinx.coroutines.channels.Channel +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.random.Well19937c import org.opendc.compute.api.* import org.opendc.compute.service.ComputeService import org.opendc.compute.service.scheduler.ComputeScheduler @@ -98,15 +100,16 @@ fun createFaultInjector( random: Random, failureInterval: Double ): FaultInjector { + val rng = Well19937c(random.nextLong()) + // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 // GRID'5000 return CorrelatedFaultInjector( coroutineScope, clock, - iatScale = ln(failureInterval), iatShape = 1.03, // Hours - sizeScale = ln(2.0), sizeShape = ln(1.0), // Expect 2 machines, with variation of 1 - dScale = ln(60.0), dShape = ln(60.0 * 8), // Minutes - random = random + iat = LogNormalDistribution(rng, ln(failureInterval), 1.03), + size = LogNormalDistribution(rng, 1.88, 1.25), + duration = LogNormalDistribution(rng, 8.89, 2.71) ) } -- cgit v1.2.3 From 04945381d01d8c6e59befe6843f2c6f6da5e91bf Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 9 Sep 2021 16:46:33 +0200 Subject: refactor(capelin): Terminate servers after reaching deadline This change updates the Capelin experiment helpers to terminate a server when it has reached its end-date. --- .../experiments/capelin/ExperimentHelpers.kt | 33 +++++++++++++--------- .../experiments/capelin/CapelinIntegrationTest.kt | 24 ++++++++-------- .../src/test/resources/env/single.txt | 2 +- 3 files changed, 33 insertions(+), 26 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 3d605300..512b754d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -54,7 +54,6 @@ import org.opendc.simulator.resources.SimResourceInterpreter import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.sdk.toOtelClock import java.time.Clock -import kotlin.coroutines.resume import kotlin.math.ln import kotlin.math.max import kotlin.random.Random @@ -169,10 +168,19 @@ suspend fun processTrace( monitor: ComputeMonitor? = null, ) { val client = scheduler.newClient() + val watcher = object : ServerWatcher { + override fun onStateChanged(server: Server, newState: ServerState) { + monitor?.onStateChange(clock.millis(), server, newState) + } + } + + // Create new image for the virtual machine val image = client.newImage("vm-image") - var offset = Long.MIN_VALUE + try { coroutineScope { + var offset = Long.MIN_VALUE + while (reader.hasNext()) { val entry = reader.next() @@ -183,9 +191,12 @@ suspend fun processTrace( // Make sure the trace entries are ordered by submission time assert(entry.start - offset >= 0) { "Invalid trace order" } delay(max(0, (entry.start - offset) - clock.millis())) + launch { chan.send(Unit) - val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, offset = -offset + 300001) + + val workloadOffset = -offset + 300001 + val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, workloadOffset) val server = client.newServer( entry.name, image, @@ -196,18 +207,14 @@ suspend fun processTrace( ), meta = entry.meta + mapOf("workload" to workload) ) + server.watch(watcher) - suspendCancellableCoroutine { cont -> - server.watch(object : ServerWatcher { - override fun onStateChanged(server: Server, newState: ServerState) { - monitor?.onStateChange(clock.millis(), server, newState) + // Wait for the server reach its end time + val endTime = entry.meta["end-time"] as Long + delay(endTime + workloadOffset - clock.millis() + 1) - if (newState == ServerState.TERMINATED) { - cont.resume(Unit) - } - } - }) - } + // Delete the server after reaching the end-time of the virtual machine + server.delete() } } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index aed9a4bb..ab33bc25 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -52,7 +52,7 @@ import java.io.File import java.util.* /** - * An integration test suite for the SC20 experiments. + * An integration test suite for the Capelin experiments. */ class CapelinIntegrationTest { /** @@ -146,7 +146,7 @@ class CapelinIntegrationTest { filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) ) - val traceReader = createTestTraceReader(0.5, seed) + val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") val meterProvider = createMeterProvider(clock) @@ -174,9 +174,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38051879552, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(34888186408, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(971668973, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(39183961335, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(35649903197, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(1043641877, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -226,10 +226,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38051879552, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(34888186408, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(971668973, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, - { assertEquals(13910814, monitor.totalInterferedWork) { "Total interfered work incorrect" } } + { assertEquals(39183961335, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(35649903197, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(1043641877, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(2960970230, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -284,9 +284,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(25412073109, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(23695061858, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(368502468, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(38385852453, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(34886665781, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(979997253, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt index 53b3c2d7..5642003d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt @@ -1,3 +1,3 @@ ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;8;3.2;64;1;64;8 +A01;A01;8;3.2;128;1;128;8 -- cgit v1.2.3 From d24cc0cc9c4fe2145f0337d65e9a75f631365973 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 10 Sep 2021 10:59:44 +0200 Subject: refactor(compute): Integrate fault injection into compute simulator This change moves the fault injection logic directly into the opendc-compute-simulator module, so that it can operate at a higher abstraction. In the future, we might again split the module if we can re-use some of its logic. --- .../opendc-experiments-capelin/build.gradle.kts | 1 - .../experiments/capelin/ExperimentHelpers.kt | 61 ++++++---------------- .../org/opendc/experiments/capelin/Portfolio.kt | 17 +++--- .../experiments/capelin/CapelinIntegrationTest.kt | 41 +++------------ .../opendc-experiments-energy21/build.gradle.kts | 1 - .../experiments/energy21/EnergyExperiment.kt | 3 -- .../opendc-experiments-radice/build.gradle.kts | 1 - 7 files changed, 29 insertions(+), 96 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index b2330af0..036d0638 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -35,7 +35,6 @@ dependencies { implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(projects.opendcTelemetry.opendcTelemetryCompute) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt index 512b754d..8227bca9 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt @@ -25,7 +25,6 @@ package org.opendc.experiments.capelin import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.SdkMeterProvider import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel import org.apache.commons.math3.distribution.LogNormalDistribution import org.apache.commons.math3.random.Well19937c import org.opendc.compute.api.* @@ -41,6 +40,9 @@ import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher import org.opendc.compute.service.scheduler.weights.RamWeigher import org.opendc.compute.service.scheduler.weights.VCpuWeigher import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.failure.HostFaultInjector +import org.opendc.compute.simulator.failure.StartStopHostFault +import org.opendc.compute.simulator.failure.StochasticVictimSelector import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.trace.TraceReader import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider @@ -48,67 +50,36 @@ import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.failures.CorrelatedFaultInjector -import org.opendc.simulator.failures.FaultInjector import org.opendc.simulator.resources.SimResourceInterpreter import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.sdk.toOtelClock import java.time.Clock +import kotlin.coroutines.CoroutineContext import kotlin.math.ln import kotlin.math.max import kotlin.random.Random -/** - * Construct the failure domain for the experiments. - */ -fun createFailureDomain( - coroutineScope: CoroutineScope, - clock: Clock, - seed: Int, - failureInterval: Double, - service: ComputeService, - chan: Channel -): CoroutineScope { - val job = coroutineScope.launch { - chan.receive() - val random = Random(seed) - val injectors = mutableMapOf() - for (host in service.hosts) { - val cluster = host.meta["cluster"] as String - val injector = - injectors.getOrPut(cluster) { - createFaultInjector( - this, - clock, - random, - failureInterval - ) - } - injector.enqueue(host as SimHost) - } - } - return CoroutineScope(coroutineScope.coroutineContext + job) -} - /** * Obtain the [FaultInjector] to use for the experiments. */ fun createFaultInjector( - coroutineScope: CoroutineScope, + context: CoroutineContext, clock: Clock, - random: Random, + hosts: Set, + seed: Int, failureInterval: Double -): FaultInjector { - val rng = Well19937c(random.nextLong()) +): HostFaultInjector { + val rng = Well19937c(seed) // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 // GRID'5000 - return CorrelatedFaultInjector( - coroutineScope, + return HostFaultInjector( + context, clock, + hosts, iat = LogNormalDistribution(rng, ln(failureInterval), 1.03), - size = LogNormalDistribution(rng, 1.88, 1.25), - duration = LogNormalDistribution(rng, 8.89, 2.71) + selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), + fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) ) } @@ -164,7 +135,6 @@ suspend fun processTrace( clock: Clock, reader: TraceReader, scheduler: ComputeService, - chan: Channel, monitor: ComputeMonitor? = null, ) { val client = scheduler.newClient() @@ -193,10 +163,9 @@ suspend fun processTrace( delay(max(0, (entry.start - offset) - clock.millis())) launch { - chan.send(Unit) - val workloadOffset = -offset + 300001 val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, workloadOffset) + val server = client.newServer( entry.name, image, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 4db04591..82794471 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -25,9 +25,8 @@ package org.opendc.experiments.capelin import com.typesafe.config.ConfigFactory import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel import mu.KotlinLogging +import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor import org.opendc.experiments.capelin.model.CompositeWorkload @@ -103,7 +102,6 @@ abstract class Portfolio(name: String) : Experiment(name) { val seeder = Random(repeat.toLong()) val environment = ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) - val chan = Channel(Channel.CONFLATED) val allocationPolicy = createComputeScheduler(allocationPolicy, seeder.asKotlinRandom(), vmPlacements) val meterProvider = createMeterProvider(clock) @@ -137,31 +135,30 @@ abstract class Portfolio(name: String) : Experiment(name) { ) withComputeService(clock, meterProvider, environment, allocationPolicy, performanceInterferenceModel) { scheduler -> - val failureDomain = if (operationalPhenomena.failureFrequency > 0) { + val faultInjector = if (operationalPhenomena.failureFrequency > 0) { logger.debug("ENABLING failures") - createFailureDomain( - this, + createFaultInjector( + coroutineContext, clock, + scheduler.hosts.map { it as SimHost }.toSet(), seeder.nextInt(), operationalPhenomena.failureFrequency, - scheduler, - chan ) } else { null } withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { + faultInjector?.start() processTrace( clock, trace, scheduler, - chan, monitor ) } - failureDomain?.cancel() + faultInjector?.close() monitor.close() } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index ab33bc25..44cf92a8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -23,8 +23,6 @@ package org.opendc.experiments.capelin import io.opentelemetry.sdk.metrics.export.MetricProducer -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -34,6 +32,7 @@ import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher +import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload @@ -73,9 +72,6 @@ class CapelinIntegrationTest { */ @Test fun testLarge() = runBlockingSimulation { - val failures = false - val seed = 0 - val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) @@ -85,31 +81,14 @@ class CapelinIntegrationTest { val meterProvider = createMeterProvider(clock) withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - val failureDomain = if (failures) { - println("ENABLING failures") - createFailureDomain( - this, - clock, - seed, - 24.0 * 7, - scheduler, - chan - ) - } else { - null - } - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { processTrace( clock, traceReader, scheduler, - chan, monitor ) } - - failureDomain?.cancel() } val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) @@ -141,7 +120,6 @@ class CapelinIntegrationTest { @Test fun testSmall() = runBlockingSimulation { val seed = 1 - val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) @@ -157,7 +135,6 @@ class CapelinIntegrationTest { clock, traceReader, scheduler, - chan, monitor ) } @@ -187,7 +164,6 @@ class CapelinIntegrationTest { @Test fun testInterference() = runBlockingSimulation { val seed = 1 - val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) @@ -209,7 +185,6 @@ class CapelinIntegrationTest { clock, traceReader, scheduler, - chan, monitor ) } @@ -239,7 +214,6 @@ class CapelinIntegrationTest { @Test fun testFailures() = runBlockingSimulation { val seed = 1 - val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) @@ -250,27 +224,26 @@ class CapelinIntegrationTest { val meterProvider = createMeterProvider(clock) withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - val failureDomain = - createFailureDomain( - this, + val faultInjector = + createFaultInjector( + coroutineContext, clock, + scheduler.hosts.map { it as SimHost }.toSet(), seed, 24.0 * 7, - scheduler, - chan ) withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { + faultInjector.start() processTrace( clock, traceReader, scheduler, - chan, monitor ) } - failureDomain.cancel() + faultInjector.close() } val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) diff --git a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts index 40ac2967..cc58e5f1 100644 --- a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts @@ -33,7 +33,6 @@ dependencies { api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcExperiments.opendcExperimentsCapelin) implementation(projects.opendcTelemetry.opendcTelemetrySdk) diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt index 02aaab3c..d9194969 100644 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt @@ -26,7 +26,6 @@ import com.typesafe.config.ConfigFactory import io.opentelemetry.api.metrics.MeterProvider import io.opentelemetry.sdk.metrics.export.MetricProducer import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import mu.KotlinLogging import org.opendc.compute.service.ComputeService @@ -81,7 +80,6 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { private val powerModel by anyOf(PowerModelType.LINEAR, PowerModelType.CUBIC, PowerModelType.INTERPOLATION) override fun doRun(repeat: Int): Unit = runBlockingSimulation { - val chan = Channel(Channel.CONFLATED) val allocationPolicy = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(1.0), RamFilter(1.0)), weighers = listOf(), @@ -98,7 +96,6 @@ public class EnergyExperiment : Experiment("Energy Modeling 2021") { clock, trace, scheduler, - chan, monitor ) } diff --git a/opendc-experiments/opendc-experiments-radice/build.gradle.kts b/opendc-experiments/opendc-experiments-radice/build.gradle.kts index c1515165..0c716183 100644 --- a/opendc-experiments/opendc-experiments-radice/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-radice/build.gradle.kts @@ -34,7 +34,6 @@ dependencies { implementation(projects.opendcFormat) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) -- cgit v1.2.3 From fa08b63bd749e9fbe1a1d04ef2ebd7a86453fa4b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 11 Sep 2021 10:52:30 +0200 Subject: perf(trace): Keep reader state in own class This change removes the external class that holds the state of the reader and instead puts the state in the reader implementation. Maintaining a separate class for the state increases the complexity and has worse performance characteristics due to the bytecode produced by Kotlin for property accesses. --- .../capelin/trace/RawParquetTraceReader.kt | 4 +- .../capelin/trace/bp/BPResourceTable.kt | 2 +- .../capelin/trace/bp/BPResourceTableReader.kt | 10 +- .../capelin/trace/sv/SvResourceStateTable.kt | 5 +- .../capelin/trace/sv/SvResourceStateTableReader.kt | 142 +++++++++------------ 5 files changed, 73 insertions(+), 90 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt index fa4e9ed8..ca937328 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt @@ -90,9 +90,9 @@ class RawParquetTraceReader(private val path: File) { } val submissionTime = reader.get(RESOURCE_START_TIME) - val endTime = reader.get(RESOURCE_END_TIME) + val endTime = reader.get(RESOURCE_STOP_TIME) val maxCores = reader.getInt(RESOURCE_NCPUS) - val requiredMemory = reader.getDouble(RESOURCE_MEM_CAPACITY) + val requiredMemory = reader.getDouble(RESOURCE_MEM_CAPACITY) / 1000.0 // Convert from KB to MB val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) val vmFragments = fragments.getValue(id).asSequence() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt index 74d1e574..bff8c55e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt @@ -38,7 +38,7 @@ internal class BPResourceTable(private val path: Path) : Table { return when (column) { RESOURCE_ID -> true RESOURCE_START_TIME -> true - RESOURCE_END_TIME -> true + RESOURCE_STOP_TIME -> true RESOURCE_NCPUS -> true RESOURCE_MEM_CAPACITY -> true else -> false diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt index 0a105783..4416aae8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt @@ -45,7 +45,7 @@ internal class BPResourceTableReader(private val reader: LocalParquetReader true RESOURCE_START_TIME -> true - RESOURCE_END_TIME -> true + RESOURCE_STOP_TIME -> true RESOURCE_NCPUS -> true RESOURCE_MEM_CAPACITY -> true else -> false @@ -59,9 +59,9 @@ internal class BPResourceTableReader(private val reader: LocalParquetReader record["id"].toString() RESOURCE_START_TIME -> Instant.ofEpochMilli(record["submissionTime"] as Long) - RESOURCE_END_TIME -> Instant.ofEpochMilli(record["endTime"] as Long) - RESOURCE_NCPUS -> record["maxCores"] - RESOURCE_MEM_CAPACITY -> (record["requiredMemory"] as Number).toDouble() + RESOURCE_STOP_TIME -> Instant.ofEpochMilli(record["endTime"] as Long) + RESOURCE_NCPUS -> getInt(RESOURCE_NCPUS) + RESOURCE_MEM_CAPACITY -> getDouble(RESOURCE_MEM_CAPACITY) else -> throw IllegalArgumentException("Invalid column") } @@ -90,7 +90,7 @@ internal class BPResourceTableReader(private val reader: LocalParquetReader (record["requiredMemory"] as Number).toDouble() + RESOURCE_MEM_CAPACITY -> (record["requiredMemory"] as Number).toDouble() * 1000.0 // MB to KB else -> throw IllegalArgumentException("Invalid column") } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt index 24abb109..3a9bda69 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt @@ -31,7 +31,7 @@ import kotlin.io.path.extension import kotlin.io.path.nameWithoutExtension /** - * The resource state [Table] in the Bitbrains format. + * The resource state [Table] in the extended Bitbrains format. */ internal class SvResourceStateTable(path: Path) : Table { /** @@ -40,6 +40,7 @@ internal class SvResourceStateTable(path: Path) : Table { private val partitions = Files.walk(path, 1) .filter { !Files.isDirectory(it) && it.extension == "txt" } .collect(Collectors.toMap({ it.nameWithoutExtension }, { it })) + .toSortedMap() override val name: String = TABLE_RESOURCE_STATES @@ -126,7 +127,7 @@ internal class SvResourceStateTable(path: Path) : Table { } } - override fun toString(): String = "BitbrainsCompositeTableReader" + override fun toString(): String = "SvCompositeTableReader" } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt index 1a556f8d..a7d2d70a 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt @@ -30,13 +30,8 @@ import java.time.Instant * A [TableReader] for the Bitbrains resource state table. */ internal class SvResourceStateTableReader(private val reader: BufferedReader) : TableReader { - /** - * The current parser state. - */ - private val state = RowState() - override fun nextRow(): Boolean { - state.reset() + reset() var line: String var num = 0 @@ -75,18 +70,18 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : val field = line.subSequence(start, end) as String when (col++) { - COL_TIMESTAMP -> state.timestamp = Instant.ofEpochSecond(field.toLong(10)) - COL_CPU_USAGE -> state.cpuUsage = field.toDouble() - COL_CPU_DEMAND -> state.cpuDemand = field.toDouble() - COL_DISK_READ -> state.diskRead = field.toDouble() - COL_DISK_WRITE -> state.diskWrite = field.toDouble() - COL_CLUSTER_ID -> state.cluster = field.trim() - COL_NCPUS -> state.cpuCores = field.toInt(10) - COL_CPU_READY_PCT -> state.cpuReadyPct = field.toDouble() - COL_POWERED_ON -> state.poweredOn = field.toInt(10) == 1 - COL_CPU_CAPACITY -> state.cpuCapacity = field.toDouble() - COL_ID -> state.id = field.trim() - COL_MEM_CAPACITY -> state.memCapacity = field.toDouble() + COL_TIMESTAMP -> timestamp = Instant.ofEpochSecond(field.toLong(10)) + COL_CPU_USAGE -> cpuUsage = field.toDouble() + COL_CPU_DEMAND -> cpuDemand = field.toDouble() + COL_DISK_READ -> diskRead = field.toDouble() + COL_DISK_WRITE -> diskWrite = field.toDouble() + COL_CLUSTER_ID -> cluster = field.trim() + COL_NCPUS -> cpuCores = field.toInt(10) + COL_CPU_READY_PCT -> cpuReadyPct = field.toDouble() + COL_POWERED_ON -> poweredOn = field.toInt(10) == 1 + COL_CPU_CAPACITY -> cpuCapacity = field.toDouble() + COL_ID -> id = field.trim() + COL_MEM_CAPACITY -> memCapacity = field.toDouble() } } @@ -113,16 +108,16 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : override fun get(column: TableColumn): T { val res: Any? = when (column) { - RESOURCE_STATE_ID -> state.id - RESOURCE_STATE_CLUSTER_ID -> state.cluster - RESOURCE_STATE_TIMESTAMP -> state.timestamp - RESOURCE_STATE_NCPUS -> state.cpuCores - RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity - RESOURCE_STATE_CPU_USAGE -> state.cpuUsage - RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsage / state.cpuCapacity - RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity - RESOURCE_STATE_DISK_READ -> state.diskRead - RESOURCE_STATE_DISK_WRITE -> state.diskWrite + RESOURCE_STATE_ID -> id + RESOURCE_STATE_CLUSTER_ID -> cluster + RESOURCE_STATE_TIMESTAMP -> timestamp + RESOURCE_STATE_NCPUS -> getInt(RESOURCE_STATE_NCPUS) + RESOURCE_STATE_CPU_CAPACITY -> getDouble(RESOURCE_STATE_CPU_CAPACITY) + RESOURCE_STATE_CPU_USAGE -> getDouble(RESOURCE_STATE_CPU_USAGE) + RESOURCE_STATE_CPU_USAGE_PCT -> getDouble(RESOURCE_STATE_CPU_USAGE_PCT) + RESOURCE_STATE_MEM_CAPACITY -> getDouble(RESOURCE_STATE_MEM_CAPACITY) + RESOURCE_STATE_DISK_READ -> getDouble(RESOURCE_STATE_DISK_READ) + RESOURCE_STATE_DISK_WRITE -> getDouble(RESOURCE_STATE_DISK_WRITE) else -> throw IllegalArgumentException("Invalid column") } @@ -132,14 +127,14 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : override fun getBoolean(column: TableColumn): Boolean { return when (column) { - RESOURCE_STATE_POWERED_ON -> state.poweredOn + RESOURCE_STATE_POWERED_ON -> poweredOn else -> throw IllegalArgumentException("Invalid column") } } override fun getInt(column: TableColumn): Int { return when (column) { - RESOURCE_STATE_NCPUS -> state.cpuCores + RESOURCE_STATE_NCPUS -> cpuCores else -> throw IllegalArgumentException("Invalid column") } } @@ -150,12 +145,13 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : override fun getDouble(column: TableColumn): Double { return when (column) { - RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity - RESOURCE_STATE_CPU_USAGE -> state.cpuUsage - RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsage / state.cpuCapacity - RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity - RESOURCE_STATE_DISK_READ -> state.diskRead - RESOURCE_STATE_DISK_WRITE -> state.diskWrite + RESOURCE_STATE_CPU_CAPACITY -> cpuCapacity + RESOURCE_STATE_CPU_USAGE -> cpuUsage + RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsage / cpuCapacity + RESOURCE_STATE_CPU_DEMAND -> cpuDemand + RESOURCE_STATE_MEM_CAPACITY -> memCapacity + RESOURCE_STATE_DISK_READ -> diskRead + RESOURCE_STATE_DISK_WRITE -> diskWrite else -> throw IllegalArgumentException("Invalid column") } } @@ -165,51 +161,37 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : } /** - * The current row state. + * State fields of the reader. */ - private class RowState { - @JvmField - var id: String? = null - @JvmField - var cluster: String? = null - @JvmField - var timestamp: Instant? = null - @JvmField - var cpuCores = -1 - @JvmField - var cpuCapacity = Double.NaN - @JvmField - var cpuUsage = Double.NaN - @JvmField - var cpuDemand = Double.NaN - @JvmField - var cpuReadyPct = Double.NaN - @JvmField - var memCapacity = Double.NaN - @JvmField - var diskRead = Double.NaN - @JvmField - var diskWrite = Double.NaN - @JvmField - var poweredOn: Boolean = false - - /** - * Reset the state. - */ - fun reset() { - id = null - timestamp = null - cluster = null - cpuCores = -1 - cpuCapacity = Double.NaN - cpuUsage = Double.NaN - cpuDemand = Double.NaN - cpuReadyPct = Double.NaN - memCapacity = Double.NaN - diskRead = Double.NaN - diskWrite = Double.NaN - poweredOn = false - } + private var id: String? = null + private var cluster: String? = null + private var timestamp: Instant? = null + private var cpuCores = -1 + private var cpuCapacity = Double.NaN + private var cpuUsage = Double.NaN + private var cpuDemand = Double.NaN + private var cpuReadyPct = Double.NaN + private var memCapacity = Double.NaN + private var diskRead = Double.NaN + private var diskWrite = Double.NaN + private var poweredOn: Boolean = false + + /** + * Reset the state of the reader. + */ + private fun reset() { + id = null + timestamp = null + cluster = null + cpuCores = -1 + cpuCapacity = Double.NaN + cpuUsage = Double.NaN + cpuDemand = Double.NaN + cpuReadyPct = Double.NaN + memCapacity = Double.NaN + diskRead = Double.NaN + diskWrite = Double.NaN + poweredOn = false } /** -- cgit v1.2.3 From 6489ebe975e9ba842fd7cce1b1136944728bb21d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 11 Sep 2021 10:54:38 +0200 Subject: fix(capelin): Parse last column in Solvinity trace format This change fixes an issue where the last column in the Solvinity traces is not parsed correctly, due to the last column having no whitespace at the end to seek to. --- .../opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt index a7d2d70a..6ea403fe 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt @@ -65,7 +65,7 @@ internal class SvResourceStateTableReader(private val reader: BufferedReader) : end = line.indexOf(' ', start) if (end < 0) { - break + end = length } val field = line.subSequence(start, end) as String -- cgit v1.2.3 From 9e8ea96270701e643f95b18d2b91583d9fca08d2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 11 Sep 2021 11:10:38 +0200 Subject: feat(capelin): Implement trace API for Azure VM trace format This change adds a trace API implementation for the Azure VM traces. --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../experiments/capelin/trace/TraceConverter.kt | 488 +++++++-------------- .../capelin/trace/azure/AzureResourceStateTable.kt | 130 ++++++ .../trace/azure/AzureResourceStateTableReader.kt | 149 +++++++ .../capelin/trace/azure/AzureResourceTable.kt | 57 +++ .../trace/azure/AzureResourceTableReader.kt | 169 +++++++ .../experiments/capelin/trace/azure/AzureTrace.kt | 46 ++ .../capelin/trace/azure/AzureTraceFormat.kt | 56 +++ .../opendc/experiments/capelin/trace/bp/Schemas.kt | 55 +++ 9 files changed, 822 insertions(+), 329 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 036d0638..7dadd14d 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { implementation(libs.progressbar) implementation(libs.clikt) implementation(libs.jackson.module.kotlin) + implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) implementation(libs.parquet) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt index a021de8d..1f3878eb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt @@ -25,80 +25,74 @@ package org.opendc.experiments.capelin.trace import com.github.ajalt.clikt.core.CliktCommand import com.github.ajalt.clikt.parameters.arguments.argument import com.github.ajalt.clikt.parameters.groups.OptionGroup -import com.github.ajalt.clikt.parameters.groups.groupChoice +import com.github.ajalt.clikt.parameters.groups.cooccurring import com.github.ajalt.clikt.parameters.options.* -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.long -import me.tongfei.progressbar.ProgressBar -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder +import com.github.ajalt.clikt.parameters.types.* +import mu.KotlinLogging import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecordBuilder import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName +import org.opendc.experiments.capelin.trace.azure.AzureTraceFormat +import org.opendc.experiments.capelin.trace.bp.BP_RESOURCES_SCHEMA +import org.opendc.experiments.capelin.trace.bp.BP_RESOURCE_STATES_SCHEMA import org.opendc.experiments.capelin.trace.sv.SvTraceFormat import org.opendc.trace.* import org.opendc.trace.bitbrains.BitbrainsTraceFormat -import org.opendc.trace.spi.TraceFormat import org.opendc.trace.util.parquet.LocalOutputFile -import java.io.BufferedReader import java.io.File -import java.io.FileReader import java.util.* import kotlin.math.max import kotlin.math.min +import kotlin.math.roundToLong + +/** + * A script to convert a trace in text format into a Parquet trace. + */ +fun main(args: Array): Unit = TraceConverterCli().main(args) /** * Represents the command for converting traces */ class TraceConverterCli : CliktCommand(name = "trace-converter") { + /** + * The logger instance for the converter. + */ + private val logger = KotlinLogging.logger {} + /** * The directory where the trace should be stored. */ - private val outputPath by option("-O", "--output", help = "path to store the trace") + private val output by option("-O", "--output", help = "path to store the trace") .file(canBeFile = false, mustExist = false) .defaultLazy { File("output") } /** * The directory where the input trace is located. */ - private val inputPath by argument("input", help = "path to the input trace") + private val input by argument("input", help = "path to the input trace") .file(canBeFile = false) /** - * The input type of the trace. + * The input format of the trace. */ - private val type by option("-t", "--type", help = "input type of trace").groupChoice( - "solvinity" to SolvinityConversion(), - "bitbrains" to BitbrainsConversion(), - "azure" to AzureConversion() - ) + private val format by option("-f", "--format", help = "input format of trace") + .choice( + "solvinity" to SvTraceFormat(), + "bitbrains" to BitbrainsTraceFormat(), + "azure" to AzureTraceFormat() + ) + .required() + + /** + * The sampling options. + */ + private val samplingOptions by SamplingOptions().cooccurring() override fun run() { - val metaSchema = SchemaBuilder - .record("meta") - .namespace("org.opendc.format.sc20") - .fields() - .name("id").type().stringType().noDefault() - .name("submissionTime").type().longType().noDefault() - .name("endTime").type().longType().noDefault() - .name("maxCores").type().intType().noDefault() - .name("requiredMemory").type().longType().noDefault() - .endRecord() - val schema = SchemaBuilder - .record("trace") - .namespace("org.opendc.format.sc20") - .fields() - .name("id").type().stringType().noDefault() - .name("time").type().longType().noDefault() - .name("duration").type().longType().noDefault() - .name("cores").type().intType().noDefault() - .name("cpuUsage").type().doubleType().noDefault() - .name("flops").type().longType().noDefault() - .endRecord() - - val metaParquet = File(outputPath, "meta.parquet") - val traceParquet = File(outputPath, "trace.parquet") + val metaParquet = File(output, "meta.parquet") + val traceParquet = File(output, "trace.parquet") if (metaParquet.exists()) { metaParquet.delete() @@ -107,324 +101,160 @@ class TraceConverterCli : CliktCommand(name = "trace-converter") { traceParquet.delete() } + val trace = format.open(input.toURI().toURL()) + + logger.info { "Building resources table" } + val metaWriter = AvroParquetWriter.builder(LocalOutputFile(metaParquet)) - .withSchema(metaSchema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) + .withSchema(BP_RESOURCES_SCHEMA) + .withCompressionCodec(CompressionCodecName.ZSTD) + .enablePageWriteChecksum() .build() + val selectedVms = metaWriter.use { convertResources(trace, it) } + + logger.info { "Wrote ${selectedVms.size} rows" } + logger.info { "Building resource states table" } + val writer = AvroParquetWriter.builder(LocalOutputFile(traceParquet)) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) + .withSchema(BP_RESOURCE_STATES_SCHEMA) + .withCompressionCodec(CompressionCodecName.ZSTD) + .enableDictionaryEncoding() + .enablePageWriteChecksum() + .withBloomFilterEnabled("id", true) + .withBloomFilterNDV("id", selectedVms.size.toLong()) .build() - try { - val type = type ?: throw IllegalArgumentException("Invalid trace conversion") - val allFragments = type.read(inputPath, metaSchema, metaWriter) - allFragments.sortWith(compareBy { it.tick }.thenBy { it.id }) - - for (fragment in allFragments) { - val record = GenericData.Record(schema) - record.put("id", fragment.id) - record.put("time", fragment.tick) - record.put("duration", fragment.duration) - record.put("cores", fragment.cores) - record.put("cpuUsage", fragment.usage) - record.put("flops", fragment.flops) - - writer.write(record) - } - } finally { - writer.close() - metaWriter.close() - } + val statesCount = writer.use { convertResourceStates(trace, it, selectedVms) } + logger.info { "Wrote $statesCount rows" } } -} -/** - * The supported trace conversions. - */ -sealed class TraceConversion(name: String) : OptionGroup(name) { /** - * Read the fragments of the trace. + * Convert the resources table for the trace. */ - abstract fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList -} - -/** - * A [TraceConversion] that uses the Trace API to perform the conversion. - */ -abstract class AbstractConversion(name: String) : TraceConversion(name) { - abstract val format: TraceFormat - - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val fragments = mutableListOf() - val trace = format.open(traceDirectory.toURI().toURL()) + private fun convertResources(trace: Trace, writer: ParquetWriter): Set { + val random = samplingOptions?.let { Random(it.seed) } + val samplingFraction = samplingOptions?.fraction ?: 1.0 val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() - var lastId: String? = null - var maxCores = Int.MIN_VALUE - var requiredMemory = Long.MIN_VALUE - var minTime = Long.MAX_VALUE - var maxTime = Long.MIN_VALUE - var lastTimestamp = Long.MIN_VALUE - - while (reader.nextRow()) { - val id = reader.get(RESOURCE_STATE_ID) - - if (lastId != null && lastId != id) { - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", lastId) - metaRecord.put("submissionTime", minTime) - metaRecord.put("endTime", maxTime) - metaRecord.put("maxCores", maxCores) - metaRecord.put("requiredMemory", requiredMemory) - metaWriter.write(metaRecord) - } - lastId = id + var hasNextRow = reader.nextRow() + val selectedVms = mutableSetOf() - val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP) - val timestampMs = timestamp.toEpochMilli() - val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) - val cores = reader.getInt(RESOURCE_STATE_NCPUS) - val memCapacity = reader.getDouble(RESOURCE_STATE_MEM_CAPACITY) + while (hasNextRow) { + var id: String + var numCpus = Int.MIN_VALUE + var memCapacity = Double.MIN_VALUE + var memUsage = Double.MIN_VALUE + var startTime = Long.MAX_VALUE + var stopTime = Long.MIN_VALUE - maxCores = max(maxCores, cores) - requiredMemory = max(requiredMemory, (memCapacity / 1000).toLong()) + do { + id = reader.get(RESOURCE_STATE_ID) - if (lastTimestamp < 0) { - lastTimestamp = timestampMs - 5 * 60 * 1000L - minTime = min(minTime, lastTimestamp) - } + val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli() + startTime = min(startTime, timestamp) + stopTime = max(stopTime, timestamp) + + numCpus = max(numCpus, reader.getInt(RESOURCE_STATE_NCPUS)) - if (fragments.isEmpty()) { - val duration = 5 * 60 * 1000L - val flops: Long = (cpuUsage * duration / 1000).toLong() - fragments.add(Fragment(id, lastTimestamp, flops, duration, cpuUsage, cores)) - } else { - val last = fragments.last() - val duration = timestampMs - lastTimestamp - val flops: Long = (cpuUsage * duration / 1000).toLong() - - // Perform run-length encoding - if (last.id == id && (duration == 0L || last.usage == cpuUsage)) { - fragments[fragments.size - 1] = last.copy(duration = last.duration + duration) - } else { - fragments.add( - Fragment( - id, - lastTimestamp, - flops, - duration, - cpuUsage, - cores - ) - ) + memCapacity = max(memCapacity, reader.getDouble(RESOURCE_STATE_MEM_CAPACITY)) + if (reader.hasColumn(RESOURCE_STATE_MEM_USAGE)) { + memUsage = max(memUsage, reader.getDouble(RESOURCE_STATE_MEM_USAGE)) } + + hasNextRow = reader.nextRow() + } while (hasNextRow && id == reader.get(RESOURCE_STATE_ID)) + + // Sample only a fraction of the VMs + if (random != null && random.nextDouble() > samplingFraction) { + continue } - val last = fragments.last() - maxTime = max(maxTime, last.tick + last.duration) - lastTimestamp = timestampMs + val builder = GenericRecordBuilder(BP_RESOURCES_SCHEMA) + + builder["id"] = id + builder["submissionTime"] = startTime + builder["endTime"] = stopTime + builder["maxCores"] = numCpus + builder["requiredMemory"] = max(memCapacity, memUsage).roundToLong() + + logger.info { "Selecting VM $id" } + + writer.write(builder.build()) + selectedVms.add(id) } - return fragments + + return selectedVms } -} -class SolvinityConversion : AbstractConversion("Solvinity") { - override val format: TraceFormat = SvTraceFormat() -} + /** + * Convert the resource states table for the trace. + */ + private fun convertResourceStates(trace: Trace, writer: ParquetWriter, selectedVms: Set): Int { + val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() -/** - * Conversion of the Bitbrains public trace. - */ -class BitbrainsConversion : AbstractConversion("Bitbrains") { - override val format: TraceFormat = BitbrainsTraceFormat() -} + var hasNextRow = reader.nextRow() + var count = 0 -/** - * Conversion of the Azure public VM trace. - */ -class AzureConversion : TraceConversion("Azure") { - private val seed by option(help = "seed for trace sampling") - .long() - .default(0) - - override fun read( - traceDirectory: File, - metaSchema: Schema, - metaWriter: ParquetWriter - ): MutableList { - val random = Random(seed) - val fraction = 0.01 - - // Read VM table - val vmIdTableCol = 0 - val coreTableCol = 9 - val provisionedMemoryTableCol = 10 - - var vmId: String - var cores: Int - var requiredMemory: Long - - val vmIds = mutableSetOf() - val vmIdToMetadata = mutableMapOf() - - BufferedReader(FileReader(File(traceDirectory, "vmtable.csv"))).use { reader -> - reader.lineSequence() - .chunked(1024) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - // Sample only a fraction of the VMs - if (random.nextDouble() > fraction) { - continue - } - - val values = line.split(",") - - // Exclude VMs with a large number of cores (not specified exactly) - if (values[coreTableCol].contains(">")) { - continue - } - - vmId = values[vmIdTableCol].trim() - cores = values[coreTableCol].trim().toInt() - requiredMemory = values[provisionedMemoryTableCol].trim().toInt() * 1_000L // GB -> MB - - vmIds.add(vmId) - vmIdToMetadata[vmId] = VmInfo(cores, requiredMemory, Long.MAX_VALUE, -1L) - } + while (hasNextRow) { + var lastTimestamp = Long.MIN_VALUE + + do { + val id = reader.get(RESOURCE_STATE_ID) + + if (id !in selectedVms) { + hasNextRow = reader.nextRow() + continue } - } - // Read VM metric reading files - val timestampCol = 0 - val vmIdCol = 1 - val cpuUsageCol = 4 - val traceInterval = 5 * 60 * 1000L - - val vmIdToFragments = mutableMapOf>() - val vmIdToLastFragment = mutableMapOf() - val allFragments = mutableListOf() - - for (i in ProgressBar.wrap((1..195).toList(), "Reading Trace")) { - val readingsFile = File(File(traceDirectory, "readings"), "readings-$i.csv") - var timestamp: Long - var cpuUsage: Double - - BufferedReader(FileReader(readingsFile)).use { reader -> - reader.lineSequence() - .chunked(128) - .forEach { lines -> - for (line in lines) { - // Ignore comments in the trace - if (line.startsWith("#") || line.isBlank()) { - continue - } - - val values = line.split(",") - vmId = values[vmIdCol].trim() - - // Ignore readings for VMs not in the sample - if (!vmIds.contains(vmId)) { - continue - } - - timestamp = values[timestampCol].trim().toLong() * 1000L - vmIdToMetadata[vmId]!!.minTime = min(vmIdToMetadata[vmId]!!.minTime, timestamp) - cpuUsage = values[cpuUsageCol].trim().toDouble() * 3_000 // MHz - vmIdToMetadata[vmId]!!.maxTime = max(vmIdToMetadata[vmId]!!.maxTime, timestamp) - - val flops: Long = (cpuUsage * 5 * 60).toLong() - val lastFragment = vmIdToLastFragment[vmId] - - vmIdToLastFragment[vmId] = - if (lastFragment != null && lastFragment.flops == 0L && flops == 0L) { - Fragment( - vmId, - lastFragment.tick, - lastFragment.flops + flops, - lastFragment.duration + traceInterval, - cpuUsage, - vmIdToMetadata[vmId]!!.cores - ) - } else { - val fragment = - Fragment( - vmId, - timestamp, - flops, - traceInterval, - cpuUsage, - vmIdToMetadata[vmId]!!.cores - ) - if (lastFragment != null) { - if (vmIdToFragments[vmId] == null) { - vmIdToFragments[vmId] = mutableListOf() - } - vmIdToFragments[vmId]!!.add(lastFragment) - allFragments.add(lastFragment) - } - fragment - } - } - } - } - } + val builder = GenericRecordBuilder(BP_RESOURCE_STATES_SCHEMA) + builder["id"] = id - for (entry in vmIdToLastFragment) { - if (entry.value != null) { - if (vmIdToFragments[entry.key] == null) { - vmIdToFragments[entry.key] = mutableListOf() + val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli() + if (lastTimestamp < 0) { + lastTimestamp = timestamp - 5 * 60 * 1000L } - vmIdToFragments[entry.key]!!.add(entry.value!!) - } - } - println("Read ${vmIdToLastFragment.size} VMs") - - for (entry in vmIdToMetadata) { - val metaRecord = GenericData.Record(metaSchema) - metaRecord.put("id", entry.key) - metaRecord.put("submissionTime", entry.value.minTime) - metaRecord.put("endTime", entry.value.maxTime) - println("${entry.value.minTime} - ${entry.value.maxTime}") - metaRecord.put("maxCores", entry.value.cores) - metaRecord.put("requiredMemory", entry.value.requiredMemory) - metaWriter.write(metaRecord) - } + val duration = timestamp - lastTimestamp + val cores = reader.getInt(RESOURCE_STATE_NCPUS) + val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) + val flops = (cpuUsage * duration / 1000.0).roundToLong() - return allFragments - } -} + builder["time"] = timestamp + builder["duration"] = duration + builder["cores"] = cores + builder["cpuUsage"] = cpuUsage + builder["flops"] = flops -data class Fragment( - val id: String, - val tick: Long, - val flops: Long, - val duration: Long, - val usage: Double, - val cores: Int -) + writer.write(builder.build()) -class VmInfo(val cores: Int, val requiredMemory: Long, var minTime: Long, var maxTime: Long) + lastTimestamp = timestamp + hasNextRow = reader.nextRow() + } while (hasNextRow && id == reader.get(RESOURCE_STATE_ID)) -/** - * A script to convert a trace in text format into a Parquet trace. - */ -fun main(args: Array): Unit = TraceConverterCli().main(args) + count++ + } + + return count + } + + /** + * Options for sampling the workload trace. + */ + private class SamplingOptions : OptionGroup() { + /** + * The fraction of VMs to sample + */ + val fraction by option("--sampling-fraction", help = "fraction of the workload to sample") + .double() + .restrictTo(0.0001, 1.0) + .required() + + /** + * The seed for sampling the trace. + */ + val seed by option("--sampling-seed", help = "seed for sampling the workload") + .long() + .default(0) + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt new file mode 100644 index 00000000..f8e57d1d --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.dataformat.csv.CsvFactory +import org.opendc.trace.* +import java.nio.file.Files +import java.nio.file.Path +import java.util.stream.Collectors +import kotlin.io.path.extension +import kotlin.io.path.nameWithoutExtension + +/** + * The resource state [Table] for the Azure v1 VM traces. + */ +internal class AzureResourceStateTable(private val factory: CsvFactory, path: Path) : Table { + /** + * The partitions that belong to the table. + */ + private val partitions = Files.walk(path, 1) + .filter { !Files.isDirectory(it) && it.extension == "csv" } + .collect(Collectors.toMap({ it.nameWithoutExtension }, { it })) + .toSortedMap() + + override val name: String = TABLE_RESOURCE_STATES + + override val isSynthetic: Boolean = false + + override fun isSupported(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_CPU_USAGE_PCT -> true + else -> false + } + } + + override fun newReader(): TableReader { + val it = partitions.iterator() + + return object : TableReader { + var delegate: TableReader? = nextDelegate() + + override fun nextRow(): Boolean { + var delegate = delegate + + while (delegate != null) { + if (delegate.nextRow()) { + break + } + + delegate.close() + delegate = nextDelegate() + } + + this.delegate = delegate + return delegate != null + } + + override fun hasColumn(column: TableColumn<*>): Boolean = delegate?.hasColumn(column) ?: false + + override fun get(column: TableColumn): T { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.get(column) + } + + override fun getBoolean(column: TableColumn): Boolean { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getBoolean(column) + } + + override fun getInt(column: TableColumn): Int { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getInt(column) + } + + override fun getLong(column: TableColumn): Long { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getLong(column) + } + + override fun getDouble(column: TableColumn): Double { + val delegate = checkNotNull(delegate) { "Invalid reader state" } + return delegate.getDouble(column) + } + + override fun close() { + delegate?.close() + } + + private fun nextDelegate(): TableReader? { + return if (it.hasNext()) { + val (_, path) = it.next() + return AzureResourceStateTableReader(factory.createParser(path.toFile())) + } else { + null + } + } + + override fun toString(): String = "AzureCompositeTableReader" + } + } + + override fun newReader(partition: String): TableReader { + val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" } + return AzureResourceStateTableReader(factory.createParser(path.toFile())) + } + + override fun toString(): String = "AzureResourceStateTable" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt new file mode 100644 index 00000000..f80c0e82 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.core.JsonToken +import com.fasterxml.jackson.dataformat.csv.CsvParser +import com.fasterxml.jackson.dataformat.csv.CsvSchema +import org.opendc.trace.* +import java.time.Instant + +/** + * A [TableReader] for the Azure v1 VM resource state table. + */ +internal class AzureResourceStateTableReader(private val parser: CsvParser) : TableReader { + init { + parser.schema = schema + } + + override fun nextRow(): Boolean { + reset() + + if (!nextStart()) { + return false + } + + while (true) { + val token = parser.nextValue() + + if (token == null || token == JsonToken.END_OBJECT) { + break + } + + when (parser.currentName) { + "timestamp" -> timestamp = Instant.ofEpochSecond(parser.longValue) + "vm id" -> id = parser.text + "avg cpu" -> cpuUsagePct = parser.doubleValue + } + } + + return true + } + + override fun hasColumn(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_STATE_ID -> true + RESOURCE_STATE_TIMESTAMP -> true + RESOURCE_STATE_CPU_USAGE_PCT -> true + else -> false + } + } + + override fun get(column: TableColumn): T { + val res: Any? = when (column) { + RESOURCE_STATE_ID -> id + RESOURCE_STATE_TIMESTAMP -> timestamp + RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsagePct + else -> throw IllegalArgumentException("Invalid column") + } + + @Suppress("UNCHECKED_CAST") + return res as T + } + + override fun getBoolean(column: TableColumn): Boolean { + throw IllegalArgumentException("Invalid column") + } + + override fun getInt(column: TableColumn): Int { + throw IllegalArgumentException("Invalid column") + } + + override fun getLong(column: TableColumn): Long { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(column: TableColumn): Double { + return when (column) { + RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsagePct + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun close() { + parser.close() + } + + /** + * Advance the parser until the next object start. + */ + private fun nextStart(): Boolean { + var token = parser.nextValue() + + while (token != null && token != JsonToken.START_OBJECT) { + token = parser.nextValue() + } + + return token != null + } + + /** + * State fields of the reader. + */ + private var id: String? = null + private var timestamp: Instant? = null + private var cpuUsagePct = Double.NaN + + /** + * Reset the state. + */ + private fun reset() { + id = null + timestamp = null + cpuUsagePct = Double.NaN + } + + companion object { + /** + * The [CsvSchema] that is used to parse the trace. + */ + private val schema = CsvSchema.builder() + .addColumn("timestamp", CsvSchema.ColumnType.NUMBER) + .addColumn("vm id", CsvSchema.ColumnType.STRING) + .addColumn("CPU min cpu", CsvSchema.ColumnType.NUMBER) + .addColumn("CPU max cpu", CsvSchema.ColumnType.NUMBER) + .addColumn("CPU avg cpu", CsvSchema.ColumnType.NUMBER) + .setAllowComments(true) + .build() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt new file mode 100644 index 00000000..bbfd25ff --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.dataformat.csv.CsvFactory +import org.opendc.trace.* +import java.nio.file.Path + +/** + * The resource [Table] for the Azure v1 VM traces. + */ +internal class AzureResourceTable(private val factory: CsvFactory, private val path: Path) : Table { + override val name: String = TABLE_RESOURCES + + override val isSynthetic: Boolean = false + + override fun isSupported(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_ID -> true + RESOURCE_START_TIME -> true + RESOURCE_STOP_TIME -> true + RESOURCE_NCPUS -> true + RESOURCE_MEM_CAPACITY -> true + else -> false + } + } + + override fun newReader(): TableReader { + return AzureResourceTableReader(factory.createParser(path.resolve("vmtable/vmtable.csv").toFile())) + } + + override fun newReader(partition: String): TableReader { + throw IllegalArgumentException("No partition $partition") + } + + override fun toString(): String = "AzureResourceTable" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt new file mode 100644 index 00000000..b712b854 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.core.JsonToken +import com.fasterxml.jackson.dataformat.csv.CsvParser +import com.fasterxml.jackson.dataformat.csv.CsvSchema +import org.apache.parquet.example.Paper.schema +import org.opendc.trace.* +import java.time.Instant + +/** + * A [TableReader] for the Azure v1 VM resources table. + */ +internal class AzureResourceTableReader(private val parser: CsvParser) : TableReader { + init { + parser.schema = schema + } + + override fun nextRow(): Boolean { + reset() + + if (!nextStart()) { + return false + } + + while (true) { + val token = parser.nextValue() + + if (token == null || token == JsonToken.END_OBJECT) { + break + } + + when (parser.currentName) { + "vm id" -> id = parser.text + "vm created" -> startTime = Instant.ofEpochSecond(parser.longValue) + "vm deleted" -> stopTime = Instant.ofEpochSecond(parser.longValue) + "vm virtual core count" -> cpuCores = parser.intValue + "vm memory" -> memCapacity = parser.doubleValue * 1e6 // GB to KB + } + } + + return true + } + + override fun hasColumn(column: TableColumn<*>): Boolean { + return when (column) { + RESOURCE_ID -> true + RESOURCE_START_TIME -> true + RESOURCE_STOP_TIME -> true + RESOURCE_NCPUS -> true + RESOURCE_MEM_CAPACITY -> true + else -> false + } + } + + override fun get(column: TableColumn): T { + val res: Any? = when (column) { + RESOURCE_ID -> id + RESOURCE_START_TIME -> startTime + RESOURCE_STOP_TIME -> stopTime + RESOURCE_NCPUS -> getInt(RESOURCE_STATE_NCPUS) + RESOURCE_MEM_CAPACITY -> getDouble(RESOURCE_STATE_MEM_CAPACITY) + else -> throw IllegalArgumentException("Invalid column") + } + + @Suppress("UNCHECKED_CAST") + return res as T + } + + override fun getBoolean(column: TableColumn): Boolean { + throw IllegalArgumentException("Invalid column") + } + + override fun getInt(column: TableColumn): Int { + return when (column) { + RESOURCE_NCPUS -> cpuCores + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun getLong(column: TableColumn): Long { + throw IllegalArgumentException("Invalid column") + } + + override fun getDouble(column: TableColumn): Double { + return when (column) { + RESOURCE_MEM_CAPACITY -> memCapacity + else -> throw IllegalArgumentException("Invalid column") + } + } + + override fun close() { + parser.close() + } + + /** + * Advance the parser until the next object start. + */ + private fun nextStart(): Boolean { + var token = parser.nextValue() + + while (token != null && token != JsonToken.START_OBJECT) { + token = parser.nextValue() + } + + return token != null + } + + /** + * State fields of the reader. + */ + private var id: String? = null + private var startTime: Instant? = null + private var stopTime: Instant? = null + private var cpuCores = -1 + private var memCapacity = Double.NaN + + /** + * Reset the state. + */ + fun reset() { + id = null + startTime = null + stopTime = null + cpuCores = -1 + memCapacity = Double.NaN + } + + companion object { + /** + * The [CsvSchema] that is used to parse the trace. + */ + private val schema = CsvSchema.builder() + .addColumn("vm id", CsvSchema.ColumnType.NUMBER) + .addColumn("subscription id", CsvSchema.ColumnType.STRING) + .addColumn("deployment id", CsvSchema.ColumnType.NUMBER) + .addColumn("timestamp vm created", CsvSchema.ColumnType.NUMBER) + .addColumn("timestamp vm deleted", CsvSchema.ColumnType.NUMBER) + .addColumn("max cpu", CsvSchema.ColumnType.NUMBER) + .addColumn("avg cpu", CsvSchema.ColumnType.NUMBER) + .addColumn("p95 cpu", CsvSchema.ColumnType.NUMBER) + .addColumn("vm category", CsvSchema.ColumnType.NUMBER) + .addColumn("vm virtual core count", CsvSchema.ColumnType.NUMBER) + .addColumn("vm memory", CsvSchema.ColumnType.NUMBER) + .setAllowComments(true) + .build() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt new file mode 100644 index 00000000..24c60bab --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.dataformat.csv.CsvFactory +import org.opendc.trace.* +import java.nio.file.Path + +/** + * [Trace] implementation for the Azure v1 VM traces. + */ +class AzureTrace internal constructor(private val factory: CsvFactory, private val path: Path) : Trace { + override val tables: List = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES) + + override fun containsTable(name: String): Boolean = name in tables + + override fun getTable(name: String): Table? { + return when (name) { + TABLE_RESOURCES -> AzureResourceTable(factory, path) + TABLE_RESOURCE_STATES -> AzureResourceStateTable(factory, path) + else -> null + } + } + + override fun toString(): String = "AzureTrace[$path]" +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt new file mode 100644 index 00000000..744e43a0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.azure + +import com.fasterxml.jackson.dataformat.csv.CsvFactory +import com.fasterxml.jackson.dataformat.csv.CsvParser +import org.opendc.trace.spi.TraceFormat +import java.net.URL +import java.nio.file.Paths +import kotlin.io.path.exists + +/** + * A format implementation for the Azure v1 format. + */ +class AzureTraceFormat : TraceFormat { + /** + * The name of this trace format. + */ + override val name: String = "azure-v1" + + /** + * The [CsvFactory] used to create the parser. + */ + private val factory = CsvFactory() + .enable(CsvParser.Feature.ALLOW_COMMENTS) + .enable(CsvParser.Feature.TRIM_SPACES) + + /** + * Open the trace file. + */ + override fun open(url: URL): AzureTrace { + val path = Paths.get(url.toURI()) + require(path.exists()) { "URL $url does not exist" } + return AzureTrace(factory, path) + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt new file mode 100644 index 00000000..7dd8161d --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.trace.bp + +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder + +/** + * Schema for the resources table in the trace. + */ +val BP_RESOURCES_SCHEMA: Schema = SchemaBuilder + .record("meta") + .namespace("org.opendc.trace.capelin") + .fields() + .requiredString("id") + .requiredLong("submissionTime") + .requiredLong("endTime") + .requiredInt("maxCores") + .requiredLong("requiredMemory") + .endRecord() + +/** + * Schema for the resource states table in the trace. + */ +val BP_RESOURCE_STATES_SCHEMA: Schema = SchemaBuilder + .record("meta") + .namespace("org.opendc.trace.capelin") + .fields() + .requiredString("id") + .requiredLong("time") + .requiredLong("duration") + .requiredInt("cores") + .requiredDouble("cpuUsage") + .requiredLong("flops") + .endRecord() -- cgit v1.2.3 From 49dd8377c8bfde1e64e411c6a6f921c768b9b53b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 12 Sep 2021 11:22:07 +0200 Subject: refactor(trace): Add API for accessing available table columns This change adds a new API to the Table interface for accessing the table columns that the table supports. This does not necessarily mean that the column will have a value for every row, but that the table format has defined this particular column. --- .../capelin/trace/azure/AzureResourceStateTable.kt | 13 ++++----- .../capelin/trace/azure/AzureResourceTable.kt | 17 +++++------- .../capelin/trace/bp/BPResourceStateTable.kt | 17 +++++------- .../capelin/trace/bp/BPResourceTable.kt | 17 +++++------- .../capelin/trace/sv/SvResourceStateTable.kt | 31 ++++++++++------------ 5 files changed, 40 insertions(+), 55 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt index f8e57d1d..f98f4b2c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt @@ -46,14 +46,11 @@ internal class AzureResourceStateTable(private val factory: CsvFactory, path: Pa override val isSynthetic: Boolean = false - override fun isSupported(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_CPU_USAGE_PCT -> true - else -> false - } - } + override val columns: List> = listOf( + RESOURCE_STATE_ID, + RESOURCE_STATE_TIMESTAMP, + RESOURCE_STATE_CPU_USAGE_PCT + ) override fun newReader(): TableReader { val it = partitions.iterator() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt index bbfd25ff..c9d4f7eb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt @@ -34,16 +34,13 @@ internal class AzureResourceTable(private val factory: CsvFactory, private val p override val isSynthetic: Boolean = false - override fun isSupported(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_ID -> true - RESOURCE_START_TIME -> true - RESOURCE_STOP_TIME -> true - RESOURCE_NCPUS -> true - RESOURCE_MEM_CAPACITY -> true - else -> false - } - } + override val columns: List> = listOf( + RESOURCE_ID, + RESOURCE_START_TIME, + RESOURCE_STOP_TIME, + RESOURCE_NCPUS, + RESOURCE_MEM_CAPACITY + ) override fun newReader(): TableReader { return AzureResourceTableReader(factory.createParser(path.resolve("vmtable/vmtable.csv").toFile())) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt index 35bfd5ef..f051bf88 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt @@ -34,16 +34,13 @@ internal class BPResourceStateTable(private val path: Path) : Table { override val name: String = TABLE_RESOURCE_STATES override val isSynthetic: Boolean = false - override fun isSupported(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_DURATION -> true - RESOURCE_STATE_NCPUS -> true - RESOURCE_STATE_CPU_USAGE -> true - else -> false - } - } + override val columns: List> = listOf( + RESOURCE_STATE_ID, + RESOURCE_STATE_TIMESTAMP, + RESOURCE_STATE_DURATION, + RESOURCE_STATE_NCPUS, + RESOURCE_STATE_CPU_USAGE, + ) override fun newReader(): TableReader { val reader = LocalParquetReader(path.resolve("trace.parquet")) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt index bff8c55e..5b0f013f 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt @@ -34,16 +34,13 @@ internal class BPResourceTable(private val path: Path) : Table { override val name: String = TABLE_RESOURCES override val isSynthetic: Boolean = false - override fun isSupported(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_ID -> true - RESOURCE_START_TIME -> true - RESOURCE_STOP_TIME -> true - RESOURCE_NCPUS -> true - RESOURCE_MEM_CAPACITY -> true - else -> false - } - } + override val columns: List> = listOf( + RESOURCE_ID, + RESOURCE_START_TIME, + RESOURCE_STOP_TIME, + RESOURCE_NCPUS, + RESOURCE_MEM_CAPACITY + ) override fun newReader(): TableReader { val reader = LocalParquetReader(path.resolve("meta.parquet")) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt index 3a9bda69..67140fe9 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt @@ -46,23 +46,20 @@ internal class SvResourceStateTable(path: Path) : Table { override val isSynthetic: Boolean = false - override fun isSupported(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_CLUSTER_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_NCPUS -> true - RESOURCE_STATE_CPU_CAPACITY -> true - RESOURCE_STATE_CPU_USAGE -> true - RESOURCE_STATE_CPU_USAGE_PCT -> true - RESOURCE_STATE_CPU_DEMAND -> true - RESOURCE_STATE_CPU_READY_PCT -> true - RESOURCE_STATE_MEM_CAPACITY -> true - RESOURCE_STATE_DISK_READ -> true - RESOURCE_STATE_DISK_WRITE -> true - else -> false - } - } + override val columns: List> = listOf( + RESOURCE_STATE_ID, + RESOURCE_STATE_CLUSTER_ID, + RESOURCE_STATE_TIMESTAMP, + RESOURCE_STATE_NCPUS, + RESOURCE_STATE_CPU_CAPACITY, + RESOURCE_STATE_CPU_USAGE, + RESOURCE_STATE_CPU_USAGE_PCT, + RESOURCE_STATE_CPU_DEMAND, + RESOURCE_STATE_CPU_READY_PCT, + RESOURCE_STATE_MEM_CAPACITY, + RESOURCE_STATE_DISK_READ, + RESOURCE_STATE_DISK_WRITE, + ) override fun newReader(): TableReader { val it = partitions.iterator() -- cgit v1.2.3 From 144d9d0c118097900c086b7fb8b1cf22a788592b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 13 Sep 2021 12:22:32 +0200 Subject: build(telemetry): Update to OpenTelemetry 1.6.0 This change updates the opentelemetry-java library to version 1.6.0. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 44cf92a8..cf88535d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -110,7 +110,7 @@ class CapelinIntegrationTest { { assertEquals(206667809529, monitor.totalGrantedWork) { "Incorrect granted burst" } }, { assertEquals(1151611104, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } }, - { assertEquals(1.7671768767192196E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, + { assertEquals(1.8175860403178412E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, ) } -- cgit v1.2.3 From 7960791430b0536df4704493c01d32e38f37f022 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Sep 2021 12:52:17 +0200 Subject: refactor(experiments): Remove energy experiments shell This change removes the energy experiments. The experiments only provided a setup for the original experiments and is not able to reproduce the results without further worker. --- .../opendc-experiments-energy21/.gitignore | 3 - .../opendc-experiments-energy21/build.gradle.kts | 47 ----- .../experiments/energy21/EnergyExperiment.kt | 208 --------------------- .../src/main/resources/application.conf | 14 -- 4 files changed, 272 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-energy21/.gitignore delete mode 100644 opendc-experiments/opendc-experiments-energy21/build.gradle.kts delete mode 100644 opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt delete mode 100644 opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-energy21/.gitignore b/opendc-experiments/opendc-experiments-energy21/.gitignore deleted file mode 100644 index 55da79f8..00000000 --- a/opendc-experiments/opendc-experiments-energy21/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -input/ -output/ -.ipynb_checkpoints diff --git a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts deleted file mode 100644 index cc58e5f1..00000000 --- a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -description = "Experiments for the OpenDC Energy work" - -/* Build configuration */ -plugins { - `experiment-conventions` - `testing-conventions` -} - -dependencies { - api(platform(projects.opendcPlatform)) - api(projects.opendcHarness.opendcHarnessApi) - implementation(projects.opendcSimulator.opendcSimulatorCore) - implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcCompute.opendcComputeSimulator) - implementation(projects.opendcExperiments.opendcExperimentsCapelin) - implementation(projects.opendcTelemetry.opendcTelemetrySdk) - implementation(projects.opendcTelemetry.opendcTelemetryCompute) - implementation(libs.kotlin.logging) - implementation(libs.config) - - implementation(libs.parquet) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } -} diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt b/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt deleted file mode 100644 index d9194969..00000000 --- a/opendc-experiments/opendc-experiments-energy21/src/main/kotlin/org/opendc/experiments/energy21/EnergyExperiment.kt +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.energy21 - -import com.typesafe.config.ConfigFactory -import io.opentelemetry.api.metrics.MeterProvider -import io.opentelemetry.sdk.metrics.export.MetricProducer -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.coroutineScope -import mu.KotlinLogging -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.ComputeScheduler -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.simulator.SimHost -import org.opendc.experiments.capelin.* -import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor -import org.opendc.experiments.capelin.trace.StreamingParquetTraceReader -import org.opendc.harness.dsl.Experiment -import org.opendc.harness.dsl.anyOf -import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider -import org.opendc.simulator.compute.kernel.cpufreq.PerformanceScalingGovernor -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.model.ProcessingNode -import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.* -import org.opendc.simulator.core.runBlockingSimulation -import org.opendc.simulator.resources.SimResourceInterpreter -import org.opendc.telemetry.compute.collectServiceMetrics -import org.opendc.telemetry.compute.withMonitor -import java.io.File -import java.time.Clock -import java.util.* - -/** - * Experiments for the OpenDC project on Energy modeling. - */ -public class EnergyExperiment : Experiment("Energy Modeling 2021") { - /** - * The logger for this portfolio instance. - */ - private val logger = KotlinLogging.logger {} - - /** - * The configuration to use. - */ - private val config = ConfigFactory.load().getConfig("opendc.experiments.energy21") - - /** - * The traces to test. - */ - private val trace by anyOf("solvinity") - - /** - * The power models to test. - */ - private val powerModel by anyOf(PowerModelType.LINEAR, PowerModelType.CUBIC, PowerModelType.INTERPOLATION) - - override fun doRun(repeat: Int): Unit = runBlockingSimulation { - val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(1.0), RamFilter(1.0)), - weighers = listOf(), - subsetSize = Int.MAX_VALUE - ) - - val meterProvider: MeterProvider = createMeterProvider(clock) - val monitor = ParquetExportMonitor(File(config.getString("output-path")), "power_model=$powerModel/run_id=$repeat", 4096) - val trace = StreamingParquetTraceReader(File(config.getString("trace-path"), trace)) - - withComputeService(clock, meterProvider, allocationPolicy) { scheduler -> - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - processTrace( - clock, - trace, - scheduler, - monitor - ) - } - } - - val monitorResults = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) - logger.debug { - "Finish SUBMIT=${monitorResults.instanceCount} " + - "FAIL=${monitorResults.failedInstanceCount} " + - "QUEUE=${monitorResults.queuedInstanceCount} " + - "RUNNING=${monitorResults.runningInstanceCount}" - } - } - - /** - * Construct the environment for a simulated compute service.. - */ - public suspend fun withComputeService( - clock: Clock, - meterProvider: MeterProvider, - scheduler: ComputeScheduler, - block: suspend CoroutineScope.(ComputeService) -> Unit - ): Unit = coroutineScope { - val model = createMachineModel() - val interpreter = SimResourceInterpreter(coroutineContext, clock) - val hosts = List(64) { id -> - SimHost( - UUID(0, id.toLong()), - "node-$id", - model, - emptyMap(), - coroutineContext, - interpreter, - meterProvider.get("opendc-compute-simulator"), - SimFairShareHypervisorProvider(), - PerformanceScalingGovernor(), - powerModel.driver - ) - } - - val serviceMeter = meterProvider.get("opendc-compute") - val service = - ComputeService(coroutineContext, clock, serviceMeter, scheduler) - - for (host in hosts) { - service.addHost(host) - } - - try { - block(this, service) - } finally { - service.close() - hosts.forEach(SimHost::close) - } - } - - /** - * The machine model based on: https://www.spec.org/power_ssj2008/results/res2020q1/power_ssj2008-20191125-01012.html - */ - private fun createMachineModel(): MachineModel { - val node = ProcessingNode("AMD", "am64", "EPYC 7742", 64) - val cpus = List(node.coreCount) { id -> ProcessingUnit(node, id, 3400.0) } - val memory = List(8) { MemoryUnit("Samsung", "Unknown", 2933.0, 16_000) } - - return MachineModel(cpus, memory) - } - - /** - * The power models to test. - */ - public enum class PowerModelType { - CUBIC { - override val driver: PowerDriver = SimplePowerDriver(CubicPowerModel(206.0, 56.4)) - }, - - LINEAR { - override val driver: PowerDriver = SimplePowerDriver(LinearPowerModel(206.0, 56.4)) - }, - - SQRT { - override val driver: PowerDriver = SimplePowerDriver(SqrtPowerModel(206.0, 56.4)) - }, - - SQUARE { - override val driver: PowerDriver = SimplePowerDriver(SquarePowerModel(206.0, 56.4)) - }, - - INTERPOLATION { - override val driver: PowerDriver = SimplePowerDriver( - InterpolationPowerModel( - listOf(56.4, 100.0, 107.0, 117.0, 127.0, 138.0, 149.0, 162.0, 177.0, 191.0, 206.0) - ) - ) - }, - - MSE { - override val driver: PowerDriver = SimplePowerDriver(MsePowerModel(206.0, 56.4, 1.4)) - }, - - ASYMPTOTIC { - override val driver: PowerDriver = SimplePowerDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, false)) - }, - - ASYMPTOTIC_DVFS { - override val driver: PowerDriver = SimplePowerDriver(AsymptoticPowerModel(206.0, 56.4, 0.3, true)) - }; - - public abstract val driver: PowerDriver - } -} diff --git a/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf b/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf deleted file mode 100644 index 263da0fe..00000000 --- a/opendc-experiments/opendc-experiments-energy21/src/main/resources/application.conf +++ /dev/null @@ -1,14 +0,0 @@ -# Default configuration for the energy experiments -opendc.experiments.energy21 { - # Path to the directory containing the input traces - trace-path = input/traces - - # Path to the output directory to write the results to - output-path = output -} - -opendc.experiments.capelin { - env-path = input/environments/ - trace-path = input/traces/ - output-path = output -} -- cgit v1.2.3 From eef8ea3ab40a4e4a12ba96f839c35c5804884bc1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 13 Sep 2021 12:25:51 +0200 Subject: refactor(capelin): Improve ParquetDataWriter implementation This change improves the ParquetDataWriter class to support more complex use-cases. It now allows subclasses to modify the writer options. In addition to this, a subclass for writing server data is added. --- .../capelin/export/parquet/ParquetDataWriter.kt | 125 ++++++++++++--------- .../capelin/export/parquet/ParquetExportMonitor.kt | 14 ++- .../export/parquet/ParquetHostDataWriter.kt | 74 +++++++----- .../export/parquet/ParquetServerDataWriter.kt | 73 ++++++++++++ .../export/parquet/ParquetServiceDataWriter.kt | 46 ++++---- 5 files changed, 225 insertions(+), 107 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt index c5cb80e2..5684bde9 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt @@ -25,11 +25,13 @@ package org.opendc.experiments.capelin.export.parquet import mu.KotlinLogging import org.apache.avro.Schema import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecordBuilder import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.example.Paper.schema import org.apache.parquet.hadoop.ParquetFileWriter +import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName import org.opendc.trace.util.parquet.LocalOutputFile -import java.io.Closeable import java.io.File import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.BlockingQueue @@ -38,50 +40,94 @@ import kotlin.concurrent.thread /** * A writer that writes data in Parquet format. */ -public open class ParquetDataWriter( - private val path: File, +abstract class ParquetDataWriter( + path: File, private val schema: Schema, - private val converter: (T, GenericData.Record) -> Unit, - private val bufferSize: Int = 4096 -) : Runnable, Closeable { + bufferSize: Int = 4096 +) : AutoCloseable { /** * The logging instance to use. */ private val logger = KotlinLogging.logger {} /** - * The writer to write the Parquet file. + * The queue of commands to process. */ - private val writer = AvroParquetWriter.builder(LocalOutputFile(path)) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.SNAPPY) - .withPageSize(4 * 1024 * 1024) // For compression - .withRowGroupSize(16 * 1024 * 1024) // For write buffering (Page size) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - .build() + private val queue: BlockingQueue = ArrayBlockingQueue(bufferSize) /** - * The queue of commands to process. + * An exception to be propagated to the actual writer. */ - private val queue: BlockingQueue = ArrayBlockingQueue(bufferSize) + private var exception: Throwable? = null /** * The thread that is responsible for writing the Parquet records. */ - private val writerThread = thread(start = false, name = this.toString()) { run() } + private val writerThread = thread(start = false, name = this.toString()) { + val writer = let { + val builder = AvroParquetWriter.builder(LocalOutputFile(path)) + .withSchema(schema) + .withCompressionCodec(CompressionCodecName.ZSTD) + .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) + buildWriter(builder) + } + + val queue = queue + val buf = mutableListOf() + var shouldStop = false + + try { + while (!shouldStop) { + try { + process(writer, queue.take()) + } catch (e: InterruptedException) { + shouldStop = true + } + + if (queue.drainTo(buf) > 0) { + for (data in buf) { + process(writer, data) + } + buf.clear() + } + } + } catch (e: Throwable) { + logger.error(e) { "Failure in Parquet data writer" } + exception = e + } finally { + writer.close() + } + } + + /** + * Build the [ParquetWriter] used to write the Parquet files. + */ + protected open fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { + return builder.build() + } + + /** + * Convert the specified [data] into a Parquet record. + */ + protected abstract fun convert(builder: GenericRecordBuilder, data: T) /** * Write the specified metrics to the database. */ - public fun write(event: T) { - queue.put(Action.Write(event)) + fun write(data: T) { + val exception = exception + if (exception != null) { + throw IllegalStateException("Writer thread failed", exception) + } + + queue.put(data) } /** * Signal the writer to stop. */ - public override fun close() { - queue.put(Action.Stop) + override fun close() { + writerThread.interrupt() writerThread.join() } @@ -90,38 +136,11 @@ public open class ParquetDataWriter( } /** - * Start the writer thread. + * Process the specified [data] to be written to the Parquet file. */ - override fun run() { - try { - loop@ while (true) { - val action = queue.take() - when (action) { - is Action.Stop -> break@loop - is Action.Write<*> -> { - val record = GenericData.Record(schema) - @Suppress("UNCHECKED_CAST") - converter(action.data as T, record) - writer.write(record) - } - } - } - } catch (e: Throwable) { - logger.error("Writer failed", e) - } finally { - writer.close() - } - } - - public sealed class Action { - /** - * A poison pill that will stop the writer thread. - */ - public object Stop : Action() - - /** - * Write the specified metrics to the database. - */ - public data class Write(val data: T) : Action() + private fun process(writer: ParquetWriter, data: T) { + val builder = GenericRecordBuilder(schema) + convert(builder, data) + writer.write(builder.build()) } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt index 79b84e9d..b057e932 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt @@ -24,22 +24,33 @@ package org.opendc.experiments.capelin.export.parquet import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.compute.table.HostData +import org.opendc.telemetry.compute.table.ServerData import org.opendc.telemetry.compute.table.ServiceData import java.io.File /** * A [ComputeMonitor] that logs the events to a Parquet file. */ -public class ParquetExportMonitor(base: File, partition: String, bufferSize: Int) : ComputeMonitor, AutoCloseable { +class ParquetExportMonitor(base: File, partition: String, bufferSize: Int) : ComputeMonitor, AutoCloseable { + private val serverWriter = ParquetServerDataWriter( + File(base, "server/$partition/data.parquet").also { it.parentFile.mkdirs() }, + bufferSize + ) + private val hostWriter = ParquetHostDataWriter( File(base, "host/$partition/data.parquet").also { it.parentFile.mkdirs() }, bufferSize ) + private val serviceWriter = ParquetServiceDataWriter( File(base, "service/$partition/data.parquet").also { it.parentFile.mkdirs() }, bufferSize ) + override fun record(data: ServerData) { + serverWriter.write(data) + } + override fun record(data: HostData) { hostWriter.write(data) } @@ -51,5 +62,6 @@ public class ParquetExportMonitor(base: File, partition: String, bufferSize: Int override fun close() { hostWriter.close() serviceWriter.close() + serverWriter.close() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt index 8912c12e..7062a275 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt @@ -25,6 +25,10 @@ package org.opendc.experiments.capelin.export.parquet import org.apache.avro.Schema import org.apache.avro.SchemaBuilder import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecordBuilder +import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.hadoop.ParquetWriter +import org.opendc.compute.service.driver.HostState import org.opendc.telemetry.compute.table.HostData import java.io.File @@ -32,42 +36,52 @@ import java.io.File * A Parquet event writer for [HostData]s. */ public class ParquetHostDataWriter(path: File, bufferSize: Int) : - ParquetDataWriter(path, schema, convert, bufferSize) { + ParquetDataWriter(path, SCHEMA, bufferSize) { - override fun toString(): String = "host-writer" + override fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { + return builder + .withDictionaryEncoding("host_id", true) + .build() + } - public companion object { - private val convert: (HostData, GenericData.Record) -> Unit = { data, record -> - record.put("host_id", data.host.name) - record.put("state", data.host.state.name) - record.put("timestamp", data.timestamp) - record.put("total_work", data.totalWork) - record.put("granted_work", data.grantedWork) - record.put("overcommitted_work", data.overcommittedWork) - record.put("interfered_work", data.interferedWork) - record.put("cpu_usage", data.cpuUsage) - record.put("cpu_demand", data.cpuDemand) - record.put("power_draw", data.powerDraw) - record.put("instance_count", data.instanceCount) - record.put("cores", data.host.model.cpuCount) - } + override fun convert(builder: GenericRecordBuilder, data: HostData) { + builder["timestamp"] = data.timestamp + builder["host_id"] = data.host.name + builder["powered_on"] = data.host.state == HostState.UP + builder["uptime"] = data.uptime + builder["downtime"] = data.downtime + builder["total_work"] = data.totalWork + builder["granted_work"] = data.grantedWork + builder["overcommitted_work"] = data.overcommittedWork + builder["interfered_work"] = data.interferedWork + builder["cpu_usage"] = data.cpuUsage + builder["cpu_demand"] = data.cpuDemand + builder["power_draw"] = data.powerDraw + builder["num_instances"] = data.instanceCount + builder["num_cpus"] = data.host.model.cpuCount + } + + override fun toString(): String = "host-writer" - private val schema: Schema = SchemaBuilder + companion object { + private val SCHEMA: Schema = SchemaBuilder .record("host") .namespace("org.opendc.telemetry.compute") .fields() - .name("timestamp").type().longType().noDefault() - .name("host_id").type().stringType().noDefault() - .name("state").type().stringType().noDefault() - .name("requested_work").type().longType().noDefault() - .name("granted_work").type().longType().noDefault() - .name("overcommitted_work").type().longType().noDefault() - .name("interfered_work").type().longType().noDefault() - .name("cpu_usage").type().doubleType().noDefault() - .name("cpu_demand").type().doubleType().noDefault() - .name("power_draw").type().doubleType().noDefault() - .name("instance_count").type().intType().noDefault() - .name("cores").type().intType().noDefault() + .requiredLong("timestamp") + .requiredString("host_id") + .requiredBoolean("powered_on") + .requiredLong("uptime") + .requiredLong("downtime") + .requiredDouble("total_work") + .requiredDouble("granted_work") + .requiredDouble("overcommitted_work") + .requiredDouble("interfered_work") + .requiredDouble("cpu_usage") + .requiredDouble("cpu_demand") + .requiredDouble("power_draw") + .requiredInt("num_instances") + .requiredInt("num_cpus") .endRecord() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt new file mode 100644 index 00000000..9904adde --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.export.parquet + +import org.apache.avro.Schema +import org.apache.avro.SchemaBuilder +import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecordBuilder +import org.apache.parquet.avro.AvroParquetWriter +import org.apache.parquet.hadoop.ParquetWriter +import org.opendc.telemetry.compute.table.ServerData +import java.io.File + +/** + * A Parquet event writer for [ServerData]s. + */ +public class ParquetServerDataWriter(path: File, bufferSize: Int) : + ParquetDataWriter(path, SCHEMA, bufferSize) { + + override fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { + return builder + .withDictionaryEncoding("server_id", true) + .withDictionaryEncoding("state", true) + .build() + } + + override fun convert(builder: GenericRecordBuilder, data: ServerData) { + builder["timestamp"] = data.timestamp + builder["server_id"] = data.server.uid.toString() + builder["state"] = data.server.state + builder["uptime"] = data.uptime + builder["downtime"] = data.downtime + builder["num_vcpus"] = data.server.flavor.cpuCount + builder["mem_capacity"] = data.server.flavor.memorySize + } + + override fun toString(): String = "server-writer" + + companion object { + private val SCHEMA: Schema = SchemaBuilder + .record("server") + .namespace("org.opendc.telemetry.compute") + .fields() + .requiredLong("timestamp") + .requiredString("server_id") + .requiredString("state") + .requiredLong("uptime") + .requiredLong("downtime") + .requiredInt("num_vcpus") + .requiredLong("mem_capacity") + .endRecord() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt index 36d630f3..e1428fe7 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt @@ -24,7 +24,7 @@ package org.opendc.experiments.capelin.export.parquet import org.apache.avro.Schema import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData +import org.apache.avro.generic.GenericRecordBuilder import org.opendc.telemetry.compute.table.ServiceData import java.io.File @@ -32,34 +32,34 @@ import java.io.File * A Parquet event writer for [ServiceData]s. */ public class ParquetServiceDataWriter(path: File, bufferSize: Int) : - ParquetDataWriter(path, schema, convert, bufferSize) { + ParquetDataWriter(path, SCHEMA, bufferSize) { - override fun toString(): String = "service-writer" + override fun convert(builder: GenericRecordBuilder, data: ServiceData) { + builder["timestamp"] = data.timestamp + builder["host_total_count"] = data.hostCount + builder["host_available_count"] = data.activeHostCount + builder["instance_total_count"] = data.instanceCount + builder["instance_active_count"] = data.runningInstanceCount + builder["instance_inactive_count"] = data.finishedInstanceCount + builder["instance_waiting_count"] = data.queuedInstanceCount + builder["instance_failed_count"] = data.failedInstanceCount + } - public companion object { - private val convert: (ServiceData, GenericData.Record) -> Unit = { data, record -> - record.put("timestamp", data.timestamp) - record.put("host_total_count", data.hostCount) - record.put("host_available_count", data.activeHostCount) - record.put("instance_total_count", data.instanceCount) - record.put("instance_active_count", data.runningInstanceCount) - record.put("instance_inactive_count", data.finishedInstanceCount) - record.put("instance_waiting_count", data.queuedInstanceCount) - record.put("instance_failed_count", data.failedInstanceCount) - } + override fun toString(): String = "service-writer" - private val schema: Schema = SchemaBuilder + companion object { + private val SCHEMA: Schema = SchemaBuilder .record("service") .namespace("org.opendc.telemetry.compute") .fields() - .name("timestamp").type().longType().noDefault() - .name("host_total_count").type().intType().noDefault() - .name("host_available_count").type().intType().noDefault() - .name("instance_total_count").type().intType().noDefault() - .name("instance_active_count").type().intType().noDefault() - .name("instance_inactive_count").type().intType().noDefault() - .name("instance_waiting_count").type().intType().noDefault() - .name("instance_failed_count").type().intType().noDefault() + .requiredLong("timestamp") + .requiredInt("host_total_count") + .requiredInt("host_available_count") + .requiredInt("instance_total_count") + .requiredInt("instance_active_count") + .requiredInt("instance_inactive_count") + .requiredInt("instance_waiting_count") + .requiredInt("instance_failed_count") .endRecord() } } -- cgit v1.2.3 From 3ca64e0110adab65526a0ccfd5b252e9f047ab10 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Sep 2021 14:41:05 +0200 Subject: refactor(telemetry): Create separate MeterProvider per service/host This change refactors the telemetry implementation by creating a separate MeterProvider per service or host. This means we have to keep track of multiple metric producers, but that we can attach resource information to each of the MeterProviders like we would in a real world scenario. --- .../experiments/capelin/ExperimentHelpers.kt | 256 --------------------- .../org/opendc/experiments/capelin/Portfolio.kt | 67 +++--- .../export/parquet/ParquetHostDataWriter.kt | 7 +- .../export/parquet/ParquetServerDataWriter.kt | 8 +- .../experiments/capelin/util/ComputeSchedulers.kt | 86 +++++++ .../capelin/util/ComputeServiceSimulator.kt | 222 ++++++++++++++++++ .../experiments/capelin/util/FailureModel.kt | 38 +++ .../experiments/capelin/util/FailureModels.kt | 97 ++++++++ .../experiments/capelin/CapelinIntegrationTest.kt | 190 ++++++++------- .../experiments/serverless/ServerlessExperiment.kt | 3 +- 10 files changed, 574 insertions(+), 400 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt deleted file mode 100644 index 8227bca9..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin - -import io.opentelemetry.api.metrics.MeterProvider -import io.opentelemetry.sdk.metrics.SdkMeterProvider -import kotlinx.coroutines.* -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.opendc.compute.api.* -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.ComputeScheduler -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.ReplayScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher -import org.opendc.compute.service.scheduler.weights.RamWeigher -import org.opendc.compute.service.scheduler.weights.VCpuWeigher -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.StartStopHostFault -import org.opendc.compute.simulator.failure.StochasticVictimSelector -import org.opendc.experiments.capelin.env.EnvironmentReader -import org.opendc.experiments.capelin.trace.TraceReader -import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider -import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.compute.power.SimplePowerDriver -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.SimResourceInterpreter -import org.opendc.telemetry.compute.ComputeMonitor -import org.opendc.telemetry.sdk.toOtelClock -import java.time.Clock -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln -import kotlin.math.max -import kotlin.random.Random - -/** - * Obtain the [FaultInjector] to use for the experiments. - */ -fun createFaultInjector( - context: CoroutineContext, - clock: Clock, - hosts: Set, - seed: Int, - failureInterval: Double -): HostFaultInjector { - val rng = Well19937c(seed) - - // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 - // GRID'5000 - return HostFaultInjector( - context, - clock, - hosts, - iat = LogNormalDistribution(rng, ln(failureInterval), 1.03), - selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), - fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) - ) -} - -/** - * Construct the environment for a simulated compute service.. - */ -suspend fun withComputeService( - clock: Clock, - meterProvider: MeterProvider, - environmentReader: EnvironmentReader, - scheduler: ComputeScheduler, - interferenceModel: VmInterferenceModel? = null, - block: suspend CoroutineScope.(ComputeService) -> Unit -): Unit = coroutineScope { - val interpreter = SimResourceInterpreter(coroutineContext, clock) - val hosts = environmentReader - .use { it.read() } - .map { def -> - SimHost( - def.uid, - def.name, - def.model, - def.meta, - coroutineContext, - interpreter, - meterProvider.get("opendc-compute-simulator"), - SimFairShareHypervisorProvider(), - powerDriver = SimplePowerDriver(def.powerModel), - interferenceDomain = interferenceModel?.newDomain() - ) - } - - val serviceMeter = meterProvider.get("opendc-compute") - val service = - ComputeService(coroutineContext, clock, serviceMeter, scheduler) - - for (host in hosts) { - service.addHost(host) - } - - try { - block(this, service) - } finally { - service.close() - hosts.forEach(SimHost::close) - } -} - -/** - * Process the trace. - */ -suspend fun processTrace( - clock: Clock, - reader: TraceReader, - scheduler: ComputeService, - monitor: ComputeMonitor? = null, -) { - val client = scheduler.newClient() - val watcher = object : ServerWatcher { - override fun onStateChanged(server: Server, newState: ServerState) { - monitor?.onStateChange(clock.millis(), server, newState) - } - } - - // Create new image for the virtual machine - val image = client.newImage("vm-image") - - try { - coroutineScope { - var offset = Long.MIN_VALUE - - while (reader.hasNext()) { - val entry = reader.next() - - if (offset < 0) { - offset = entry.start - clock.millis() - } - - // Make sure the trace entries are ordered by submission time - assert(entry.start - offset >= 0) { "Invalid trace order" } - delay(max(0, (entry.start - offset) - clock.millis())) - - launch { - val workloadOffset = -offset + 300001 - val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, workloadOffset) - - val server = client.newServer( - entry.name, - image, - client.newFlavor( - entry.name, - entry.meta["cores"] as Int, - entry.meta["required-memory"] as Long - ), - meta = entry.meta + mapOf("workload" to workload) - ) - server.watch(watcher) - - // Wait for the server reach its end time - val endTime = entry.meta["end-time"] as Long - delay(endTime + workloadOffset - clock.millis() + 1) - - // Delete the server after reaching the end-time of the virtual machine - server.delete() - } - } - } - - yield() - } finally { - reader.close() - client.close() - } -} - -/** - * Create a [MeterProvider] instance for the experiment. - */ -fun createMeterProvider(clock: Clock): MeterProvider { - return SdkMeterProvider - .builder() - .setClock(clock.toOtelClock()) - .build() -} - -/** - * Create a [ComputeScheduler] for the experiment. - */ -fun createComputeScheduler(allocationPolicy: String, seeder: Random, vmPlacements: Map = emptyMap()): ComputeScheduler { - val cpuAllocationRatio = 16.0 - val ramAllocationRatio = 1.5 - return when (allocationPolicy) { - "mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = 1.0)) - ) - "mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = -1.0)) - ) - "core-mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) - "core-mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = -1.0)) - ) - "active-servers" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = -1.0)) - ) - "active-servers-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = 1.0)) - ) - "provisioned-cores" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)) - ) - "provisioned-cores-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)) - ) - "random" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = emptyList(), - subsetSize = Int.MAX_VALUE, - random = java.util.Random(seeder.nextLong()) - ) - "replay" -> ReplayScheduler(vmPlacements) - else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 82794471..f7f9336e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -23,10 +23,7 @@ package org.opendc.experiments.capelin import com.typesafe.config.ConfigFactory -import io.opentelemetry.sdk.metrics.export.MetricProducer -import kotlinx.coroutines.ExperimentalCoroutinesApi import mu.KotlinLogging -import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor import org.opendc.experiments.capelin.model.CompositeWorkload @@ -36,17 +33,21 @@ import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader +import org.opendc.experiments.capelin.util.ComputeServiceSimulator +import org.opendc.experiments.capelin.util.createComputeScheduler import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.compute.ComputeMetricExporter import org.opendc.telemetry.compute.collectServiceMetrics -import org.opendc.telemetry.compute.withMonitor +import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File import java.io.FileInputStream +import java.time.Duration import java.util.* import java.util.concurrent.ConcurrentHashMap -import kotlin.random.asKotlinRandom +import kotlin.math.roundToLong /** * A portfolio represents a collection of scenarios are tested for the work. @@ -97,28 +98,23 @@ abstract class Portfolio(name: String) : Experiment(name) { /** * Perform a single trial for this portfolio. */ - @OptIn(ExperimentalCoroutinesApi::class) override fun doRun(repeat: Int): Unit = runBlockingSimulation { val seeder = Random(repeat.toLong()) val environment = ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) - val allocationPolicy = createComputeScheduler(allocationPolicy, seeder.asKotlinRandom(), vmPlacements) - - val meterProvider = createMeterProvider(clock) val workload = workload val workloadNames = if (workload is CompositeWorkload) { workload.workloads.map { it.name } } else { listOf(workload.name) } - val rawReaders = workloadNames.map { workloadName -> traceReaders.computeIfAbsent(workloadName) { logger.info { "Loading trace $workloadName" } RawParquetTraceReader(File(config.getString("trace-path"), workloadName)) } } - + val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) val performanceInterferenceModel = if (operationalPhenomena.hasInterference) PerformanceInterferenceReader() .read(FileInputStream(config.getString("interference-model"))) @@ -126,43 +122,36 @@ abstract class Portfolio(name: String) : Experiment(name) { else null - val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) + val computeScheduler = createComputeScheduler(allocationPolicy, seeder, vmPlacements) + val failureModel = + if (operationalPhenomena.failureFrequency > 0) + grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong()), seeder.nextInt()) + else + null + val simulator = ComputeServiceSimulator( + coroutineContext, + clock, + computeScheduler, + environment.read(), + failureModel, + performanceInterferenceModel + ) val monitor = ParquetExportMonitor( File(config.getString("output-path")), "portfolio_id=$name/scenario_id=$id/run_id=$repeat", 4096 ) + val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) - withComputeService(clock, meterProvider, environment, allocationPolicy, performanceInterferenceModel) { scheduler -> - val faultInjector = if (operationalPhenomena.failureFrequency > 0) { - logger.debug("ENABLING failures") - createFaultInjector( - coroutineContext, - clock, - scheduler.hosts.map { it as SimHost }.toSet(), - seeder.nextInt(), - operationalPhenomena.failureFrequency, - ) - } else { - null - } - - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - faultInjector?.start() - processTrace( - clock, - trace, - scheduler, - monitor - ) - } - - faultInjector?.close() - monitor.close() + try { + simulator.run(trace) + } finally { + simulator.close() + metricReader.close() } - val monitorResults = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + val monitorResults = collectServiceMetrics(clock.millis(), simulator.producers[0]) logger.debug { "Finish " + "SUBMIT=${monitorResults.instanceCount} " + diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt index 7062a275..fa00fc35 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt @@ -28,7 +28,6 @@ import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecordBuilder import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter -import org.opendc.compute.service.driver.HostState import org.opendc.telemetry.compute.table.HostData import java.io.File @@ -46,8 +45,8 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : override fun convert(builder: GenericRecordBuilder, data: HostData) { builder["timestamp"] = data.timestamp - builder["host_id"] = data.host.name - builder["powered_on"] = data.host.state == HostState.UP + builder["host_id"] = data.host.id + builder["powered_on"] = true builder["uptime"] = data.uptime builder["downtime"] = data.downtime builder["total_work"] = data.totalWork @@ -58,7 +57,7 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : builder["cpu_demand"] = data.cpuDemand builder["power_draw"] = data.powerDraw builder["num_instances"] = data.instanceCount - builder["num_cpus"] = data.host.model.cpuCount + builder["num_cpus"] = data.host.cpuCount } override fun toString(): String = "host-writer" diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt index 9904adde..bb2db4b7 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt @@ -46,12 +46,12 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : override fun convert(builder: GenericRecordBuilder, data: ServerData) { builder["timestamp"] = data.timestamp - builder["server_id"] = data.server.uid.toString() - builder["state"] = data.server.state + builder["server_id"] = data.server + // builder["state"] = data.server.state builder["uptime"] = data.uptime builder["downtime"] = data.downtime - builder["num_vcpus"] = data.server.flavor.cpuCount - builder["mem_capacity"] = data.server.flavor.memorySize + // builder["num_vcpus"] = data.server.flavor.cpuCount + // builder["mem_capacity"] = data.server.flavor.memorySize } override fun toString(): String = "server-writer" diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt new file mode 100644 index 00000000..3b7c3f0f --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("ComputeSchedulers") +package org.opendc.experiments.capelin.util + +import org.opendc.compute.service.scheduler.ComputeScheduler +import org.opendc.compute.service.scheduler.FilterScheduler +import org.opendc.compute.service.scheduler.ReplayScheduler +import org.opendc.compute.service.scheduler.filters.ComputeFilter +import org.opendc.compute.service.scheduler.filters.RamFilter +import org.opendc.compute.service.scheduler.filters.VCpuFilter +import org.opendc.compute.service.scheduler.weights.CoreRamWeigher +import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher +import org.opendc.compute.service.scheduler.weights.RamWeigher +import org.opendc.compute.service.scheduler.weights.VCpuWeigher +import java.util.* + +/** + * Create a [ComputeScheduler] for the experiment. + */ +fun createComputeScheduler(allocationPolicy: String, seeder: Random, vmPlacements: Map = emptyMap()): ComputeScheduler { + val cpuAllocationRatio = 16.0 + val ramAllocationRatio = 1.5 + return when (allocationPolicy) { + "mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = 1.0)) + ) + "mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = -1.0)) + ) + "core-mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) + "core-mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(CoreRamWeigher(multiplier = -1.0)) + ) + "active-servers" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = -1.0)) + ) + "active-servers-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(InstanceCountWeigher(multiplier = 1.0)) + ) + "provisioned-cores" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)) + ) + "provisioned-cores-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)) + ) + "random" -> FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = emptyList(), + subsetSize = Int.MAX_VALUE, + random = Random(seeder.nextLong()) + ) + "replay" -> ReplayScheduler(vmPlacements) + else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt new file mode 100644 index 00000000..065a8c93 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.util + +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.export.MetricProducer +import io.opentelemetry.sdk.resources.Resource +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.yield +import org.opendc.compute.service.ComputeService +import org.opendc.compute.service.scheduler.ComputeScheduler +import org.opendc.compute.simulator.SimHost +import org.opendc.experiments.capelin.env.MachineDef +import org.opendc.experiments.capelin.trace.TraceReader +import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider +import org.opendc.simulator.compute.kernel.SimHypervisorProvider +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel +import org.opendc.simulator.compute.power.SimplePowerDriver +import org.opendc.simulator.compute.workload.SimTraceWorkload +import org.opendc.simulator.compute.workload.SimWorkload +import org.opendc.simulator.resources.SimResourceInterpreter +import org.opendc.telemetry.compute.* +import org.opendc.telemetry.sdk.toOtelClock +import java.time.Clock +import kotlin.coroutines.CoroutineContext +import kotlin.math.max + +/** + * Helper class to manage a [ComputeService] simulation. + */ +class ComputeServiceSimulator( + private val context: CoroutineContext, + private val clock: Clock, + scheduler: ComputeScheduler, + machines: List, + private val failureModel: FailureModel? = null, + interferenceModel: VmInterferenceModel? = null, + hypervisorProvider: SimHypervisorProvider = SimFairShareHypervisorProvider() +) : AutoCloseable { + /** + * The [ComputeService] that has been configured by the manager. + */ + val service: ComputeService + + /** + * The [MetricProducer] that are used by the [ComputeService] and the simulated hosts. + */ + val producers: List + get() = _metricProducers + private val _metricProducers = mutableListOf() + + /** + * The [SimResourceInterpreter] to simulate the hosts. + */ + private val interpreter = SimResourceInterpreter(context, clock) + + /** + * The hosts that belong to this class. + */ + private val hosts = mutableSetOf() + + init { + val (service, serviceMeterProvider) = createService(scheduler) + this._metricProducers.add(serviceMeterProvider) + this.service = service + + for (def in machines) { + val (host, hostMeterProvider) = createHost(def, hypervisorProvider, interferenceModel) + this._metricProducers.add(hostMeterProvider) + hosts.add(host) + this.service.addHost(host) + } + } + + /** + * Run a simulation of the [ComputeService] by replaying the workload trace given by [reader]. + */ + suspend fun run(reader: TraceReader) { + val injector = failureModel?.createInjector(context, clock, service) + val client = service.newClient() + + // Create new image for the virtual machine + val image = client.newImage("vm-image") + + try { + coroutineScope { + // Start the fault injector + injector?.start() + + var offset = Long.MIN_VALUE + + while (reader.hasNext()) { + val entry = reader.next() + + if (offset < 0) { + offset = entry.start - clock.millis() + } + + // Make sure the trace entries are ordered by submission time + assert(entry.start - offset >= 0) { "Invalid trace order" } + delay(max(0, (entry.start - offset) - clock.millis())) + + launch { + val workloadOffset = -offset + 300001 + val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, workloadOffset) + + val server = client.newServer( + entry.name, + image, + client.newFlavor( + entry.name, + entry.meta["cores"] as Int, + entry.meta["required-memory"] as Long + ), + meta = entry.meta + mapOf("workload" to workload) + ) + + // Wait for the server reach its end time + val endTime = entry.meta["end-time"] as Long + delay(endTime + workloadOffset - clock.millis() + 1) + + // Delete the server after reaching the end-time of the virtual machine + server.delete() + } + } + } + + yield() + } finally { + injector?.close() + reader.close() + client.close() + } + } + + override fun close() { + service.close() + + for (host in hosts) { + host.close() + } + + hosts.clear() + } + + /** + * Construct a [ComputeService] instance. + */ + private fun createService(scheduler: ComputeScheduler): Pair { + val resource = Resource.builder() + .put(ResourceAttributes.SERVICE_NAME, "opendc-compute") + .build() + + val meterProvider = SdkMeterProvider.builder() + .setClock(clock.toOtelClock()) + .setResource(resource) + .build() + + val service = ComputeService(context, clock, meterProvider, scheduler) + return service to meterProvider + } + + /** + * Construct a [SimHost] instance for the specified [MachineDef]. + */ + private fun createHost( + def: MachineDef, + hypervisorProvider: SimHypervisorProvider, + interferenceModel: VmInterferenceModel? = null + ): Pair { + val resource = Resource.builder() + .put(HOST_ID, def.uid.toString()) + .put(HOST_NAME, def.name) + .put(HOST_ARCH, ResourceAttributes.HostArchValues.AMD64) + .put(HOST_NCPUS, def.model.cpus.size) + .put(HOST_MEM_CAPACITY, def.model.memory.sumOf { it.size }) + .build() + + val meterProvider = SdkMeterProvider.builder() + .setClock(clock.toOtelClock()) + .setResource(resource) + .build() + + val host = SimHost( + def.uid, + def.name, + def.model, + def.meta, + context, + interpreter, + meterProvider, + hypervisorProvider, + powerDriver = SimplePowerDriver(def.powerModel), + interferenceDomain = interferenceModel?.newDomain() + ) + + return host to meterProvider + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt new file mode 100644 index 00000000..83393896 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.util + +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.failure.HostFaultInjector +import java.time.Clock +import kotlin.coroutines.CoroutineContext + +/** + * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. + */ +interface FailureModel { + /** + * Construct a [HostFaultInjector] for the specified [service]. + */ + fun createInjector(context: CoroutineContext, clock: Clock, service: ComputeService): HostFaultInjector +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt new file mode 100644 index 00000000..89b4a31c --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("FailureModels") +package org.opendc.experiments.capelin + +import org.apache.commons.math3.distribution.LogNormalDistribution +import org.apache.commons.math3.random.Well19937c +import org.opendc.compute.service.ComputeService +import org.opendc.compute.simulator.SimHost +import org.opendc.compute.simulator.failure.HostFaultInjector +import org.opendc.compute.simulator.failure.StartStopHostFault +import org.opendc.compute.simulator.failure.StochasticVictimSelector +import org.opendc.experiments.capelin.util.FailureModel +import java.time.Clock +import java.time.Duration +import kotlin.coroutines.CoroutineContext +import kotlin.math.ln +import kotlin.random.Random + +/** + * Obtain a [FailureModel] based on the GRID'5000 failure trace. + * + * This fault injector uses parameters from the GRID'5000 failure trace as described in + * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. + */ +fun grid5000(failureInterval: Duration, seed: Int): FailureModel { + return object : FailureModel { + override fun createInjector( + context: CoroutineContext, + clock: Clock, + service: ComputeService + ): HostFaultInjector { + val rng = Well19937c(seed) + val hosts = service.hosts.map { it as SimHost }.toSet() + + // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 + // GRID'5000 + return HostFaultInjector( + context, + clock, + hosts, + iat = LogNormalDistribution(rng, ln(failureInterval.toHours().toDouble()), 1.03), + selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), + fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) + ) + } + + override fun toString(): String = "Grid5000FailureModel" + } +} + +/** + * Obtain the [HostFaultInjector] to use for the experiments. + * + * This fault injector uses parameters from the GRID'5000 failure trace as described in + * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. + */ +fun createFaultInjector( + context: CoroutineContext, + clock: Clock, + hosts: Set, + seed: Int, + failureInterval: Double +): HostFaultInjector { + val rng = Well19937c(seed) + + // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 + // GRID'5000 + return HostFaultInjector( + context, + clock, + hosts, + iat = LogNormalDistribution(rng, ln(failureInterval), 1.03), + selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), + fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) + ) +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index cf88535d..f4cf3e5e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -22,7 +22,6 @@ package org.opendc.experiments.capelin -import io.opentelemetry.sdk.metrics.export.MetricProducer import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -32,7 +31,6 @@ import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.simulator.SimHost import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload @@ -40,15 +38,19 @@ import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader import org.opendc.experiments.capelin.trace.RawParquetTraceReader import org.opendc.experiments.capelin.trace.TraceReader +import org.opendc.experiments.capelin.util.ComputeServiceSimulator import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.compute.ComputeMetricExporter import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.compute.collectServiceMetrics import org.opendc.telemetry.compute.table.HostData -import org.opendc.telemetry.compute.withMonitor +import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File +import java.time.Duration import java.util.* +import kotlin.math.roundToLong /** * An integration test suite for the Capelin experiments. @@ -59,12 +61,21 @@ class CapelinIntegrationTest { */ private lateinit var monitor: TestExperimentReporter + /** + * The [FilterScheduler] to use for all experiments. + */ + private lateinit var computeScheduler: FilterScheduler + /** * Setup the experimental environment. */ @BeforeEach fun setUp() { monitor = TestExperimentReporter() + computeScheduler = FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) } /** @@ -72,26 +83,26 @@ class CapelinIntegrationTest { */ @Test fun testLarge() = runBlockingSimulation { - val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) val traceReader = createTestTraceReader() val environmentReader = createTestEnvironmentReader() - val meterProvider = createMeterProvider(clock) - withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - processTrace( - clock, - traceReader, - scheduler, - monitor - ) - } + val simulator = ComputeServiceSimulator( + coroutineContext, + clock, + computeScheduler, + environmentReader.read(), + ) + + val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + + try { + simulator.run(traceReader) + } finally { + simulator.close() + metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( "Finish " + "SUBMIT=${serviceMetrics.instanceCount} " + @@ -106,11 +117,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.runningInstanceCount, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.failedInstanceCount, "No VM should not be unscheduled") }, { assertEquals(0, serviceMetrics.queuedInstanceCount, "No VM should not be in the queue") }, - { assertEquals(220346369753, monitor.totalWork) { "Incorrect requested burst" } }, - { assertEquals(206667809529, monitor.totalGrantedWork) { "Incorrect granted burst" } }, - { assertEquals(1151611104, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, + { assertEquals(220346412191, monitor.totalWork) { "Incorrect requested burst" } }, + { assertEquals(206667852689, monitor.totalGrantedWork) { "Incorrect granted burst" } }, + { assertEquals(1151612221, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } }, - { assertEquals(1.8175860403178412E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, + { assertEquals(9.088769763540529E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, ) } @@ -120,27 +131,26 @@ class CapelinIntegrationTest { @Test fun testSmall() = runBlockingSimulation { val seed = 1 - val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") - val meterProvider = createMeterProvider(clock) - - withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - processTrace( - clock, - traceReader, - scheduler, - monitor - ) - } + val simulator = ComputeServiceSimulator( + coroutineContext, + clock, + computeScheduler, + environmentReader.read(), + ) + + val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + + try { + simulator.run(traceReader) + } finally { + simulator.close() + metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( "Finish " + "SUBMIT=${serviceMetrics.instanceCount} " + @@ -151,9 +161,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(39183961335, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(35649903197, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(1043641877, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(39183965664, monitor.totalWork) { "Total work incorrect" } }, + { assertEquals(35649907631, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(1043642275, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -164,10 +174,6 @@ class CapelinIntegrationTest { @Test fun testInterference() = runBlockingSimulation { val seed = 1 - val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") @@ -177,20 +183,24 @@ class CapelinIntegrationTest { .read(perfInterferenceInput) .let { VmInterferenceModel(it, Random(seed.toLong())) } - val meterProvider = createMeterProvider(clock) - - withComputeService(clock, meterProvider, environmentReader, allocationPolicy, performanceInterferenceModel) { scheduler -> - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - processTrace( - clock, - traceReader, - scheduler, - monitor - ) - } + val simulator = ComputeServiceSimulator( + coroutineContext, + clock, + computeScheduler, + environmentReader.read(), + interferenceModel = performanceInterferenceModel + ) + + val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + + try { + simulator.run(traceReader) + } finally { + simulator.close() + metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( "Finish " + "SUBMIT=${serviceMetrics.instanceCount} " + @@ -201,10 +211,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(39183961335, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(35649903197, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(1043641877, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, - { assertEquals(2960970230, monitor.totalInterferedWork) { "Total interfered work incorrect" } } + { assertEquals(39183965664, monitor.totalWork) { "Total work incorrect" } }, + { assertEquals(35649907631, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(1043642275, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(2960974524, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -214,39 +224,27 @@ class CapelinIntegrationTest { @Test fun testFailures() = runBlockingSimulation { val seed = 1 - val allocationPolicy = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") - val meterProvider = createMeterProvider(clock) - - withComputeService(clock, meterProvider, environmentReader, allocationPolicy) { scheduler -> - val faultInjector = - createFaultInjector( - coroutineContext, - clock, - scheduler.hosts.map { it as SimHost }.toSet(), - seed, - 24.0 * 7, - ) - - withMonitor(scheduler, clock, meterProvider as MetricProducer, monitor) { - faultInjector.start() - processTrace( - clock, - traceReader, - scheduler, - monitor - ) - } - - faultInjector.close() + val simulator = ComputeServiceSimulator( + coroutineContext, + clock, + computeScheduler, + environmentReader.read(), + grid5000(Duration.ofDays(7), seed) + ) + + val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + + try { + simulator.run(traceReader) + } finally { + simulator.close() + metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), meterProvider as MetricProducer) + val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( "Finish " + "SUBMIT=${serviceMetrics.instanceCount} " + @@ -257,9 +255,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38385852453, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(34886665781, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(979997253, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, + { assertEquals(38385856700, monitor.totalWork) { "Total requested work incorrect" } }, + { assertEquals(34886670127, monitor.totalGrantedWork) { "Total granted work incorrect" } }, + { assertEquals(979997628, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } ) } @@ -291,10 +289,10 @@ class CapelinIntegrationTest { var totalPowerDraw = 0.0 override fun record(data: HostData) { - this.totalWork += data.totalWork.toLong() - totalGrantedWork += data.grantedWork.toLong() - totalOvercommittedWork += data.overcommittedWork.toLong() - totalInterferedWork += data.interferedWork.toLong() + this.totalWork += data.totalWork.roundToLong() + totalGrantedWork += data.grantedWork.roundToLong() + totalOvercommittedWork += data.overcommittedWork.roundToLong() + totalInterferedWork += data.interferedWork.roundToLong() totalPowerDraw += data.powerDraw } } diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt index 650416f5..3312d6c0 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/ServerlessExperiment.kt @@ -46,6 +46,7 @@ import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.core.runBlockingSimulation import org.opendc.telemetry.sdk.toOtelClock import java.io.File +import java.time.Duration import java.util.* import kotlin.math.max @@ -85,7 +86,7 @@ public class ServerlessExperiment : Experiment("Serverless") { val delayInjector = StochasticDelayInjector(coldStartModel, Random()) val deployer = SimFunctionDeployer(clock, this, createMachineModel(), delayInjector) { FunctionTraceWorkload(traceById.getValue(it.name)) } val service = - FaaSService(coroutineContext, clock, meterProvider.get("opendc-serverless"), deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = 10L * 60 * 1000)) + FaaSService(coroutineContext, clock, meterProvider, deployer, routingPolicy, FunctionTerminationPolicyFixed(coroutineContext, clock, timeout = Duration.ofMinutes(10))) val client = service.newClient() coroutineScope { -- cgit v1.2.3 From 8d899e29dbd757f6df320212d6e0d77ce8216ab9 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Sep 2021 15:38:38 +0200 Subject: refactor(telemetry): Standardize compute scheduler metrics This change updates the OpenDC compute service implementation with multiple meters that follow the OpenTelemetry conventions. --- .../org/opendc/experiments/capelin/Portfolio.kt | 11 ++--- .../export/parquet/ParquetServiceDataWriter.kt | 28 ++++++------ .../experiments/capelin/CapelinIntegrationTest.kt | 52 ++++++++++++---------- 3 files changed, 48 insertions(+), 43 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index f7f9336e..3ec424f1 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -153,11 +153,12 @@ abstract class Portfolio(name: String) : Experiment(name) { val monitorResults = collectServiceMetrics(clock.millis(), simulator.producers[0]) logger.debug { - "Finish " + - "SUBMIT=${monitorResults.instanceCount} " + - "FAIL=${monitorResults.failedInstanceCount} " + - "QUEUE=${monitorResults.queuedInstanceCount} " + - "RUNNING=${monitorResults.activeHostCount}" + "Scheduler " + + "Success=${monitorResults.attemptsSuccess} " + + "Failure=${monitorResults.attemptsFailure} " + + "Error=${monitorResults.attemptsError} " + + "Pending=${monitorResults.serversPending} " + + "Active=${monitorResults.serversActive}" } } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt index e1428fe7..29b48878 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt @@ -36,13 +36,13 @@ public class ParquetServiceDataWriter(path: File, bufferSize: Int) : override fun convert(builder: GenericRecordBuilder, data: ServiceData) { builder["timestamp"] = data.timestamp - builder["host_total_count"] = data.hostCount - builder["host_available_count"] = data.activeHostCount - builder["instance_total_count"] = data.instanceCount - builder["instance_active_count"] = data.runningInstanceCount - builder["instance_inactive_count"] = data.finishedInstanceCount - builder["instance_waiting_count"] = data.queuedInstanceCount - builder["instance_failed_count"] = data.failedInstanceCount + builder["hosts_up"] = data.hostsUp + builder["hosts_down"] = data.hostsDown + builder["servers_pending"] = data.serversPending + builder["servers_active"] = data.serversActive + builder["attempts_success"] = data.attemptsSuccess + builder["attempts_failure"] = data.attemptsFailure + builder["attempts_error"] = data.attemptsError } override fun toString(): String = "service-writer" @@ -53,13 +53,13 @@ public class ParquetServiceDataWriter(path: File, bufferSize: Int) : .namespace("org.opendc.telemetry.compute") .fields() .requiredLong("timestamp") - .requiredInt("host_total_count") - .requiredInt("host_available_count") - .requiredInt("instance_total_count") - .requiredInt("instance_active_count") - .requiredInt("instance_inactive_count") - .requiredInt("instance_waiting_count") - .requiredInt("instance_failed_count") + .requiredInt("hosts_up") + .requiredInt("hosts_down") + .requiredInt("servers_pending") + .requiredInt("servers_active") + .requiredInt("attempts_success") + .requiredInt("attempts_failure") + .requiredInt("attempts_error") .endRecord() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index f4cf3e5e..81405acf 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -104,19 +104,20 @@ class CapelinIntegrationTest { val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( - "Finish " + - "SUBMIT=${serviceMetrics.instanceCount} " + - "FAIL=${serviceMetrics.failedInstanceCount} " + - "QUEUE=${serviceMetrics.queuedInstanceCount} " + - "RUNNING=${serviceMetrics.runningInstanceCount}" + "Scheduler " + + "Success=${serviceMetrics.attemptsSuccess} " + + "Failure=${serviceMetrics.attemptsFailure} " + + "Error=${serviceMetrics.attemptsError} " + + "Pending=${serviceMetrics.serversPending} " + + "Active=${serviceMetrics.serversActive}" ) // Note that these values have been verified beforehand assertAll( - { assertEquals(50, serviceMetrics.instanceCount, "The trace contains 50 VMs") }, - { assertEquals(0, serviceMetrics.runningInstanceCount, "All VMs should finish after a run") }, - { assertEquals(0, serviceMetrics.failedInstanceCount, "No VM should not be unscheduled") }, - { assertEquals(0, serviceMetrics.queuedInstanceCount, "No VM should not be in the queue") }, + { assertEquals(50, serviceMetrics.attemptsSuccess, "The scheduler should schedule 50 VMs") }, + { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, + { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, + { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, { assertEquals(220346412191, monitor.totalWork) { "Incorrect requested burst" } }, { assertEquals(206667852689, monitor.totalGrantedWork) { "Incorrect granted burst" } }, { assertEquals(1151612221, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, @@ -152,11 +153,12 @@ class CapelinIntegrationTest { val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( - "Finish " + - "SUBMIT=${serviceMetrics.instanceCount} " + - "FAIL=${serviceMetrics.failedInstanceCount} " + - "QUEUE=${serviceMetrics.queuedInstanceCount} " + - "RUNNING=${serviceMetrics.runningInstanceCount}" + "Scheduler " + + "Success=${serviceMetrics.attemptsSuccess} " + + "Failure=${serviceMetrics.attemptsFailure} " + + "Error=${serviceMetrics.attemptsError} " + + "Pending=${serviceMetrics.serversPending} " + + "Active=${serviceMetrics.serversActive}" ) // Note that these values have been verified beforehand @@ -202,11 +204,12 @@ class CapelinIntegrationTest { val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( - "Finish " + - "SUBMIT=${serviceMetrics.instanceCount} " + - "FAIL=${serviceMetrics.failedInstanceCount} " + - "QUEUE=${serviceMetrics.queuedInstanceCount} " + - "RUNNING=${serviceMetrics.runningInstanceCount}" + "Scheduler " + + "Success=${serviceMetrics.attemptsSuccess} " + + "Failure=${serviceMetrics.attemptsFailure} " + + "Error=${serviceMetrics.attemptsError} " + + "Pending=${serviceMetrics.serversPending} " + + "Active=${serviceMetrics.serversActive}" ) // Note that these values have been verified beforehand @@ -246,11 +249,12 @@ class CapelinIntegrationTest { val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) println( - "Finish " + - "SUBMIT=${serviceMetrics.instanceCount} " + - "FAIL=${serviceMetrics.failedInstanceCount} " + - "QUEUE=${serviceMetrics.queuedInstanceCount} " + - "RUNNING=${serviceMetrics.runningInstanceCount}" + "Scheduler " + + "Success=${serviceMetrics.attemptsSuccess} " + + "Failure=${serviceMetrics.attemptsFailure} " + + "Error=${serviceMetrics.attemptsError} " + + "Pending=${serviceMetrics.serversPending} " + + "Active=${serviceMetrics.serversActive}" ) // Note that these values have been verified beforehand -- cgit v1.2.3 From 0d8bccc68705d036fbf60f312d9c34ca4392c6b2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 7 Sep 2021 17:30:46 +0200 Subject: refactor(telemetry): Standardize SimHost metrics This change standardizes the metrics emitted by SimHost instances and their guests based on the OpenTelemetry semantic conventions. We now also report CPU time as opposed to CPU work as this metric is more commonly used. --- .../org/opendc/experiments/capelin/Portfolio.kt | 3 +- .../capelin/export/parquet/ParquetDataWriter.kt | 1 - .../export/parquet/ParquetHostDataWriter.kt | 56 +++++++++++------- .../export/parquet/ParquetServerDataWriter.kt | 38 ++++++++++--- .../export/parquet/ParquetServiceDataWriter.kt | 2 +- .../experiments/capelin/CapelinIntegrationTest.kt | 66 +++++++++++----------- 6 files changed, 101 insertions(+), 65 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 3ec424f1..6261ebbf 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -149,9 +149,10 @@ abstract class Portfolio(name: String) : Experiment(name) { } finally { simulator.close() metricReader.close() + monitor.close() } - val monitorResults = collectServiceMetrics(clock.millis(), simulator.producers[0]) + val monitorResults = collectServiceMetrics(clock.instant(), simulator.producers[0]) logger.debug { "Scheduler " + "Success=${monitorResults.attemptsSuccess} " + diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt index 5684bde9..e3d15c3b 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt @@ -27,7 +27,6 @@ import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.avro.generic.GenericRecordBuilder import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.example.Paper.schema import org.apache.parquet.hadoop.ParquetFileWriter import org.apache.parquet.hadoop.ParquetWriter import org.apache.parquet.hadoop.metadata.CompressionCodecName diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt index fa00fc35..36207045 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt @@ -44,20 +44,31 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : } override fun convert(builder: GenericRecordBuilder, data: HostData) { - builder["timestamp"] = data.timestamp + builder["timestamp"] = data.timestamp.toEpochMilli() + builder["host_id"] = data.host.id - builder["powered_on"] = true + builder["num_cpus"] = data.host.cpuCount + builder["mem_capacity"] = data.host.memCapacity + builder["uptime"] = data.uptime builder["downtime"] = data.downtime - builder["total_work"] = data.totalWork - builder["granted_work"] = data.grantedWork - builder["overcommitted_work"] = data.overcommittedWork - builder["interfered_work"] = data.interferedWork - builder["cpu_usage"] = data.cpuUsage - builder["cpu_demand"] = data.cpuDemand - builder["power_draw"] = data.powerDraw - builder["num_instances"] = data.instanceCount - builder["num_cpus"] = data.host.cpuCount + val bootTime = data.bootTime + if (bootTime != null) { + builder["boot_time"] = bootTime.toEpochMilli() + } + + builder["cpu_limit"] = data.cpuLimit + builder["cpu_time_active"] = data.cpuActiveTime + builder["cpu_time_idle"] = data.cpuIdleTime + builder["cpu_time_steal"] = data.cpuStealTime + builder["cpu_time_lost"] = data.cpuLostTime + + builder["power_total"] = data.powerTotal + + builder["guests_terminated"] = data.guestsTerminated + builder["guests_running"] = data.guestsRunning + builder["guests_error"] = data.guestsError + builder["guests_invalid"] = data.guestsInvalid } override fun toString(): String = "host-writer" @@ -69,18 +80,21 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : .fields() .requiredLong("timestamp") .requiredString("host_id") - .requiredBoolean("powered_on") + .requiredInt("num_cpus") + .requiredLong("mem_capacity") .requiredLong("uptime") .requiredLong("downtime") - .requiredDouble("total_work") - .requiredDouble("granted_work") - .requiredDouble("overcommitted_work") - .requiredDouble("interfered_work") - .requiredDouble("cpu_usage") - .requiredDouble("cpu_demand") - .requiredDouble("power_draw") - .requiredInt("num_instances") - .requiredInt("num_cpus") + .optionalLong("boot_time") + .requiredDouble("cpu_limit") + .requiredLong("cpu_time_active") + .requiredLong("cpu_time_idle") + .requiredLong("cpu_time_steal") + .requiredLong("cpu_time_lost") + .requiredDouble("power_total") + .requiredInt("guests_terminated") + .requiredInt("guests_running") + .requiredInt("guests_error") + .requiredInt("guests_invalid") .endRecord() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt index bb2db4b7..c5a5e7c0 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt @@ -40,18 +40,31 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : override fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { return builder .withDictionaryEncoding("server_id", true) - .withDictionaryEncoding("state", true) + .withDictionaryEncoding("host_id", true) .build() } override fun convert(builder: GenericRecordBuilder, data: ServerData) { - builder["timestamp"] = data.timestamp - builder["server_id"] = data.server - // builder["state"] = data.server.state + builder["timestamp"] = data.timestamp.toEpochMilli() + + builder["server_id"] = data.server.id + builder["host_id"] = data.host?.id + builder["num_vcpus"] = data.server.cpuCount + builder["mem_capacity"] = data.server.memCapacity + builder["uptime"] = data.uptime builder["downtime"] = data.downtime - // builder["num_vcpus"] = data.server.flavor.cpuCount - // builder["mem_capacity"] = data.server.flavor.memorySize + val bootTime = data.bootTime + if (bootTime != null) { + builder["boot_time"] = bootTime.toEpochMilli() + } + builder["scheduling_latency"] = data.schedulingLatency + + builder["cpu_limit"] = data.cpuLimit + builder["cpu_time_active"] = data.cpuActiveTime + builder["cpu_time_idle"] = data.cpuIdleTime + builder["cpu_time_steal"] = data.cpuStealTime + builder["cpu_time_lost"] = data.cpuLostTime } override fun toString(): String = "server-writer" @@ -63,11 +76,18 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : .fields() .requiredLong("timestamp") .requiredString("server_id") - .requiredString("state") - .requiredLong("uptime") - .requiredLong("downtime") + .optionalString("host_id") .requiredInt("num_vcpus") .requiredLong("mem_capacity") + .requiredLong("uptime") + .requiredLong("downtime") + .optionalLong("boot_time") + .requiredLong("scheduling_latency") + .requiredDouble("cpu_limit") + .requiredLong("cpu_time_active") + .requiredLong("cpu_time_idle") + .requiredLong("cpu_time_steal") + .requiredLong("cpu_time_lost") .endRecord() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt index 29b48878..d9ca55cb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt @@ -35,7 +35,7 @@ public class ParquetServiceDataWriter(path: File, bufferSize: Int) : ParquetDataWriter(path, SCHEMA, bufferSize) { override fun convert(builder: GenericRecordBuilder, data: ServiceData) { - builder["timestamp"] = data.timestamp + builder["timestamp"] = data.timestamp.toEpochMilli() builder["hosts_up"] = data.hostsUp builder["hosts_down"] = data.hostsDown builder["servers_pending"] = data.serversPending diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 81405acf..727530e3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -50,7 +50,6 @@ import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File import java.time.Duration import java.util.* -import kotlin.math.roundToLong /** * An integration test suite for the Capelin experiments. @@ -102,7 +101,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -118,11 +117,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(220346412191, monitor.totalWork) { "Incorrect requested burst" } }, - { assertEquals(206667852689, monitor.totalGrantedWork) { "Incorrect granted burst" } }, - { assertEquals(1151612221, monitor.totalOvercommittedWork) { "Incorrect overcommitted burst" } }, - { assertEquals(0, monitor.totalInterferedWork) { "Incorrect interfered burst" } }, - { assertEquals(9.088769763540529E7, monitor.totalPowerDraw, 0.01) { "Incorrect power draw" } }, + { assertEquals(223856043, monitor.idleTime) { "Incorrect idle time" } }, + { assertEquals(66481557, monitor.activeTime) { "Incorrect active time" } }, + { assertEquals(360441, monitor.stealTime) { "Incorrect steal time" } }, + { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, + { assertEquals(5.418336360461193E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -151,7 +150,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -163,10 +162,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(39183965664, monitor.totalWork) { "Total work incorrect" } }, - { assertEquals(35649907631, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(1043642275, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, - { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } + { assertEquals(9597804, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(11140596, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(326138, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -202,7 +201,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -214,10 +213,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(39183965664, monitor.totalWork) { "Total work incorrect" } }, - { assertEquals(35649907631, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(1043642275, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, - { assertEquals(2960974524, monitor.totalInterferedWork) { "Total interfered work incorrect" } } + { assertEquals(9597804, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(11140596, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(326138, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(925305, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -247,7 +246,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.millis(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -259,10 +258,11 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(38385856700, monitor.totalWork) { "Total requested work incorrect" } }, - { assertEquals(34886670127, monitor.totalGrantedWork) { "Total granted work incorrect" } }, - { assertEquals(979997628, monitor.totalOvercommittedWork) { "Total overcommitted work incorrect" } }, - { assertEquals(0, monitor.totalInterferedWork) { "Total interfered work incorrect" } } + { assertEquals(9836315, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(10902085, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(306249, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, + { assertEquals(2540877457, monitor.uptime) { "Uptime incorrect" } } ) } @@ -286,18 +286,20 @@ class CapelinIntegrationTest { } class TestExperimentReporter : ComputeMonitor { - var totalWork = 0L - var totalGrantedWork = 0L - var totalOvercommittedWork = 0L - var totalInterferedWork = 0L - var totalPowerDraw = 0.0 + var idleTime = 0L + var activeTime = 0L + var stealTime = 0L + var lostTime = 0L + var energyUsage = 0.0 + var uptime = 0L override fun record(data: HostData) { - this.totalWork += data.totalWork.roundToLong() - totalGrantedWork += data.grantedWork.roundToLong() - totalOvercommittedWork += data.overcommittedWork.roundToLong() - totalInterferedWork += data.interferedWork.roundToLong() - totalPowerDraw += data.powerDraw + idleTime += data.cpuIdleTime + activeTime += data.cpuActiveTime + stealTime += data.cpuStealTime + lostTime += data.cpuLostTime + energyUsage += data.powerTotal + uptime += data.uptime } } } -- cgit v1.2.3 From e2537c59bef0645b948e92553cc5a42a8c0f7256 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 15 Sep 2021 21:34:00 +0200 Subject: feat(capelin): Use logical types for Parquet output columns This change updates the output schema for the experiment data to use logical types where possible. This adds additional context for the writer and the reader on how to process the column (efficiently). --- .../capelin/export/parquet/AvroUtils.kt | 44 ++++++++++++++++++++++ .../export/parquet/ParquetHostDataWriter.kt | 15 ++++---- .../export/parquet/ParquetServerDataWriter.kt | 18 +++++---- .../export/parquet/ParquetServiceDataWriter.kt | 2 +- 4 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt new file mode 100644 index 00000000..a4676f31 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("AvroUtils") +package org.opendc.experiments.capelin.export.parquet + +import org.apache.avro.LogicalTypes +import org.apache.avro.Schema + +/** + * Schema for UUID type. + */ +internal val UUID_SCHEMA = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)) + +/** + * Schema for timestamp type. + */ +internal val TIMESTAMP_SCHEMA = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)) + +/** + * Helper function to make a [Schema] field optional. + */ +internal fun Schema.optional(): Schema { + return Schema.createUnion(Schema.create(Schema.Type.NULL), this) +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt index 36207045..58388cb1 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt @@ -47,8 +47,6 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : builder["timestamp"] = data.timestamp.toEpochMilli() builder["host_id"] = data.host.id - builder["num_cpus"] = data.host.cpuCount - builder["mem_capacity"] = data.host.memCapacity builder["uptime"] = data.uptime builder["downtime"] = data.downtime @@ -57,12 +55,15 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : builder["boot_time"] = bootTime.toEpochMilli() } + builder["cpu_count"] = data.host.cpuCount builder["cpu_limit"] = data.cpuLimit builder["cpu_time_active"] = data.cpuActiveTime builder["cpu_time_idle"] = data.cpuIdleTime builder["cpu_time_steal"] = data.cpuStealTime builder["cpu_time_lost"] = data.cpuLostTime + builder["mem_limit"] = data.host.memCapacity + builder["power_total"] = data.powerTotal builder["guests_terminated"] = data.guestsTerminated @@ -78,18 +79,18 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) : .record("host") .namespace("org.opendc.telemetry.compute") .fields() - .requiredLong("timestamp") - .requiredString("host_id") - .requiredInt("num_cpus") - .requiredLong("mem_capacity") + .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() + .name("host_id").type(UUID_SCHEMA).noDefault() .requiredLong("uptime") .requiredLong("downtime") - .optionalLong("boot_time") + .name("boot_time").type(TIMESTAMP_SCHEMA.optional()).noDefault() + .requiredInt("cpu_count") .requiredDouble("cpu_limit") .requiredLong("cpu_time_active") .requiredLong("cpu_time_idle") .requiredLong("cpu_time_steal") .requiredLong("cpu_time_lost") + .requiredLong("mem_limit") .requiredDouble("power_total") .requiredInt("guests_terminated") .requiredInt("guests_running") diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt index c5a5e7c0..43b5f469 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt @@ -30,6 +30,7 @@ import org.apache.parquet.avro.AvroParquetWriter import org.apache.parquet.hadoop.ParquetWriter import org.opendc.telemetry.compute.table.ServerData import java.io.File +import java.util.* /** * A Parquet event writer for [ServerData]s. @@ -49,8 +50,6 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : builder["server_id"] = data.server.id builder["host_id"] = data.host?.id - builder["num_vcpus"] = data.server.cpuCount - builder["mem_capacity"] = data.server.memCapacity builder["uptime"] = data.uptime builder["downtime"] = data.downtime @@ -60,11 +59,14 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : } builder["scheduling_latency"] = data.schedulingLatency + builder["cpu_count"] = data.server.cpuCount builder["cpu_limit"] = data.cpuLimit builder["cpu_time_active"] = data.cpuActiveTime builder["cpu_time_idle"] = data.cpuIdleTime builder["cpu_time_steal"] = data.cpuStealTime builder["cpu_time_lost"] = data.cpuLostTime + + builder["mem_limit"] = data.server.memCapacity } override fun toString(): String = "server-writer" @@ -74,20 +76,20 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) : .record("server") .namespace("org.opendc.telemetry.compute") .fields() - .requiredLong("timestamp") - .requiredString("server_id") - .optionalString("host_id") - .requiredInt("num_vcpus") - .requiredLong("mem_capacity") + .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() + .name("server_id").type(UUID_SCHEMA).noDefault() + .name("host_id").type(UUID_SCHEMA.optional()).noDefault() .requiredLong("uptime") .requiredLong("downtime") - .optionalLong("boot_time") + .name("boot_time").type(TIMESTAMP_SCHEMA.optional()).noDefault() .requiredLong("scheduling_latency") + .requiredInt("cpu_count") .requiredDouble("cpu_limit") .requiredLong("cpu_time_active") .requiredLong("cpu_time_idle") .requiredLong("cpu_time_steal") .requiredLong("cpu_time_lost") + .requiredLong("mem_limit") .endRecord() } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt index d9ca55cb..2928f445 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt @@ -52,7 +52,7 @@ public class ParquetServiceDataWriter(path: File, bufferSize: Int) : .record("service") .namespace("org.opendc.telemetry.compute") .fields() - .requiredLong("timestamp") + .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() .requiredInt("hosts_up") .requiredInt("hosts_down") .requiredInt("servers_pending") -- cgit v1.2.3 From 859ce303f0b9110c7110b918e5957c2156fa8b26 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 17 Sep 2021 17:48:02 +0200 Subject: refactor(capelin): Extract common code out of Capelin experiments This change creates a new module for doing simulations with virtual machine workloads. We have found that a lot of code in the Capelin experiments code is being re-used by non-experiment modules. --- .../opendc-experiments-capelin/build.gradle.kts | 11 +- .../org/opendc/experiments/capelin/Portfolio.kt | 21 +- .../capelin/env/ClusterEnvironmentReader.kt | 1 + .../experiments/capelin/env/EnvironmentReader.kt | 1 + .../opendc/experiments/capelin/env/MachineDef.kt | 38 --- .../capelin/export/parquet/AvroUtils.kt | 44 ---- .../capelin/export/parquet/ParquetDataWriter.kt | 145 ------------ .../capelin/export/parquet/ParquetExportMonitor.kt | 67 ------ .../export/parquet/ParquetHostDataWriter.kt | 101 -------- .../export/parquet/ParquetServerDataWriter.kt | 95 -------- .../export/parquet/ParquetServiceDataWriter.kt | 65 ----- .../capelin/trace/ParquetTraceReader.kt | 10 +- .../capelin/trace/PerformanceInterferenceReader.kt | 60 ----- .../capelin/trace/RawParquetTraceReader.kt | 139 ----------- .../capelin/trace/StreamingParquetTraceReader.kt | 261 --------------------- .../experiments/capelin/trace/TraceConverter.kt | 260 -------------------- .../opendc/experiments/capelin/trace/TraceEntry.kt | 44 ---- .../experiments/capelin/trace/TraceReader.kt | 32 --- .../experiments/capelin/trace/VmPlacementReader.kt | 47 ---- .../experiments/capelin/trace/WorkloadSampler.kt | 3 +- .../capelin/trace/azure/AzureResourceStateTable.kt | 127 ---------- .../trace/azure/AzureResourceStateTableReader.kt | 149 ------------ .../capelin/trace/azure/AzureResourceTable.kt | 54 ----- .../trace/azure/AzureResourceTableReader.kt | 169 ------------- .../experiments/capelin/trace/azure/AzureTrace.kt | 46 ---- .../capelin/trace/azure/AzureTraceFormat.kt | 56 ----- .../capelin/trace/bp/BPResourceStateTable.kt | 53 ----- .../capelin/trace/bp/BPResourceStateTableReader.kt | 103 -------- .../capelin/trace/bp/BPResourceTable.kt | 53 ----- .../capelin/trace/bp/BPResourceTableReader.kt | 103 -------- .../opendc/experiments/capelin/trace/bp/BPTrace.kt | 49 ---- .../experiments/capelin/trace/bp/BPTraceFormat.kt | 47 ---- .../opendc/experiments/capelin/trace/bp/Schemas.kt | 55 ----- .../capelin/trace/sv/SvResourceStateTable.kt | 138 ----------- .../capelin/trace/sv/SvResourceStateTableReader.kt | 212 ----------------- .../opendc/experiments/capelin/trace/sv/SvTrace.kt | 45 ---- .../experiments/capelin/trace/sv/SvTraceFormat.kt | 47 ---- .../capelin/util/ComputeServiceSimulator.kt | 222 ------------------ .../experiments/capelin/util/FailureModel.kt | 38 --- .../experiments/capelin/util/FailureModels.kt | 97 -------- .../experiments/capelin/util/VmPlacementReader.kt | 47 ++++ .../src/main/resources/log4j2.xml | 2 +- .../experiments/capelin/CapelinIntegrationTest.kt | 17 +- .../trace/PerformanceInterferenceReaderTest.kt | 45 ---- 44 files changed, 87 insertions(+), 3332 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/VmPlacementReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 7dadd14d..010d18b0 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -31,6 +31,8 @@ plugins { dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) + api(projects.opendcCompute.opendcComputeWorkload) + implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) @@ -38,16 +40,13 @@ dependencies { implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(projects.opendcTelemetry.opendcTelemetryCompute) - implementation(libs.opentelemetry.semconv) - implementation(libs.kotlin.logging) implementation(libs.config) - implementation(libs.progressbar) - implementation(libs.clikt) + implementation(libs.kotlin.logging) + implementation(libs.jackson.databind) implementation(libs.jackson.module.kotlin) - implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) + implementation(libs.opentelemetry.semconv) - implementation(libs.parquet) testImplementation(libs.log4j.slf4j) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 6261ebbf..06db5569 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -24,16 +24,17 @@ package org.opendc.experiments.capelin import com.typesafe.config.ConfigFactory import mu.KotlinLogging +import org.opendc.compute.workload.ComputeWorkloadRunner +import org.opendc.compute.workload.export.parquet.ParquetExportMonitor +import org.opendc.compute.workload.grid5000 +import org.opendc.compute.workload.trace.RawParquetTraceReader +import org.opendc.compute.workload.util.PerformanceInterferenceReader import org.opendc.experiments.capelin.env.ClusterEnvironmentReader -import org.opendc.experiments.capelin.export.parquet.ParquetExportMonitor import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.trace.ParquetTraceReader -import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader -import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.experiments.capelin.util.ComputeServiceSimulator import org.opendc.experiments.capelin.util.createComputeScheduler import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf @@ -43,7 +44,6 @@ import org.opendc.telemetry.compute.ComputeMetricExporter import org.opendc.telemetry.compute.collectServiceMetrics import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File -import java.io.FileInputStream import java.time.Duration import java.util.* import java.util.concurrent.ConcurrentHashMap @@ -100,7 +100,12 @@ abstract class Portfolio(name: String) : Experiment(name) { */ override fun doRun(repeat: Int): Unit = runBlockingSimulation { val seeder = Random(repeat.toLong()) - val environment = ClusterEnvironmentReader(File(config.getString("env-path"), "${topology.name}.txt")) + val environment = ClusterEnvironmentReader( + File( + config.getString("env-path"), + "${topology.name}.txt" + ) + ) val workload = workload val workloadNames = if (workload is CompositeWorkload) { @@ -117,7 +122,7 @@ abstract class Portfolio(name: String) : Experiment(name) { val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) val performanceInterferenceModel = if (operationalPhenomena.hasInterference) PerformanceInterferenceReader() - .read(FileInputStream(config.getString("interference-model"))) + .read(File(config.getString("interference-model"))) .let { VmInterferenceModel(it, Random(seeder.nextLong())) } else null @@ -128,7 +133,7 @@ abstract class Portfolio(name: String) : Experiment(name) { grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong()), seeder.nextInt()) else null - val simulator = ComputeServiceSimulator( + val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt index babd8ada..8d9b24f4 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt @@ -22,6 +22,7 @@ package org.opendc.experiments.capelin.env +import org.opendc.compute.workload.env.MachineDef import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt index a968b043..8d61c530 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt @@ -22,6 +22,7 @@ package org.opendc.experiments.capelin.env +import org.opendc.compute.workload.env.MachineDef import java.io.Closeable /** diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt deleted file mode 100644 index b0c0318f..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/MachineDef.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.env - -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.power.PowerModel -import java.util.* - -/** - * A definition of a machine in a cluster. - */ -public data class MachineDef( - val uid: UUID, - val name: String, - val meta: Map, - val model: MachineModel, - val powerModel: PowerModel -) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt deleted file mode 100644 index a4676f31..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/AvroUtils.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("AvroUtils") -package org.opendc.experiments.capelin.export.parquet - -import org.apache.avro.LogicalTypes -import org.apache.avro.Schema - -/** - * Schema for UUID type. - */ -internal val UUID_SCHEMA = LogicalTypes.uuid().addToSchema(Schema.create(Schema.Type.STRING)) - -/** - * Schema for timestamp type. - */ -internal val TIMESTAMP_SCHEMA = LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)) - -/** - * Helper function to make a [Schema] field optional. - */ -internal fun Schema.optional(): Schema { - return Schema.createUnion(Schema.create(Schema.Type.NULL), this) -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt deleted file mode 100644 index e3d15c3b..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.export.parquet - -import mu.KotlinLogging -import org.apache.avro.Schema -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericRecordBuilder -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetFileWriter -import org.apache.parquet.hadoop.ParquetWriter -import org.apache.parquet.hadoop.metadata.CompressionCodecName -import org.opendc.trace.util.parquet.LocalOutputFile -import java.io.File -import java.util.concurrent.ArrayBlockingQueue -import java.util.concurrent.BlockingQueue -import kotlin.concurrent.thread - -/** - * A writer that writes data in Parquet format. - */ -abstract class ParquetDataWriter( - path: File, - private val schema: Schema, - bufferSize: Int = 4096 -) : AutoCloseable { - /** - * The logging instance to use. - */ - private val logger = KotlinLogging.logger {} - - /** - * The queue of commands to process. - */ - private val queue: BlockingQueue = ArrayBlockingQueue(bufferSize) - - /** - * An exception to be propagated to the actual writer. - */ - private var exception: Throwable? = null - - /** - * The thread that is responsible for writing the Parquet records. - */ - private val writerThread = thread(start = false, name = this.toString()) { - val writer = let { - val builder = AvroParquetWriter.builder(LocalOutputFile(path)) - .withSchema(schema) - .withCompressionCodec(CompressionCodecName.ZSTD) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - buildWriter(builder) - } - - val queue = queue - val buf = mutableListOf() - var shouldStop = false - - try { - while (!shouldStop) { - try { - process(writer, queue.take()) - } catch (e: InterruptedException) { - shouldStop = true - } - - if (queue.drainTo(buf) > 0) { - for (data in buf) { - process(writer, data) - } - buf.clear() - } - } - } catch (e: Throwable) { - logger.error(e) { "Failure in Parquet data writer" } - exception = e - } finally { - writer.close() - } - } - - /** - * Build the [ParquetWriter] used to write the Parquet files. - */ - protected open fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { - return builder.build() - } - - /** - * Convert the specified [data] into a Parquet record. - */ - protected abstract fun convert(builder: GenericRecordBuilder, data: T) - - /** - * Write the specified metrics to the database. - */ - fun write(data: T) { - val exception = exception - if (exception != null) { - throw IllegalStateException("Writer thread failed", exception) - } - - queue.put(data) - } - - /** - * Signal the writer to stop. - */ - override fun close() { - writerThread.interrupt() - writerThread.join() - } - - init { - writerThread.start() - } - - /** - * Process the specified [data] to be written to the Parquet file. - */ - private fun process(writer: ParquetWriter, data: T) { - val builder = GenericRecordBuilder(schema) - convert(builder, data) - writer.write(builder.build()) - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt deleted file mode 100644 index b057e932..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetExportMonitor.kt +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.export.parquet - -import org.opendc.telemetry.compute.ComputeMonitor -import org.opendc.telemetry.compute.table.HostData -import org.opendc.telemetry.compute.table.ServerData -import org.opendc.telemetry.compute.table.ServiceData -import java.io.File - -/** - * A [ComputeMonitor] that logs the events to a Parquet file. - */ -class ParquetExportMonitor(base: File, partition: String, bufferSize: Int) : ComputeMonitor, AutoCloseable { - private val serverWriter = ParquetServerDataWriter( - File(base, "server/$partition/data.parquet").also { it.parentFile.mkdirs() }, - bufferSize - ) - - private val hostWriter = ParquetHostDataWriter( - File(base, "host/$partition/data.parquet").also { it.parentFile.mkdirs() }, - bufferSize - ) - - private val serviceWriter = ParquetServiceDataWriter( - File(base, "service/$partition/data.parquet").also { it.parentFile.mkdirs() }, - bufferSize - ) - - override fun record(data: ServerData) { - serverWriter.write(data) - } - - override fun record(data: HostData) { - hostWriter.write(data) - } - - override fun record(data: ServiceData) { - serviceWriter.write(data) - } - - override fun close() { - hostWriter.close() - serviceWriter.close() - serverWriter.close() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt deleted file mode 100644 index 58388cb1..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.export.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericRecordBuilder -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetWriter -import org.opendc.telemetry.compute.table.HostData -import java.io.File - -/** - * A Parquet event writer for [HostData]s. - */ -public class ParquetHostDataWriter(path: File, bufferSize: Int) : - ParquetDataWriter(path, SCHEMA, bufferSize) { - - override fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { - return builder - .withDictionaryEncoding("host_id", true) - .build() - } - - override fun convert(builder: GenericRecordBuilder, data: HostData) { - builder["timestamp"] = data.timestamp.toEpochMilli() - - builder["host_id"] = data.host.id - - builder["uptime"] = data.uptime - builder["downtime"] = data.downtime - val bootTime = data.bootTime - if (bootTime != null) { - builder["boot_time"] = bootTime.toEpochMilli() - } - - builder["cpu_count"] = data.host.cpuCount - builder["cpu_limit"] = data.cpuLimit - builder["cpu_time_active"] = data.cpuActiveTime - builder["cpu_time_idle"] = data.cpuIdleTime - builder["cpu_time_steal"] = data.cpuStealTime - builder["cpu_time_lost"] = data.cpuLostTime - - builder["mem_limit"] = data.host.memCapacity - - builder["power_total"] = data.powerTotal - - builder["guests_terminated"] = data.guestsTerminated - builder["guests_running"] = data.guestsRunning - builder["guests_error"] = data.guestsError - builder["guests_invalid"] = data.guestsInvalid - } - - override fun toString(): String = "host-writer" - - companion object { - private val SCHEMA: Schema = SchemaBuilder - .record("host") - .namespace("org.opendc.telemetry.compute") - .fields() - .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() - .name("host_id").type(UUID_SCHEMA).noDefault() - .requiredLong("uptime") - .requiredLong("downtime") - .name("boot_time").type(TIMESTAMP_SCHEMA.optional()).noDefault() - .requiredInt("cpu_count") - .requiredDouble("cpu_limit") - .requiredLong("cpu_time_active") - .requiredLong("cpu_time_idle") - .requiredLong("cpu_time_steal") - .requiredLong("cpu_time_lost") - .requiredLong("mem_limit") - .requiredDouble("power_total") - .requiredInt("guests_terminated") - .requiredInt("guests_running") - .requiredInt("guests_error") - .requiredInt("guests_invalid") - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt deleted file mode 100644 index 43b5f469..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.export.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericRecordBuilder -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetWriter -import org.opendc.telemetry.compute.table.ServerData -import java.io.File -import java.util.* - -/** - * A Parquet event writer for [ServerData]s. - */ -public class ParquetServerDataWriter(path: File, bufferSize: Int) : - ParquetDataWriter(path, SCHEMA, bufferSize) { - - override fun buildWriter(builder: AvroParquetWriter.Builder): ParquetWriter { - return builder - .withDictionaryEncoding("server_id", true) - .withDictionaryEncoding("host_id", true) - .build() - } - - override fun convert(builder: GenericRecordBuilder, data: ServerData) { - builder["timestamp"] = data.timestamp.toEpochMilli() - - builder["server_id"] = data.server.id - builder["host_id"] = data.host?.id - - builder["uptime"] = data.uptime - builder["downtime"] = data.downtime - val bootTime = data.bootTime - if (bootTime != null) { - builder["boot_time"] = bootTime.toEpochMilli() - } - builder["scheduling_latency"] = data.schedulingLatency - - builder["cpu_count"] = data.server.cpuCount - builder["cpu_limit"] = data.cpuLimit - builder["cpu_time_active"] = data.cpuActiveTime - builder["cpu_time_idle"] = data.cpuIdleTime - builder["cpu_time_steal"] = data.cpuStealTime - builder["cpu_time_lost"] = data.cpuLostTime - - builder["mem_limit"] = data.server.memCapacity - } - - override fun toString(): String = "server-writer" - - companion object { - private val SCHEMA: Schema = SchemaBuilder - .record("server") - .namespace("org.opendc.telemetry.compute") - .fields() - .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() - .name("server_id").type(UUID_SCHEMA).noDefault() - .name("host_id").type(UUID_SCHEMA.optional()).noDefault() - .requiredLong("uptime") - .requiredLong("downtime") - .name("boot_time").type(TIMESTAMP_SCHEMA.optional()).noDefault() - .requiredLong("scheduling_latency") - .requiredInt("cpu_count") - .requiredDouble("cpu_limit") - .requiredLong("cpu_time_active") - .requiredLong("cpu_time_idle") - .requiredLong("cpu_time_steal") - .requiredLong("cpu_time_lost") - .requiredLong("mem_limit") - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt deleted file mode 100644 index 2928f445..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.export.parquet - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericRecordBuilder -import org.opendc.telemetry.compute.table.ServiceData -import java.io.File - -/** - * A Parquet event writer for [ServiceData]s. - */ -public class ParquetServiceDataWriter(path: File, bufferSize: Int) : - ParquetDataWriter(path, SCHEMA, bufferSize) { - - override fun convert(builder: GenericRecordBuilder, data: ServiceData) { - builder["timestamp"] = data.timestamp.toEpochMilli() - builder["hosts_up"] = data.hostsUp - builder["hosts_down"] = data.hostsDown - builder["servers_pending"] = data.serversPending - builder["servers_active"] = data.serversActive - builder["attempts_success"] = data.attemptsSuccess - builder["attempts_failure"] = data.attemptsFailure - builder["attempts_error"] = data.attemptsError - } - - override fun toString(): String = "service-writer" - - companion object { - private val SCHEMA: Schema = SchemaBuilder - .record("service") - .namespace("org.opendc.telemetry.compute") - .fields() - .name("timestamp").type(TIMESTAMP_SCHEMA).noDefault() - .requiredInt("hosts_up") - .requiredInt("hosts_down") - .requiredInt("servers_pending") - .requiredInt("servers_active") - .requiredInt("attempts_success") - .requiredInt("attempts_failure") - .requiredInt("attempts_error") - .endRecord() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt index 0bf4ada6..498636ba 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,9 @@ package org.opendc.experiments.capelin.trace +import org.opendc.compute.workload.trace.RawParquetTraceReader +import org.opendc.compute.workload.trace.TraceEntry +import org.opendc.compute.workload.trace.TraceReader import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.Workload import org.opendc.simulator.compute.workload.SimWorkload @@ -51,7 +54,10 @@ public class ParquetTraceReader( this.zip(listOf(workload)) } } - .flatMap { sampleWorkload(it.first, workload, it.second, seed).sortedBy(TraceEntry::start) } + .flatMap { + sampleWorkload(it.first, workload, it.second, seed) + .sortedBy(TraceEntry::start) + } .iterator() override fun hasNext(): Boolean = iterator.hasNext() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt deleted file mode 100644 index 9549af42..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReader.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import com.fasterxml.jackson.annotation.JsonProperty -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue -import org.opendc.simulator.compute.kernel.interference.VmInterferenceGroup -import java.io.InputStream - -/** - * A parser for the JSON performance interference setup files used for the TPDS article on Capelin. - */ -class PerformanceInterferenceReader { - /** - * The [ObjectMapper] to use. - */ - private val mapper = jacksonObjectMapper() - - init { - mapper.addMixIn(VmInterferenceGroup::class.java, GroupMixin::class.java) - } - - /** - * Read the performance interface model from the input. - */ - fun read(input: InputStream): List { - return input.use { mapper.readValue(input) } - } - - private data class GroupMixin( - @JsonProperty("minServerLoad") - val targetLoad: Double, - @JsonProperty("performanceScore") - val score: Double, - @JsonProperty("vms") - val members: Set, - ) -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt deleted file mode 100644 index ca937328..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import org.opendc.experiments.capelin.trace.bp.BPTraceFormat -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.trace.* -import java.io.File -import java.util.UUID - -/** - * A [TraceReader] for the internal VM workload trace format. - * - * @param path The directory of the traces. - */ -class RawParquetTraceReader(private val path: File) { - /** - * The [Trace] that represents this trace. - */ - private val trace = BPTraceFormat().open(path.toURI().toURL()) - - /** - * Read the fragments into memory. - */ - private fun parseFragments(): Map> { - val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() - - val fragments = mutableMapOf>() - - return try { - while (reader.nextRow()) { - val id = reader.get(RESOURCE_STATE_ID) - val time = reader.get(RESOURCE_STATE_TIMESTAMP) - val duration = reader.get(RESOURCE_STATE_DURATION) - val cores = reader.getInt(RESOURCE_STATE_NCPUS) - val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) - - val fragment = SimTraceWorkload.Fragment( - time.toEpochMilli(), - duration.toMillis(), - cpuUsage, - cores - ) - - fragments.getOrPut(id) { mutableListOf() }.add(fragment) - } - - fragments - } finally { - reader.close() - } - } - - /** - * Read the metadata into a workload. - */ - private fun parseMeta(fragments: Map>): List> { - val reader = checkNotNull(trace.getTable(TABLE_RESOURCES)).newReader() - - var counter = 0 - val entries = mutableListOf>() - - return try { - while (reader.nextRow()) { - - val id = reader.get(RESOURCE_ID) - if (!fragments.containsKey(id)) { - continue - } - - val submissionTime = reader.get(RESOURCE_START_TIME) - val endTime = reader.get(RESOURCE_STOP_TIME) - val maxCores = reader.getInt(RESOURCE_NCPUS) - val requiredMemory = reader.getDouble(RESOURCE_MEM_CAPACITY) / 1000.0 // Convert from KB to MB - val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) - - val vmFragments = fragments.getValue(id).asSequence() - val totalLoad = vmFragments.sumOf { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs - val workload = SimTraceWorkload(vmFragments) - entries.add( - TraceEntry( - uid, id, submissionTime.toEpochMilli(), workload, - mapOf( - "submit-time" to submissionTime.toEpochMilli(), - "end-time" to endTime.toEpochMilli(), - "total-load" to totalLoad, - "cores" to maxCores, - "required-memory" to requiredMemory.toLong(), - "workload" to workload - ) - ) - ) - } - - entries - } catch (e: Exception) { - e.printStackTrace() - throw e - } finally { - reader.close() - } - } - - /** - * The entries in the trace. - */ - private val entries: List> - - init { - val fragments = parseFragments() - entries = parseMeta(fragments) - } - - /** - * Read the entries in the trace. - */ - fun read(): List> = entries -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt deleted file mode 100644 index ed82217d..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import mu.KotlinLogging -import org.apache.avro.generic.GenericData -import org.apache.parquet.avro.AvroParquetReader -import org.apache.parquet.filter2.compat.FilterCompat -import org.apache.parquet.filter2.predicate.FilterApi -import org.apache.parquet.filter2.predicate.Statistics -import org.apache.parquet.filter2.predicate.UserDefinedPredicate -import org.apache.parquet.io.api.Binary -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.trace.util.parquet.LocalInputFile -import java.io.File -import java.io.Serializable -import java.util.SortedSet -import java.util.TreeSet -import java.util.UUID -import java.util.concurrent.ArrayBlockingQueue -import kotlin.concurrent.thread - -/** - * A [TraceReader] for the internal VM workload trace format that streams workloads on the fly. - * - * @param traceFile The directory of the traces. - * @param selectedVms The list of VMs to read from the trace. - */ -class StreamingParquetTraceReader(traceFile: File, selectedVms: List = emptyList()) : TraceReader { - private val logger = KotlinLogging.logger {} - - /** - * The internal iterator to use for this reader. - */ - private val iterator: Iterator> - - /** - * The intermediate buffer to store the read records in. - */ - private val queue = ArrayBlockingQueue>(1024) - - /** - * An optional filter for filtering the selected VMs - */ - private val filter = - if (selectedVms.isEmpty()) - null - else - FilterCompat.get( - FilterApi.userDefined( - FilterApi.binaryColumn("id"), - SelectedVmFilter( - TreeSet(selectedVms) - ) - ) - ) - - /** - * A poisonous fragment. - */ - private val poison = Pair("\u0000", SimTraceWorkload.Fragment(0L, 0, 0.0, 0)) - - /** - * The thread to read the records in. - */ - private val readerThread = thread(start = true, name = "sc20-reader") { - val reader = AvroParquetReader - .builder(LocalInputFile(File(traceFile, "trace.parquet"))) - .disableCompatibility() - .withFilter(filter) - .build() - - try { - while (true) { - val record = reader.read() - - if (record == null) { - queue.put(poison) - break - } - - val id = record["id"].toString() - val time = record["time"] as Long - val duration = record["duration"] as Long - val cores = record["cores"] as Int - val cpuUsage = record["cpuUsage"] as Double - - val fragment = SimTraceWorkload.Fragment( - time, - duration, - cpuUsage, - cores - ) - - queue.put(id to fragment) - } - } catch (e: InterruptedException) { - // Do not rethrow this - } finally { - reader.close() - } - } - - /** - * Fill the buffers with the VMs - */ - private fun pull(buffers: Map>>) { - if (!hasNext) { - return - } - - val fragments = mutableListOf>() - queue.drainTo(fragments) - - for ((id, fragment) in fragments) { - if (id == poison.first) { - hasNext = false - return - } - buffers[id]?.forEach { it.add(fragment) } - } - } - - /** - * A flag to indicate whether the reader has more entries. - */ - private var hasNext: Boolean = true - - /** - * Initialize the reader. - */ - init { - val takenIds = mutableSetOf() - val entries = mutableMapOf() - val buffers = mutableMapOf>>() - - val metaReader = AvroParquetReader - .builder(LocalInputFile(File(traceFile, "meta.parquet"))) - .disableCompatibility() - .withFilter(filter) - .build() - - while (true) { - val record = metaReader.read() ?: break - val id = record["id"].toString() - entries[id] = record - } - - metaReader.close() - - val selection = selectedVms.ifEmpty { entries.keys } - - // Create the entry iterator - iterator = selection.asSequence() - .mapNotNull { entries[it] } - .mapIndexed { index, record -> - val id = record["id"].toString() - val submissionTime = record["submissionTime"] as Long - val endTime = record["endTime"] as Long - val maxCores = record["maxCores"] as Int - val requiredMemory = record["requiredMemory"] as Long - val uid = UUID.nameUUIDFromBytes("$id-$index".toByteArray()) - - assert(uid !in takenIds) - takenIds += uid - - logger.info { "Processing VM $id" } - - val internalBuffer = mutableListOf() - val externalBuffer = mutableListOf() - buffers.getOrPut(id) { mutableListOf() }.add(externalBuffer) - val fragments = sequence { - var time = submissionTime - repeat@ while (true) { - if (externalBuffer.isEmpty()) { - if (hasNext) { - pull(buffers) - continue - } else { - break - } - } - - internalBuffer.addAll(externalBuffer) - externalBuffer.clear() - - for (fragment in internalBuffer) { - yield(fragment) - - time += fragment.duration - if (time >= endTime) { - break@repeat - } - } - - internalBuffer.clear() - } - - buffers.remove(id) - } - val workload = SimTraceWorkload(fragments) - val meta = mapOf( - "cores" to maxCores, - "required-memory" to requiredMemory, - "workload" to workload - ) - - TraceEntry(uid, id, submissionTime, workload, meta) - } - .sortedBy { it.start } - .toList() - .iterator() - } - - override fun hasNext(): Boolean = iterator.hasNext() - - override fun next(): TraceEntry = iterator.next() - - override fun close() { - readerThread.interrupt() - } - - private class SelectedVmFilter(val selectedVms: SortedSet) : UserDefinedPredicate(), Serializable { - override fun keep(value: Binary?): Boolean = value != null && selectedVms.contains(value.toStringUsingUTF8()) - - override fun canDrop(statistics: Statistics): Boolean { - val min = statistics.min - val max = statistics.max - - return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isEmpty() - } - - override fun inverseCanDrop(statistics: Statistics): Boolean { - val min = statistics.min - val max = statistics.max - - return selectedVms.subSet(min.toStringUsingUTF8(), max.toStringUsingUTF8() + "\u0000").isNotEmpty() - } - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt deleted file mode 100644 index 1f3878eb..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceConverter.kt +++ /dev/null @@ -1,260 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.arguments.argument -import com.github.ajalt.clikt.parameters.groups.OptionGroup -import com.github.ajalt.clikt.parameters.groups.cooccurring -import com.github.ajalt.clikt.parameters.options.* -import com.github.ajalt.clikt.parameters.types.* -import mu.KotlinLogging -import org.apache.avro.generic.GenericData -import org.apache.avro.generic.GenericRecordBuilder -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetWriter -import org.apache.parquet.hadoop.metadata.CompressionCodecName -import org.opendc.experiments.capelin.trace.azure.AzureTraceFormat -import org.opendc.experiments.capelin.trace.bp.BP_RESOURCES_SCHEMA -import org.opendc.experiments.capelin.trace.bp.BP_RESOURCE_STATES_SCHEMA -import org.opendc.experiments.capelin.trace.sv.SvTraceFormat -import org.opendc.trace.* -import org.opendc.trace.bitbrains.BitbrainsTraceFormat -import org.opendc.trace.util.parquet.LocalOutputFile -import java.io.File -import java.util.* -import kotlin.math.max -import kotlin.math.min -import kotlin.math.roundToLong - -/** - * A script to convert a trace in text format into a Parquet trace. - */ -fun main(args: Array): Unit = TraceConverterCli().main(args) - -/** - * Represents the command for converting traces - */ -class TraceConverterCli : CliktCommand(name = "trace-converter") { - /** - * The logger instance for the converter. - */ - private val logger = KotlinLogging.logger {} - - /** - * The directory where the trace should be stored. - */ - private val output by option("-O", "--output", help = "path to store the trace") - .file(canBeFile = false, mustExist = false) - .defaultLazy { File("output") } - - /** - * The directory where the input trace is located. - */ - private val input by argument("input", help = "path to the input trace") - .file(canBeFile = false) - - /** - * The input format of the trace. - */ - private val format by option("-f", "--format", help = "input format of trace") - .choice( - "solvinity" to SvTraceFormat(), - "bitbrains" to BitbrainsTraceFormat(), - "azure" to AzureTraceFormat() - ) - .required() - - /** - * The sampling options. - */ - private val samplingOptions by SamplingOptions().cooccurring() - - override fun run() { - val metaParquet = File(output, "meta.parquet") - val traceParquet = File(output, "trace.parquet") - - if (metaParquet.exists()) { - metaParquet.delete() - } - if (traceParquet.exists()) { - traceParquet.delete() - } - - val trace = format.open(input.toURI().toURL()) - - logger.info { "Building resources table" } - - val metaWriter = AvroParquetWriter.builder(LocalOutputFile(metaParquet)) - .withSchema(BP_RESOURCES_SCHEMA) - .withCompressionCodec(CompressionCodecName.ZSTD) - .enablePageWriteChecksum() - .build() - - val selectedVms = metaWriter.use { convertResources(trace, it) } - - logger.info { "Wrote ${selectedVms.size} rows" } - logger.info { "Building resource states table" } - - val writer = AvroParquetWriter.builder(LocalOutputFile(traceParquet)) - .withSchema(BP_RESOURCE_STATES_SCHEMA) - .withCompressionCodec(CompressionCodecName.ZSTD) - .enableDictionaryEncoding() - .enablePageWriteChecksum() - .withBloomFilterEnabled("id", true) - .withBloomFilterNDV("id", selectedVms.size.toLong()) - .build() - - val statesCount = writer.use { convertResourceStates(trace, it, selectedVms) } - logger.info { "Wrote $statesCount rows" } - } - - /** - * Convert the resources table for the trace. - */ - private fun convertResources(trace: Trace, writer: ParquetWriter): Set { - val random = samplingOptions?.let { Random(it.seed) } - val samplingFraction = samplingOptions?.fraction ?: 1.0 - val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() - - var hasNextRow = reader.nextRow() - val selectedVms = mutableSetOf() - - while (hasNextRow) { - var id: String - var numCpus = Int.MIN_VALUE - var memCapacity = Double.MIN_VALUE - var memUsage = Double.MIN_VALUE - var startTime = Long.MAX_VALUE - var stopTime = Long.MIN_VALUE - - do { - id = reader.get(RESOURCE_STATE_ID) - - val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli() - startTime = min(startTime, timestamp) - stopTime = max(stopTime, timestamp) - - numCpus = max(numCpus, reader.getInt(RESOURCE_STATE_NCPUS)) - - memCapacity = max(memCapacity, reader.getDouble(RESOURCE_STATE_MEM_CAPACITY)) - if (reader.hasColumn(RESOURCE_STATE_MEM_USAGE)) { - memUsage = max(memUsage, reader.getDouble(RESOURCE_STATE_MEM_USAGE)) - } - - hasNextRow = reader.nextRow() - } while (hasNextRow && id == reader.get(RESOURCE_STATE_ID)) - - // Sample only a fraction of the VMs - if (random != null && random.nextDouble() > samplingFraction) { - continue - } - - val builder = GenericRecordBuilder(BP_RESOURCES_SCHEMA) - - builder["id"] = id - builder["submissionTime"] = startTime - builder["endTime"] = stopTime - builder["maxCores"] = numCpus - builder["requiredMemory"] = max(memCapacity, memUsage).roundToLong() - - logger.info { "Selecting VM $id" } - - writer.write(builder.build()) - selectedVms.add(id) - } - - return selectedVms - } - - /** - * Convert the resource states table for the trace. - */ - private fun convertResourceStates(trace: Trace, writer: ParquetWriter, selectedVms: Set): Int { - val reader = checkNotNull(trace.getTable(TABLE_RESOURCE_STATES)).newReader() - - var hasNextRow = reader.nextRow() - var count = 0 - - while (hasNextRow) { - var lastTimestamp = Long.MIN_VALUE - - do { - val id = reader.get(RESOURCE_STATE_ID) - - if (id !in selectedVms) { - hasNextRow = reader.nextRow() - continue - } - - val builder = GenericRecordBuilder(BP_RESOURCE_STATES_SCHEMA) - builder["id"] = id - - val timestamp = reader.get(RESOURCE_STATE_TIMESTAMP).toEpochMilli() - if (lastTimestamp < 0) { - lastTimestamp = timestamp - 5 * 60 * 1000L - } - - val duration = timestamp - lastTimestamp - val cores = reader.getInt(RESOURCE_STATE_NCPUS) - val cpuUsage = reader.getDouble(RESOURCE_STATE_CPU_USAGE) - val flops = (cpuUsage * duration / 1000.0).roundToLong() - - builder["time"] = timestamp - builder["duration"] = duration - builder["cores"] = cores - builder["cpuUsage"] = cpuUsage - builder["flops"] = flops - - writer.write(builder.build()) - - lastTimestamp = timestamp - hasNextRow = reader.nextRow() - } while (hasNextRow && id == reader.get(RESOURCE_STATE_ID)) - - count++ - } - - return count - } - - /** - * Options for sampling the workload trace. - */ - private class SamplingOptions : OptionGroup() { - /** - * The fraction of VMs to sample - */ - val fraction by option("--sampling-fraction", help = "fraction of the workload to sample") - .double() - .restrictTo(0.0001, 1.0) - .required() - - /** - * The seed for sampling the trace. - */ - val seed by option("--sampling-seed", help = "seed for sampling the workload") - .long() - .default(0) - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt deleted file mode 100644 index 303a6a8c..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt +++ /dev/null @@ -1,44 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2019 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import java.util.UUID - -/** - * An entry in a workload trace. - * - * @param uid The unique identifier of the entry. - * @param name The name of the entry. - * @param start The start time of the workload. - * @param workload The workload of the entry. - * @param meta The meta-data associated with the workload. - */ -public data class TraceEntry( - val uid: UUID, - val name: String, - val start: Long, - val workload: T, - val meta: Map -) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt deleted file mode 100644 index 08304edc..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -/** - * An interface for reading workloads into memory. - * - * This interface must guarantee that the entries are delivered in order of submission time. - * - * @param T The shape of the workloads supported by this reader. - */ -public interface TraceReader : Iterator>, AutoCloseable diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt deleted file mode 100644 index b55bd577..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/VmPlacementReader.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue -import java.io.InputStream - -/** - * A parser for the JSON VM placement data files used for the TPDS article on Capelin. - */ -class VmPlacementReader { - /** - * The [ObjectMapper] to parse the placement. - */ - private val mapper = jacksonObjectMapper() - - /** - * Read the VM placements from the input. - */ - fun read(input: InputStream): Map { - return mapper.readValue>(input) - .mapKeys { "vm__workload__${it.key}.txt" } - .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt index cb32ce88..b42951df 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,7 @@ package org.opendc.experiments.capelin.trace import mu.KotlinLogging +import org.opendc.compute.workload.trace.TraceEntry import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.SamplingStrategy import org.opendc.experiments.capelin.model.Workload diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt deleted file mode 100644 index f98f4b2c..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTable.kt +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.dataformat.csv.CsvFactory -import org.opendc.trace.* -import java.nio.file.Files -import java.nio.file.Path -import java.util.stream.Collectors -import kotlin.io.path.extension -import kotlin.io.path.nameWithoutExtension - -/** - * The resource state [Table] for the Azure v1 VM traces. - */ -internal class AzureResourceStateTable(private val factory: CsvFactory, path: Path) : Table { - /** - * The partitions that belong to the table. - */ - private val partitions = Files.walk(path, 1) - .filter { !Files.isDirectory(it) && it.extension == "csv" } - .collect(Collectors.toMap({ it.nameWithoutExtension }, { it })) - .toSortedMap() - - override val name: String = TABLE_RESOURCE_STATES - - override val isSynthetic: Boolean = false - - override val columns: List> = listOf( - RESOURCE_STATE_ID, - RESOURCE_STATE_TIMESTAMP, - RESOURCE_STATE_CPU_USAGE_PCT - ) - - override fun newReader(): TableReader { - val it = partitions.iterator() - - return object : TableReader { - var delegate: TableReader? = nextDelegate() - - override fun nextRow(): Boolean { - var delegate = delegate - - while (delegate != null) { - if (delegate.nextRow()) { - break - } - - delegate.close() - delegate = nextDelegate() - } - - this.delegate = delegate - return delegate != null - } - - override fun hasColumn(column: TableColumn<*>): Boolean = delegate?.hasColumn(column) ?: false - - override fun get(column: TableColumn): T { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.get(column) - } - - override fun getBoolean(column: TableColumn): Boolean { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getBoolean(column) - } - - override fun getInt(column: TableColumn): Int { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getInt(column) - } - - override fun getLong(column: TableColumn): Long { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getLong(column) - } - - override fun getDouble(column: TableColumn): Double { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getDouble(column) - } - - override fun close() { - delegate?.close() - } - - private fun nextDelegate(): TableReader? { - return if (it.hasNext()) { - val (_, path) = it.next() - return AzureResourceStateTableReader(factory.createParser(path.toFile())) - } else { - null - } - } - - override fun toString(): String = "AzureCompositeTableReader" - } - } - - override fun newReader(partition: String): TableReader { - val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" } - return AzureResourceStateTableReader(factory.createParser(path.toFile())) - } - - override fun toString(): String = "AzureResourceStateTable" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt deleted file mode 100644 index f80c0e82..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceStateTableReader.kt +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.core.JsonToken -import com.fasterxml.jackson.dataformat.csv.CsvParser -import com.fasterxml.jackson.dataformat.csv.CsvSchema -import org.opendc.trace.* -import java.time.Instant - -/** - * A [TableReader] for the Azure v1 VM resource state table. - */ -internal class AzureResourceStateTableReader(private val parser: CsvParser) : TableReader { - init { - parser.schema = schema - } - - override fun nextRow(): Boolean { - reset() - - if (!nextStart()) { - return false - } - - while (true) { - val token = parser.nextValue() - - if (token == null || token == JsonToken.END_OBJECT) { - break - } - - when (parser.currentName) { - "timestamp" -> timestamp = Instant.ofEpochSecond(parser.longValue) - "vm id" -> id = parser.text - "avg cpu" -> cpuUsagePct = parser.doubleValue - } - } - - return true - } - - override fun hasColumn(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_CPU_USAGE_PCT -> true - else -> false - } - } - - override fun get(column: TableColumn): T { - val res: Any? = when (column) { - RESOURCE_STATE_ID -> id - RESOURCE_STATE_TIMESTAMP -> timestamp - RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsagePct - else -> throw IllegalArgumentException("Invalid column") - } - - @Suppress("UNCHECKED_CAST") - return res as T - } - - override fun getBoolean(column: TableColumn): Boolean { - throw IllegalArgumentException("Invalid column") - } - - override fun getInt(column: TableColumn): Int { - throw IllegalArgumentException("Invalid column") - } - - override fun getLong(column: TableColumn): Long { - throw IllegalArgumentException("Invalid column") - } - - override fun getDouble(column: TableColumn): Double { - return when (column) { - RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsagePct - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun close() { - parser.close() - } - - /** - * Advance the parser until the next object start. - */ - private fun nextStart(): Boolean { - var token = parser.nextValue() - - while (token != null && token != JsonToken.START_OBJECT) { - token = parser.nextValue() - } - - return token != null - } - - /** - * State fields of the reader. - */ - private var id: String? = null - private var timestamp: Instant? = null - private var cpuUsagePct = Double.NaN - - /** - * Reset the state. - */ - private fun reset() { - id = null - timestamp = null - cpuUsagePct = Double.NaN - } - - companion object { - /** - * The [CsvSchema] that is used to parse the trace. - */ - private val schema = CsvSchema.builder() - .addColumn("timestamp", CsvSchema.ColumnType.NUMBER) - .addColumn("vm id", CsvSchema.ColumnType.STRING) - .addColumn("CPU min cpu", CsvSchema.ColumnType.NUMBER) - .addColumn("CPU max cpu", CsvSchema.ColumnType.NUMBER) - .addColumn("CPU avg cpu", CsvSchema.ColumnType.NUMBER) - .setAllowComments(true) - .build() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt deleted file mode 100644 index c9d4f7eb..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTable.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.dataformat.csv.CsvFactory -import org.opendc.trace.* -import java.nio.file.Path - -/** - * The resource [Table] for the Azure v1 VM traces. - */ -internal class AzureResourceTable(private val factory: CsvFactory, private val path: Path) : Table { - override val name: String = TABLE_RESOURCES - - override val isSynthetic: Boolean = false - - override val columns: List> = listOf( - RESOURCE_ID, - RESOURCE_START_TIME, - RESOURCE_STOP_TIME, - RESOURCE_NCPUS, - RESOURCE_MEM_CAPACITY - ) - - override fun newReader(): TableReader { - return AzureResourceTableReader(factory.createParser(path.resolve("vmtable/vmtable.csv").toFile())) - } - - override fun newReader(partition: String): TableReader { - throw IllegalArgumentException("No partition $partition") - } - - override fun toString(): String = "AzureResourceTable" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt deleted file mode 100644 index b712b854..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureResourceTableReader.kt +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.core.JsonToken -import com.fasterxml.jackson.dataformat.csv.CsvParser -import com.fasterxml.jackson.dataformat.csv.CsvSchema -import org.apache.parquet.example.Paper.schema -import org.opendc.trace.* -import java.time.Instant - -/** - * A [TableReader] for the Azure v1 VM resources table. - */ -internal class AzureResourceTableReader(private val parser: CsvParser) : TableReader { - init { - parser.schema = schema - } - - override fun nextRow(): Boolean { - reset() - - if (!nextStart()) { - return false - } - - while (true) { - val token = parser.nextValue() - - if (token == null || token == JsonToken.END_OBJECT) { - break - } - - when (parser.currentName) { - "vm id" -> id = parser.text - "vm created" -> startTime = Instant.ofEpochSecond(parser.longValue) - "vm deleted" -> stopTime = Instant.ofEpochSecond(parser.longValue) - "vm virtual core count" -> cpuCores = parser.intValue - "vm memory" -> memCapacity = parser.doubleValue * 1e6 // GB to KB - } - } - - return true - } - - override fun hasColumn(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_ID -> true - RESOURCE_START_TIME -> true - RESOURCE_STOP_TIME -> true - RESOURCE_NCPUS -> true - RESOURCE_MEM_CAPACITY -> true - else -> false - } - } - - override fun get(column: TableColumn): T { - val res: Any? = when (column) { - RESOURCE_ID -> id - RESOURCE_START_TIME -> startTime - RESOURCE_STOP_TIME -> stopTime - RESOURCE_NCPUS -> getInt(RESOURCE_STATE_NCPUS) - RESOURCE_MEM_CAPACITY -> getDouble(RESOURCE_STATE_MEM_CAPACITY) - else -> throw IllegalArgumentException("Invalid column") - } - - @Suppress("UNCHECKED_CAST") - return res as T - } - - override fun getBoolean(column: TableColumn): Boolean { - throw IllegalArgumentException("Invalid column") - } - - override fun getInt(column: TableColumn): Int { - return when (column) { - RESOURCE_NCPUS -> cpuCores - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun getLong(column: TableColumn): Long { - throw IllegalArgumentException("Invalid column") - } - - override fun getDouble(column: TableColumn): Double { - return when (column) { - RESOURCE_MEM_CAPACITY -> memCapacity - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun close() { - parser.close() - } - - /** - * Advance the parser until the next object start. - */ - private fun nextStart(): Boolean { - var token = parser.nextValue() - - while (token != null && token != JsonToken.START_OBJECT) { - token = parser.nextValue() - } - - return token != null - } - - /** - * State fields of the reader. - */ - private var id: String? = null - private var startTime: Instant? = null - private var stopTime: Instant? = null - private var cpuCores = -1 - private var memCapacity = Double.NaN - - /** - * Reset the state. - */ - fun reset() { - id = null - startTime = null - stopTime = null - cpuCores = -1 - memCapacity = Double.NaN - } - - companion object { - /** - * The [CsvSchema] that is used to parse the trace. - */ - private val schema = CsvSchema.builder() - .addColumn("vm id", CsvSchema.ColumnType.NUMBER) - .addColumn("subscription id", CsvSchema.ColumnType.STRING) - .addColumn("deployment id", CsvSchema.ColumnType.NUMBER) - .addColumn("timestamp vm created", CsvSchema.ColumnType.NUMBER) - .addColumn("timestamp vm deleted", CsvSchema.ColumnType.NUMBER) - .addColumn("max cpu", CsvSchema.ColumnType.NUMBER) - .addColumn("avg cpu", CsvSchema.ColumnType.NUMBER) - .addColumn("p95 cpu", CsvSchema.ColumnType.NUMBER) - .addColumn("vm category", CsvSchema.ColumnType.NUMBER) - .addColumn("vm virtual core count", CsvSchema.ColumnType.NUMBER) - .addColumn("vm memory", CsvSchema.ColumnType.NUMBER) - .setAllowComments(true) - .build() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt deleted file mode 100644 index 24c60bab..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTrace.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.dataformat.csv.CsvFactory -import org.opendc.trace.* -import java.nio.file.Path - -/** - * [Trace] implementation for the Azure v1 VM traces. - */ -class AzureTrace internal constructor(private val factory: CsvFactory, private val path: Path) : Trace { - override val tables: List = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES) - - override fun containsTable(name: String): Boolean = name in tables - - override fun getTable(name: String): Table? { - return when (name) { - TABLE_RESOURCES -> AzureResourceTable(factory, path) - TABLE_RESOURCE_STATES -> AzureResourceStateTable(factory, path) - else -> null - } - } - - override fun toString(): String = "AzureTrace[$path]" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt deleted file mode 100644 index 744e43a0..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/azure/AzureTraceFormat.kt +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.azure - -import com.fasterxml.jackson.dataformat.csv.CsvFactory -import com.fasterxml.jackson.dataformat.csv.CsvParser -import org.opendc.trace.spi.TraceFormat -import java.net.URL -import java.nio.file.Paths -import kotlin.io.path.exists - -/** - * A format implementation for the Azure v1 format. - */ -class AzureTraceFormat : TraceFormat { - /** - * The name of this trace format. - */ - override val name: String = "azure-v1" - - /** - * The [CsvFactory] used to create the parser. - */ - private val factory = CsvFactory() - .enable(CsvParser.Feature.ALLOW_COMMENTS) - .enable(CsvParser.Feature.TRIM_SPACES) - - /** - * Open the trace file. - */ - override fun open(url: URL): AzureTrace { - val path = Paths.get(url.toURI()) - require(path.exists()) { "URL $url does not exist" } - return AzureTrace(factory, path) - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt deleted file mode 100644 index f051bf88..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTable.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.apache.avro.generic.GenericRecord -import org.opendc.trace.* -import org.opendc.trace.util.parquet.LocalParquetReader -import java.nio.file.Path - -/** - * The resource state [Table] in the Bitbrains Parquet format. - */ -internal class BPResourceStateTable(private val path: Path) : Table { - override val name: String = TABLE_RESOURCE_STATES - override val isSynthetic: Boolean = false - - override val columns: List> = listOf( - RESOURCE_STATE_ID, - RESOURCE_STATE_TIMESTAMP, - RESOURCE_STATE_DURATION, - RESOURCE_STATE_NCPUS, - RESOURCE_STATE_CPU_USAGE, - ) - - override fun newReader(): TableReader { - val reader = LocalParquetReader(path.resolve("trace.parquet")) - return BPResourceStateTableReader(reader) - } - - override fun newReader(partition: String): TableReader { - throw IllegalArgumentException("Unknown partition $partition") - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt deleted file mode 100644 index 0e7ee555..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceStateTableReader.kt +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.apache.avro.generic.GenericRecord -import org.opendc.trace.* -import org.opendc.trace.util.parquet.LocalParquetReader -import java.time.Duration -import java.time.Instant - -/** - * A [TableReader] implementation for the Bitbrains Parquet format. - */ -internal class BPResourceStateTableReader(private val reader: LocalParquetReader) : TableReader { - /** - * The current record. - */ - private var record: GenericRecord? = null - - override fun nextRow(): Boolean { - record = reader.read() - return record != null - } - - override fun hasColumn(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_DURATION -> true - RESOURCE_STATE_NCPUS -> true - RESOURCE_STATE_CPU_USAGE -> true - else -> false - } - } - - override fun get(column: TableColumn): T { - val record = checkNotNull(record) { "Reader in invalid state" } - - @Suppress("UNCHECKED_CAST") - val res: Any = when (column) { - RESOURCE_STATE_ID -> record["id"].toString() - RESOURCE_STATE_TIMESTAMP -> Instant.ofEpochMilli(record["time"] as Long) - RESOURCE_STATE_DURATION -> Duration.ofMillis(record["duration"] as Long) - RESOURCE_STATE_NCPUS -> record["cores"] - RESOURCE_STATE_CPU_USAGE -> (record["cpuUsage"] as Number).toDouble() - else -> throw IllegalArgumentException("Invalid column") - } - - @Suppress("UNCHECKED_CAST") - return res as T - } - - override fun getBoolean(column: TableColumn): Boolean { - throw IllegalArgumentException("Invalid column") - } - - override fun getInt(column: TableColumn): Int { - val record = checkNotNull(record) { "Reader in invalid state" } - - return when (column) { - RESOURCE_STATE_NCPUS -> record["cores"] as Int - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun getLong(column: TableColumn): Long { - throw IllegalArgumentException("Invalid column") - } - - override fun getDouble(column: TableColumn): Double { - val record = checkNotNull(record) { "Reader in invalid state" } - return when (column) { - RESOURCE_STATE_CPU_USAGE -> (record["cpuUsage"] as Number).toDouble() - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun close() { - reader.close() - } - - override fun toString(): String = "BPResourceStateTableReader" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt deleted file mode 100644 index 5b0f013f..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTable.kt +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.apache.avro.generic.GenericRecord -import org.opendc.trace.* -import org.opendc.trace.util.parquet.LocalParquetReader -import java.nio.file.Path - -/** - * The resource [Table] in the Bitbrains Parquet format. - */ -internal class BPResourceTable(private val path: Path) : Table { - override val name: String = TABLE_RESOURCES - override val isSynthetic: Boolean = false - - override val columns: List> = listOf( - RESOURCE_ID, - RESOURCE_START_TIME, - RESOURCE_STOP_TIME, - RESOURCE_NCPUS, - RESOURCE_MEM_CAPACITY - ) - - override fun newReader(): TableReader { - val reader = LocalParquetReader(path.resolve("meta.parquet")) - return BPResourceTableReader(reader) - } - - override fun newReader(partition: String): TableReader { - throw IllegalArgumentException("Unknown partition $partition") - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt deleted file mode 100644 index 4416aae8..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPResourceTableReader.kt +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.apache.avro.generic.GenericRecord -import org.opendc.trace.* -import org.opendc.trace.util.parquet.LocalParquetReader -import java.time.Instant - -/** - * A [TableReader] implementation for the Bitbrains Parquet format. - */ -internal class BPResourceTableReader(private val reader: LocalParquetReader) : TableReader { - /** - * The current record. - */ - private var record: GenericRecord? = null - - override fun nextRow(): Boolean { - record = reader.read() - return record != null - } - - override fun hasColumn(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_ID -> true - RESOURCE_START_TIME -> true - RESOURCE_STOP_TIME -> true - RESOURCE_NCPUS -> true - RESOURCE_MEM_CAPACITY -> true - else -> false - } - } - - override fun get(column: TableColumn): T { - val record = checkNotNull(record) { "Reader in invalid state" } - - @Suppress("UNCHECKED_CAST") - val res: Any = when (column) { - RESOURCE_ID -> record["id"].toString() - RESOURCE_START_TIME -> Instant.ofEpochMilli(record["submissionTime"] as Long) - RESOURCE_STOP_TIME -> Instant.ofEpochMilli(record["endTime"] as Long) - RESOURCE_NCPUS -> getInt(RESOURCE_NCPUS) - RESOURCE_MEM_CAPACITY -> getDouble(RESOURCE_MEM_CAPACITY) - else -> throw IllegalArgumentException("Invalid column") - } - - @Suppress("UNCHECKED_CAST") - return res as T - } - - override fun getBoolean(column: TableColumn): Boolean { - throw IllegalArgumentException("Invalid column") - } - - override fun getInt(column: TableColumn): Int { - val record = checkNotNull(record) { "Reader in invalid state" } - - return when (column) { - RESOURCE_NCPUS -> record["maxCores"] as Int - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun getLong(column: TableColumn): Long { - throw IllegalArgumentException("Invalid column") - } - - override fun getDouble(column: TableColumn): Double { - val record = checkNotNull(record) { "Reader in invalid state" } - - return when (column) { - RESOURCE_MEM_CAPACITY -> (record["requiredMemory"] as Number).toDouble() * 1000.0 // MB to KB - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun close() { - reader.close() - } - - override fun toString(): String = "BPResourceTableReader" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt deleted file mode 100644 index 486587b1..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTrace.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.opendc.trace.TABLE_RESOURCES -import org.opendc.trace.TABLE_RESOURCE_STATES -import org.opendc.trace.Table -import org.opendc.trace.Trace -import java.nio.file.Path - -/** - * A [Trace] in the Bitbrains Parquet format. - */ -public class BPTrace internal constructor(private val path: Path) : Trace { - override val tables: List = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES) - - override fun containsTable(name: String): Boolean = - name == TABLE_RESOURCES || name == TABLE_RESOURCE_STATES - - override fun getTable(name: String): Table? { - return when (name) { - TABLE_RESOURCES -> BPResourceTable(path) - TABLE_RESOURCE_STATES -> BPResourceStateTable(path) - else -> null - } - } - - override fun toString(): String = "BPTrace[$path]" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt deleted file mode 100644 index 49d5b4c5..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/BPTraceFormat.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.opendc.trace.spi.TraceFormat -import java.net.URL -import java.nio.file.Paths -import kotlin.io.path.exists - -/** - * A format implementation for the GWF trace format. - */ -public class BPTraceFormat : TraceFormat { - /** - * The name of this trace format. - */ - override val name: String = "bitbrains-parquet" - - /** - * Open a Bitbrains Parquet trace. - */ - override fun open(url: URL): BPTrace { - val path = Paths.get(url.toURI()) - require(path.exists()) { "URL $url does not exist" } - return BPTrace(path) - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt deleted file mode 100644 index 7dd8161d..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/bp/Schemas.kt +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.bp - -import org.apache.avro.Schema -import org.apache.avro.SchemaBuilder - -/** - * Schema for the resources table in the trace. - */ -val BP_RESOURCES_SCHEMA: Schema = SchemaBuilder - .record("meta") - .namespace("org.opendc.trace.capelin") - .fields() - .requiredString("id") - .requiredLong("submissionTime") - .requiredLong("endTime") - .requiredInt("maxCores") - .requiredLong("requiredMemory") - .endRecord() - -/** - * Schema for the resource states table in the trace. - */ -val BP_RESOURCE_STATES_SCHEMA: Schema = SchemaBuilder - .record("meta") - .namespace("org.opendc.trace.capelin") - .fields() - .requiredString("id") - .requiredLong("time") - .requiredLong("duration") - .requiredInt("cores") - .requiredDouble("cpuUsage") - .requiredLong("flops") - .endRecord() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt deleted file mode 100644 index 67140fe9..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTable.kt +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.sv - -import org.opendc.trace.* -import java.nio.file.Files -import java.nio.file.Path -import java.util.stream.Collectors -import kotlin.io.path.bufferedReader -import kotlin.io.path.extension -import kotlin.io.path.nameWithoutExtension - -/** - * The resource state [Table] in the extended Bitbrains format. - */ -internal class SvResourceStateTable(path: Path) : Table { - /** - * The partitions that belong to the table. - */ - private val partitions = Files.walk(path, 1) - .filter { !Files.isDirectory(it) && it.extension == "txt" } - .collect(Collectors.toMap({ it.nameWithoutExtension }, { it })) - .toSortedMap() - - override val name: String = TABLE_RESOURCE_STATES - - override val isSynthetic: Boolean = false - - override val columns: List> = listOf( - RESOURCE_STATE_ID, - RESOURCE_STATE_CLUSTER_ID, - RESOURCE_STATE_TIMESTAMP, - RESOURCE_STATE_NCPUS, - RESOURCE_STATE_CPU_CAPACITY, - RESOURCE_STATE_CPU_USAGE, - RESOURCE_STATE_CPU_USAGE_PCT, - RESOURCE_STATE_CPU_DEMAND, - RESOURCE_STATE_CPU_READY_PCT, - RESOURCE_STATE_MEM_CAPACITY, - RESOURCE_STATE_DISK_READ, - RESOURCE_STATE_DISK_WRITE, - ) - - override fun newReader(): TableReader { - val it = partitions.iterator() - - return object : TableReader { - var delegate: TableReader? = nextDelegate() - - override fun nextRow(): Boolean { - var delegate = delegate - - while (delegate != null) { - if (delegate.nextRow()) { - break - } - - delegate.close() - delegate = nextDelegate() - } - - this.delegate = delegate - return delegate != null - } - - override fun hasColumn(column: TableColumn<*>): Boolean = delegate?.hasColumn(column) ?: false - - override fun get(column: TableColumn): T { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.get(column) - } - - override fun getBoolean(column: TableColumn): Boolean { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getBoolean(column) - } - - override fun getInt(column: TableColumn): Int { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getInt(column) - } - - override fun getLong(column: TableColumn): Long { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getLong(column) - } - - override fun getDouble(column: TableColumn): Double { - val delegate = checkNotNull(delegate) { "Invalid reader state" } - return delegate.getDouble(column) - } - - override fun close() { - delegate?.close() - } - - private fun nextDelegate(): TableReader? { - return if (it.hasNext()) { - val (_, path) = it.next() - val reader = path.bufferedReader() - return SvResourceStateTableReader(reader) - } else { - null - } - } - - override fun toString(): String = "SvCompositeTableReader" - } - } - - override fun newReader(partition: String): TableReader { - val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" } - val reader = path.bufferedReader() - return SvResourceStateTableReader(reader) - } - - override fun toString(): String = "SvResourceStateTable" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt deleted file mode 100644 index 6ea403fe..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvResourceStateTableReader.kt +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.sv - -import org.opendc.trace.* -import java.io.BufferedReader -import java.time.Instant - -/** - * A [TableReader] for the Bitbrains resource state table. - */ -internal class SvResourceStateTableReader(private val reader: BufferedReader) : TableReader { - override fun nextRow(): Boolean { - reset() - - var line: String - var num = 0 - - while (true) { - line = reader.readLine() ?: return false - num++ - - if (line[0] == '#' || line.isBlank()) { - // Ignore empty lines or comments - continue - } - - break - } - - line = line.trim() - - val length = line.length - var col = 0 - var start: Int - var end = 0 - - while (end < length) { - // Trim all whitespace before the field - start = end - while (start < length && line[start].isWhitespace()) { - start++ - } - - end = line.indexOf(' ', start) - - if (end < 0) { - end = length - } - - val field = line.subSequence(start, end) as String - when (col++) { - COL_TIMESTAMP -> timestamp = Instant.ofEpochSecond(field.toLong(10)) - COL_CPU_USAGE -> cpuUsage = field.toDouble() - COL_CPU_DEMAND -> cpuDemand = field.toDouble() - COL_DISK_READ -> diskRead = field.toDouble() - COL_DISK_WRITE -> diskWrite = field.toDouble() - COL_CLUSTER_ID -> cluster = field.trim() - COL_NCPUS -> cpuCores = field.toInt(10) - COL_CPU_READY_PCT -> cpuReadyPct = field.toDouble() - COL_POWERED_ON -> poweredOn = field.toInt(10) == 1 - COL_CPU_CAPACITY -> cpuCapacity = field.toDouble() - COL_ID -> id = field.trim() - COL_MEM_CAPACITY -> memCapacity = field.toDouble() - } - } - - return true - } - - override fun hasColumn(column: TableColumn<*>): Boolean { - return when (column) { - RESOURCE_STATE_ID -> true - RESOURCE_STATE_CLUSTER_ID -> true - RESOURCE_STATE_TIMESTAMP -> true - RESOURCE_STATE_NCPUS -> true - RESOURCE_STATE_CPU_CAPACITY -> true - RESOURCE_STATE_CPU_USAGE -> true - RESOURCE_STATE_CPU_USAGE_PCT -> true - RESOURCE_STATE_CPU_DEMAND -> true - RESOURCE_STATE_CPU_READY_PCT -> true - RESOURCE_STATE_MEM_CAPACITY -> true - RESOURCE_STATE_DISK_READ -> true - RESOURCE_STATE_DISK_WRITE -> true - else -> false - } - } - - override fun get(column: TableColumn): T { - val res: Any? = when (column) { - RESOURCE_STATE_ID -> id - RESOURCE_STATE_CLUSTER_ID -> cluster - RESOURCE_STATE_TIMESTAMP -> timestamp - RESOURCE_STATE_NCPUS -> getInt(RESOURCE_STATE_NCPUS) - RESOURCE_STATE_CPU_CAPACITY -> getDouble(RESOURCE_STATE_CPU_CAPACITY) - RESOURCE_STATE_CPU_USAGE -> getDouble(RESOURCE_STATE_CPU_USAGE) - RESOURCE_STATE_CPU_USAGE_PCT -> getDouble(RESOURCE_STATE_CPU_USAGE_PCT) - RESOURCE_STATE_MEM_CAPACITY -> getDouble(RESOURCE_STATE_MEM_CAPACITY) - RESOURCE_STATE_DISK_READ -> getDouble(RESOURCE_STATE_DISK_READ) - RESOURCE_STATE_DISK_WRITE -> getDouble(RESOURCE_STATE_DISK_WRITE) - else -> throw IllegalArgumentException("Invalid column") - } - - @Suppress("UNCHECKED_CAST") - return res as T - } - - override fun getBoolean(column: TableColumn): Boolean { - return when (column) { - RESOURCE_STATE_POWERED_ON -> poweredOn - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun getInt(column: TableColumn): Int { - return when (column) { - RESOURCE_STATE_NCPUS -> cpuCores - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun getLong(column: TableColumn): Long { - throw IllegalArgumentException("Invalid column") - } - - override fun getDouble(column: TableColumn): Double { - return when (column) { - RESOURCE_STATE_CPU_CAPACITY -> cpuCapacity - RESOURCE_STATE_CPU_USAGE -> cpuUsage - RESOURCE_STATE_CPU_USAGE_PCT -> cpuUsage / cpuCapacity - RESOURCE_STATE_CPU_DEMAND -> cpuDemand - RESOURCE_STATE_MEM_CAPACITY -> memCapacity - RESOURCE_STATE_DISK_READ -> diskRead - RESOURCE_STATE_DISK_WRITE -> diskWrite - else -> throw IllegalArgumentException("Invalid column") - } - } - - override fun close() { - reader.close() - } - - /** - * State fields of the reader. - */ - private var id: String? = null - private var cluster: String? = null - private var timestamp: Instant? = null - private var cpuCores = -1 - private var cpuCapacity = Double.NaN - private var cpuUsage = Double.NaN - private var cpuDemand = Double.NaN - private var cpuReadyPct = Double.NaN - private var memCapacity = Double.NaN - private var diskRead = Double.NaN - private var diskWrite = Double.NaN - private var poweredOn: Boolean = false - - /** - * Reset the state of the reader. - */ - private fun reset() { - id = null - timestamp = null - cluster = null - cpuCores = -1 - cpuCapacity = Double.NaN - cpuUsage = Double.NaN - cpuDemand = Double.NaN - cpuReadyPct = Double.NaN - memCapacity = Double.NaN - diskRead = Double.NaN - diskWrite = Double.NaN - poweredOn = false - } - - /** - * Default column indices for the extended Bitbrains format. - */ - private val COL_TIMESTAMP = 0 - private val COL_CPU_USAGE = 1 - private val COL_CPU_DEMAND = 2 - private val COL_DISK_READ = 4 - private val COL_DISK_WRITE = 6 - private val COL_CLUSTER_ID = 10 - private val COL_NCPUS = 12 - private val COL_CPU_READY_PCT = 13 - private val COL_POWERED_ON = 14 - private val COL_CPU_CAPACITY = 18 - private val COL_ID = 19 - private val COL_MEM_CAPACITY = 20 -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt deleted file mode 100644 index dbd63de5..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTrace.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.sv - -import org.opendc.trace.* -import java.nio.file.Path - -/** - * [Trace] implementation for the extended Bitbrains format. - */ -public class SvTrace internal constructor(private val path: Path) : Trace { - override val tables: List = listOf(TABLE_RESOURCE_STATES) - - override fun containsTable(name: String): Boolean = TABLE_RESOURCE_STATES == name - - override fun getTable(name: String): Table? { - if (!containsTable(name)) { - return null - } - - return SvResourceStateTable(path) - } - - override fun toString(): String = "SvTrace[$path]" -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt deleted file mode 100644 index 0cce8559..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/sv/SvTraceFormat.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace.sv - -import org.opendc.trace.spi.TraceFormat -import java.net.URL -import java.nio.file.Paths -import kotlin.io.path.exists - -/** - * A format implementation for the extended Bitbrains trace format. - */ -public class SvTraceFormat : TraceFormat { - /** - * The name of this trace format. - */ - override val name: String = "sv" - - /** - * Open the trace file. - */ - override fun open(url: URL): SvTrace { - val path = Paths.get(url.toURI()) - require(path.exists()) { "URL $url does not exist" } - return SvTrace(path) - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt deleted file mode 100644 index 065a8c93..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeServiceSimulator.kt +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.util - -import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.export.MetricProducer -import io.opentelemetry.sdk.resources.Resource -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.yield -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.ComputeScheduler -import org.opendc.compute.simulator.SimHost -import org.opendc.experiments.capelin.env.MachineDef -import org.opendc.experiments.capelin.trace.TraceReader -import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider -import org.opendc.simulator.compute.kernel.SimHypervisorProvider -import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.compute.power.SimplePowerDriver -import org.opendc.simulator.compute.workload.SimTraceWorkload -import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.SimResourceInterpreter -import org.opendc.telemetry.compute.* -import org.opendc.telemetry.sdk.toOtelClock -import java.time.Clock -import kotlin.coroutines.CoroutineContext -import kotlin.math.max - -/** - * Helper class to manage a [ComputeService] simulation. - */ -class ComputeServiceSimulator( - private val context: CoroutineContext, - private val clock: Clock, - scheduler: ComputeScheduler, - machines: List, - private val failureModel: FailureModel? = null, - interferenceModel: VmInterferenceModel? = null, - hypervisorProvider: SimHypervisorProvider = SimFairShareHypervisorProvider() -) : AutoCloseable { - /** - * The [ComputeService] that has been configured by the manager. - */ - val service: ComputeService - - /** - * The [MetricProducer] that are used by the [ComputeService] and the simulated hosts. - */ - val producers: List - get() = _metricProducers - private val _metricProducers = mutableListOf() - - /** - * The [SimResourceInterpreter] to simulate the hosts. - */ - private val interpreter = SimResourceInterpreter(context, clock) - - /** - * The hosts that belong to this class. - */ - private val hosts = mutableSetOf() - - init { - val (service, serviceMeterProvider) = createService(scheduler) - this._metricProducers.add(serviceMeterProvider) - this.service = service - - for (def in machines) { - val (host, hostMeterProvider) = createHost(def, hypervisorProvider, interferenceModel) - this._metricProducers.add(hostMeterProvider) - hosts.add(host) - this.service.addHost(host) - } - } - - /** - * Run a simulation of the [ComputeService] by replaying the workload trace given by [reader]. - */ - suspend fun run(reader: TraceReader) { - val injector = failureModel?.createInjector(context, clock, service) - val client = service.newClient() - - // Create new image for the virtual machine - val image = client.newImage("vm-image") - - try { - coroutineScope { - // Start the fault injector - injector?.start() - - var offset = Long.MIN_VALUE - - while (reader.hasNext()) { - val entry = reader.next() - - if (offset < 0) { - offset = entry.start - clock.millis() - } - - // Make sure the trace entries are ordered by submission time - assert(entry.start - offset >= 0) { "Invalid trace order" } - delay(max(0, (entry.start - offset) - clock.millis())) - - launch { - val workloadOffset = -offset + 300001 - val workload = SimTraceWorkload((entry.meta["workload"] as SimTraceWorkload).trace, workloadOffset) - - val server = client.newServer( - entry.name, - image, - client.newFlavor( - entry.name, - entry.meta["cores"] as Int, - entry.meta["required-memory"] as Long - ), - meta = entry.meta + mapOf("workload" to workload) - ) - - // Wait for the server reach its end time - val endTime = entry.meta["end-time"] as Long - delay(endTime + workloadOffset - clock.millis() + 1) - - // Delete the server after reaching the end-time of the virtual machine - server.delete() - } - } - } - - yield() - } finally { - injector?.close() - reader.close() - client.close() - } - } - - override fun close() { - service.close() - - for (host in hosts) { - host.close() - } - - hosts.clear() - } - - /** - * Construct a [ComputeService] instance. - */ - private fun createService(scheduler: ComputeScheduler): Pair { - val resource = Resource.builder() - .put(ResourceAttributes.SERVICE_NAME, "opendc-compute") - .build() - - val meterProvider = SdkMeterProvider.builder() - .setClock(clock.toOtelClock()) - .setResource(resource) - .build() - - val service = ComputeService(context, clock, meterProvider, scheduler) - return service to meterProvider - } - - /** - * Construct a [SimHost] instance for the specified [MachineDef]. - */ - private fun createHost( - def: MachineDef, - hypervisorProvider: SimHypervisorProvider, - interferenceModel: VmInterferenceModel? = null - ): Pair { - val resource = Resource.builder() - .put(HOST_ID, def.uid.toString()) - .put(HOST_NAME, def.name) - .put(HOST_ARCH, ResourceAttributes.HostArchValues.AMD64) - .put(HOST_NCPUS, def.model.cpus.size) - .put(HOST_MEM_CAPACITY, def.model.memory.sumOf { it.size }) - .build() - - val meterProvider = SdkMeterProvider.builder() - .setClock(clock.toOtelClock()) - .setResource(resource) - .build() - - val host = SimHost( - def.uid, - def.name, - def.model, - def.meta, - context, - interpreter, - meterProvider, - hypervisorProvider, - powerDriver = SimplePowerDriver(def.powerModel), - interferenceDomain = interferenceModel?.newDomain() - ) - - return host to meterProvider - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt deleted file mode 100644 index 83393896..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModel.kt +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.util - -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.failure.HostFaultInjector -import java.time.Clock -import kotlin.coroutines.CoroutineContext - -/** - * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. - */ -interface FailureModel { - /** - * Construct a [HostFaultInjector] for the specified [service]. - */ - fun createInjector(context: CoroutineContext, clock: Clock, service: ComputeService): HostFaultInjector -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt deleted file mode 100644 index 89b4a31c..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/FailureModels.kt +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("FailureModels") -package org.opendc.experiments.capelin - -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.StartStopHostFault -import org.opendc.compute.simulator.failure.StochasticVictimSelector -import org.opendc.experiments.capelin.util.FailureModel -import java.time.Clock -import java.time.Duration -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln -import kotlin.random.Random - -/** - * Obtain a [FailureModel] based on the GRID'5000 failure trace. - * - * This fault injector uses parameters from the GRID'5000 failure trace as described in - * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. - */ -fun grid5000(failureInterval: Duration, seed: Int): FailureModel { - return object : FailureModel { - override fun createInjector( - context: CoroutineContext, - clock: Clock, - service: ComputeService - ): HostFaultInjector { - val rng = Well19937c(seed) - val hosts = service.hosts.map { it as SimHost }.toSet() - - // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 - // GRID'5000 - return HostFaultInjector( - context, - clock, - hosts, - iat = LogNormalDistribution(rng, ln(failureInterval.toHours().toDouble()), 1.03), - selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), - fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) - ) - } - - override fun toString(): String = "Grid5000FailureModel" - } -} - -/** - * Obtain the [HostFaultInjector] to use for the experiments. - * - * This fault injector uses parameters from the GRID'5000 failure trace as described in - * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. - */ -fun createFaultInjector( - context: CoroutineContext, - clock: Clock, - hosts: Set, - seed: Int, - failureInterval: Double -): HostFaultInjector { - val rng = Well19937c(seed) - - // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 - // GRID'5000 - return HostFaultInjector( - context, - clock, - hosts, - iat = LogNormalDistribution(rng, ln(failureInterval), 1.03), - selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), Random(seed)), - fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) - ) -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/VmPlacementReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/VmPlacementReader.kt new file mode 100644 index 00000000..67de2777 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/VmPlacementReader.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.util + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import java.io.InputStream + +/** + * A parser for the JSON VM placement data files used for the TPDS article on Capelin. + */ +class VmPlacementReader { + /** + * The [ObjectMapper] to parse the placement. + */ + private val mapper = jacksonObjectMapper() + + /** + * Read the VM placements from the input. + */ + fun read(input: InputStream): Map { + return mapper.readValue>(input) + .mapKeys { "vm__workload__${it.key}.txt" } + .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/resources/log4j2.xml b/opendc-experiments/opendc-experiments-capelin/src/main/resources/log4j2.xml index d1c01b8e..d46b50c3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/resources/log4j2.xml +++ b/opendc-experiments/opendc-experiments-capelin/src/main/resources/log4j2.xml @@ -36,7 +36,7 @@ - + diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 727530e3..b0f86346 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -31,14 +31,15 @@ import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher +import org.opendc.compute.workload.ComputeWorkloadRunner +import org.opendc.compute.workload.grid5000 +import org.opendc.compute.workload.trace.RawParquetTraceReader +import org.opendc.compute.workload.trace.TraceReader +import org.opendc.compute.workload.util.PerformanceInterferenceReader import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.trace.ParquetTraceReader -import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader -import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.experiments.capelin.trace.TraceReader -import org.opendc.experiments.capelin.util.ComputeServiceSimulator import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation @@ -85,7 +86,7 @@ class CapelinIntegrationTest { val traceReader = createTestTraceReader() val environmentReader = createTestEnvironmentReader() - val simulator = ComputeServiceSimulator( + val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, @@ -134,7 +135,7 @@ class CapelinIntegrationTest { val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") - val simulator = ComputeServiceSimulator( + val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, @@ -184,7 +185,7 @@ class CapelinIntegrationTest { .read(perfInterferenceInput) .let { VmInterferenceModel(it, Random(seed.toLong())) } - val simulator = ComputeServiceSimulator( + val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, @@ -229,7 +230,7 @@ class CapelinIntegrationTest { val traceReader = createTestTraceReader(0.25, seed) val environmentReader = createTestEnvironmentReader("single") - val simulator = ComputeServiceSimulator( + val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt deleted file mode 100644 index fbc39b87..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/trace/PerformanceInterferenceReaderTest.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertAll - -/** - * Test suite for the [PerformanceInterferenceReader] class. - */ -class PerformanceInterferenceReaderTest { - @Test - fun testSmoke() { - val input = checkNotNull(PerformanceInterferenceReader::class.java.getResourceAsStream("/perf-interference.json")) - val result = PerformanceInterferenceReader().read(input) - - assertAll( - { assertEquals(2, result.size) }, - { assertEquals(setOf("vm_a", "vm_c", "vm_x", "vm_y"), result[0].members) }, - { assertEquals(0.0, result[0].targetLoad, 0.001) }, - { assertEquals(0.8830158730158756, result[0].score, 0.001) } - ) - } -} -- cgit v1.2.3 From b0ece0533825f5cd7983752330847071f4e438c4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 15 Sep 2021 23:06:08 +0200 Subject: refactor(capelin): Support flexible topology creation This change adds support for creating flexible topologies by creating a TopologyFactory interface that is responsible for configuring the hosts of a compute service. --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../org/opendc/experiments/capelin/Portfolio.kt | 25 ++--- .../capelin/env/ClusterEnvironmentReader.kt | 121 --------------------- .../experiments/capelin/env/EnvironmentReader.kt | 36 ------ .../experiments/capelin/topology/ClusterSpec.kt | 46 ++++++++ .../capelin/topology/ClusterSpecReader.kt | 121 +++++++++++++++++++++ .../capelin/topology/TopologyFactories.kt | 103 ++++++++++++++++++ .../experiments/capelin/CapelinIntegrationTest.kt | 39 +++---- .../src/test/resources/perf-interference.json | 22 ---- 9 files changed, 300 insertions(+), 214 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpec.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpecReader.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 010d18b0..4bcbaf61 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.jackson.databind) implementation(libs.jackson.module.kotlin) + implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) implementation(libs.opentelemetry.semconv) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 06db5569..02811d83 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -27,13 +27,14 @@ import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkloadRunner import org.opendc.compute.workload.export.parquet.ParquetExportMonitor import org.opendc.compute.workload.grid5000 +import org.opendc.compute.workload.topology.apply import org.opendc.compute.workload.trace.RawParquetTraceReader import org.opendc.compute.workload.util.PerformanceInterferenceReader -import org.opendc.experiments.capelin.env.ClusterEnvironmentReader import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.capelin.topology.clusterTopology import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.util.createComputeScheduler import org.opendc.harness.dsl.Experiment @@ -100,12 +101,6 @@ abstract class Portfolio(name: String) : Experiment(name) { */ override fun doRun(repeat: Int): Unit = runBlockingSimulation { val seeder = Random(repeat.toLong()) - val environment = ClusterEnvironmentReader( - File( - config.getString("env-path"), - "${topology.name}.txt" - ) - ) val workload = workload val workloadNames = if (workload is CompositeWorkload) { @@ -133,11 +128,10 @@ abstract class Portfolio(name: String) : Experiment(name) { grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong()), seeder.nextInt()) else null - val simulator = ComputeWorkloadRunner( + val runner = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, - environment.read(), failureModel, performanceInterferenceModel ) @@ -147,17 +141,22 @@ abstract class Portfolio(name: String) : Experiment(name) { "portfolio_id=$name/scenario_id=$id/run_id=$repeat", 4096 ) - val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, runner.producers, ComputeMetricExporter(clock, monitor)) + val topology = clusterTopology(File(config.getString("env-path"), "${topology.name}.txt")) try { - simulator.run(trace) + // Instantiate the desired topology + runner.apply(topology) + + // Run the workload trace + runner.run(trace) } finally { - simulator.close() + runner.close() metricReader.close() monitor.close() } - val monitorResults = collectServiceMetrics(clock.instant(), simulator.producers[0]) + val monitorResults = collectServiceMetrics(clock.instant(), runner.producers[0]) logger.debug { "Scheduler " + "Success=${monitorResults.attemptsSuccess} " + diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt deleted file mode 100644 index 8d9b24f4..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/ClusterEnvironmentReader.kt +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.env - -import org.opendc.compute.workload.env.MachineDef -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.model.ProcessingNode -import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.LinearPowerModel -import java.io.File -import java.io.FileInputStream -import java.io.InputStream -import java.util.* - -/** - * A [EnvironmentReader] for the internal environment format. - * - * @param input The input stream describing the physical cluster. - */ -class ClusterEnvironmentReader(private val input: InputStream) : EnvironmentReader { - /** - * Construct a [ClusterEnvironmentReader] for the specified [file]. - */ - constructor(file: File) : this(FileInputStream(file)) - - override fun read(): List { - var clusterIdCol = 0 - var speedCol = 0 - var numberOfHostsCol = 0 - var memoryPerHostCol = 0 - var coresPerHostCol = 0 - - var clusterIdx = 0 - var clusterId: String - var speed: Double - var numberOfHosts: Int - var memoryPerHost: Long - var coresPerHost: Int - - val nodes = mutableListOf() - val random = Random(0) - - input.bufferedReader().use { reader -> - reader.lineSequence() - .filter { line -> - // Ignore comments in the file - !line.startsWith("#") && line.isNotBlank() - } - .forEachIndexed { idx, line -> - val values = line.split(";") - - if (idx == 0) { - val header = values.mapIndexed { col, name -> Pair(name.trim(), col) }.toMap() - clusterIdCol = header["ClusterID"]!! - speedCol = header["Speed"]!! - numberOfHostsCol = header["numberOfHosts"]!! - memoryPerHostCol = header["memoryCapacityPerHost"]!! - coresPerHostCol = header["coreCountPerHost"]!! - return@forEachIndexed - } - - clusterIdx++ - clusterId = values[clusterIdCol].trim() - speed = values[speedCol].trim().toDouble() * 1000.0 - numberOfHosts = values[numberOfHostsCol].trim().toInt() - memoryPerHost = values[memoryPerHostCol].trim().toLong() * 1000L - coresPerHost = values[coresPerHostCol].trim().toInt() - - val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", coresPerHost) - val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) - - repeat(numberOfHosts) { - nodes.add( - MachineDef( - UUID(random.nextLong(), random.nextLong()), - "node-$clusterId-$it", - mapOf("cluster" to clusterId), - MachineModel( - List(coresPerHost) { coreId -> - ProcessingUnit(unknownProcessingNode, coreId, speed) - }, - listOf(unknownMemoryUnit) - ), - // For now we assume a simple linear load model with an idle draw of ~200W and a maximum - // power draw of 350W. - // Source: https://stackoverflow.com/questions/6128960 - LinearPowerModel(350.0, idlePower = 200.0) - ) - ) - } - } - } - - return nodes - } - - override fun close() { - input.close() - } -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt deleted file mode 100644 index 8d61c530..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/env/EnvironmentReader.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.env - -import org.opendc.compute.workload.env.MachineDef -import java.io.Closeable - -/** - * An interface for reading descriptions of topology environments into memory. - */ -public interface EnvironmentReader : Closeable { - /** - * Read the environment into a list. - */ - public fun read(): List -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpec.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpec.kt new file mode 100644 index 00000000..b8b65d28 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpec.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.topology + +/** + * Definition of a compute cluster modeled in the simulation. + * + * @param id A unique identifier representing the compute cluster. + * @param name The name of the cluster. + * @param cpuCount The total number of CPUs in the cluster. + * @param cpuSpeed The speed of a CPU in the cluster in MHz. + * @param memCapacity The total memory capacity of the cluster (in MiB). + * @param hostCount The number of hosts in the cluster. + * @param memCapacityPerHost The memory capacity per host in the cluster (MiB). + * @param cpuCountPerHost The number of CPUs per host in the cluster. + */ +public data class ClusterSpec( + val id: String, + val name: String, + val cpuCount: Int, + val cpuSpeed: Double, + val memCapacity: Double, + val hostCount: Int, + val memCapacityPerHost: Double, + val cpuCountPerHost: Int +) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpecReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpecReader.kt new file mode 100644 index 00000000..5a175f2c --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/ClusterSpecReader.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin.topology + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.MappingIterator +import com.fasterxml.jackson.databind.ObjectReader +import com.fasterxml.jackson.dataformat.csv.CsvMapper +import com.fasterxml.jackson.dataformat.csv.CsvSchema +import java.io.File +import java.io.InputStream + +/** + * A helper class for reading a cluster specification file. + */ +class ClusterSpecReader { + /** + * The [CsvMapper] to map the environment file to an object. + */ + private val mapper = CsvMapper() + + /** + * The [ObjectReader] to convert the lines into objects. + */ + private val reader: ObjectReader = mapper.readerFor(Entry::class.java).with(schema) + + /** + * Read the specified [file]. + */ + fun read(file: File): List { + return reader.readValues(file).use { read(it) } + } + + /** + * Read the specified [input]. + */ + fun read(input: InputStream): List { + return reader.readValues(input).use { read(it) } + } + + /** + * Convert the specified [MappingIterator] into a list of [ClusterSpec]s. + */ + private fun read(it: MappingIterator): List { + val result = mutableListOf() + + for (entry in it) { + val def = ClusterSpec( + entry.id, + entry.name, + entry.cpuCount, + entry.cpuSpeed * 1000, // Convert to MHz + entry.memCapacity * 1000, // Convert to MiB + entry.hostCount, + entry.memCapacityPerHost * 1000, + entry.cpuCountPerHost + ) + result.add(def) + } + + return result + } + + private open class Entry( + @JsonProperty("ClusterID") + val id: String, + @JsonProperty("ClusterName") + val name: String, + @JsonProperty("Cores") + val cpuCount: Int, + @JsonProperty("Speed") + val cpuSpeed: Double, + @JsonProperty("Memory") + val memCapacity: Double, + @JsonProperty("numberOfHosts") + val hostCount: Int, + @JsonProperty("memoryCapacityPerHost") + val memCapacityPerHost: Double, + @JsonProperty("coreCountPerHost") + val cpuCountPerHost: Int + ) + + companion object { + /** + * The [CsvSchema] that is used to parse the trace. + */ + private val schema = CsvSchema.builder() + .addColumn("ClusterID", CsvSchema.ColumnType.STRING) + .addColumn("ClusterName", CsvSchema.ColumnType.STRING) + .addColumn("Cores", CsvSchema.ColumnType.NUMBER) + .addColumn("Speed", CsvSchema.ColumnType.NUMBER) + .addColumn("Memory", CsvSchema.ColumnType.NUMBER) + .addColumn("numberOfHosts", CsvSchema.ColumnType.NUMBER) + .addColumn("memoryCapacityPerHost", CsvSchema.ColumnType.NUMBER) + .addColumn("coreCountPerHost", CsvSchema.ColumnType.NUMBER) + .setAllowComments(true) + .setColumnSeparator(';') + .setUseHeader(true) + .build() + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt new file mode 100644 index 00000000..5ab4261a --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("TopologyFactories") +package org.opendc.experiments.capelin.topology + +import org.opendc.compute.workload.topology.HostSpec +import org.opendc.compute.workload.topology.Topology +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit +import org.opendc.simulator.compute.power.LinearPowerModel +import org.opendc.simulator.compute.power.PowerModel +import org.opendc.simulator.compute.power.SimplePowerDriver +import java.io.File +import java.io.InputStream +import java.util.* +import kotlin.math.roundToLong + +/** + * A [ClusterSpecReader] that is used to read the cluster definition file. + */ +private val reader = ClusterSpecReader() + +/** + * Construct a [Topology] from the specified [file]. + */ +fun clusterTopology( + file: File, + powerModel: PowerModel = LinearPowerModel(350.0, idlePower = 200.0), + random: Random = Random(0) +): Topology = clusterTopology(reader.read(file), powerModel, random) + +/** + * Construct a [Topology] from the specified [input]. + */ +fun clusterTopology( + input: InputStream, + powerModel: PowerModel = LinearPowerModel(350.0, idlePower = 200.0), + random: Random = Random(0) +): Topology = clusterTopology(reader.read(input), powerModel, random) + +/** + * Construct a [Topology] from the given list of [clusters]. + */ +fun clusterTopology( + clusters: List, + powerModel: PowerModel, + random: Random = Random(0) +): Topology { + return object : Topology { + override fun resolve(): List { + val hosts = mutableListOf() + for (cluster in clusters) { + val cpuSpeed = cluster.cpuSpeed + val memoryPerHost = cluster.memCapacityPerHost.roundToLong() + + val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", cluster.cpuCountPerHost) + val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) + val machineModel = MachineModel( + List(cluster.cpuCountPerHost) { coreId -> ProcessingUnit(unknownProcessingNode, coreId, cpuSpeed) }, + listOf(unknownMemoryUnit) + ) + + repeat(cluster.hostCount) { + val spec = HostSpec( + UUID(random.nextLong(), it.toLong()), + "node-${cluster.name}-$it", + mapOf("cluster" to cluster.id), + machineModel, + SimplePowerDriver(powerModel) + ) + + hosts += spec + } + } + + return hosts + } + + override fun toString(): String = "ClusterSpecTopology" + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index b0f86346..c1386bfe 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -33,12 +33,13 @@ import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher import org.opendc.compute.workload.ComputeWorkloadRunner import org.opendc.compute.workload.grid5000 +import org.opendc.compute.workload.topology.Topology +import org.opendc.compute.workload.topology.apply import org.opendc.compute.workload.trace.RawParquetTraceReader import org.opendc.compute.workload.trace.TraceReader import org.opendc.compute.workload.util.PerformanceInterferenceReader -import org.opendc.experiments.capelin.env.ClusterEnvironmentReader -import org.opendc.experiments.capelin.env.EnvironmentReader import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.capelin.topology.clusterTopology import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.compute.workload.SimWorkload @@ -84,18 +85,16 @@ class CapelinIntegrationTest { @Test fun testLarge() = runBlockingSimulation { val traceReader = createTestTraceReader() - val environmentReader = createTestEnvironmentReader() - val simulator = ComputeWorkloadRunner( coroutineContext, clock, - computeScheduler, - environmentReader.read(), + computeScheduler ) - + val topology = createTopology() val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) try { + simulator.apply(topology) simulator.run(traceReader) } finally { simulator.close() @@ -133,18 +132,17 @@ class CapelinIntegrationTest { fun testSmall() = runBlockingSimulation { val seed = 1 val traceReader = createTestTraceReader(0.25, seed) - val environmentReader = createTestEnvironmentReader("single") val simulator = ComputeWorkloadRunner( coroutineContext, clock, - computeScheduler, - environmentReader.read(), + computeScheduler ) - + val topology = createTopology("single") val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) try { + simulator.apply(topology) simulator.run(traceReader) } finally { simulator.close() @@ -177,7 +175,6 @@ class CapelinIntegrationTest { fun testInterference() = runBlockingSimulation { val seed = 1 val traceReader = createTestTraceReader(0.25, seed) - val environmentReader = createTestEnvironmentReader("single") val perfInterferenceInput = checkNotNull(CapelinIntegrationTest::class.java.getResourceAsStream("/bitbrains-perf-interference.json")) val performanceInterferenceModel = @@ -189,13 +186,13 @@ class CapelinIntegrationTest { coroutineContext, clock, computeScheduler, - environmentReader.read(), interferenceModel = performanceInterferenceModel ) - + val topology = createTopology("single") val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) try { + simulator.apply(topology) simulator.run(traceReader) } finally { simulator.close() @@ -227,20 +224,18 @@ class CapelinIntegrationTest { @Test fun testFailures() = runBlockingSimulation { val seed = 1 - val traceReader = createTestTraceReader(0.25, seed) - val environmentReader = createTestEnvironmentReader("single") - val simulator = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler, - environmentReader.read(), grid5000(Duration.ofDays(7), seed) ) - + val topology = createTopology("single") + val traceReader = createTestTraceReader(0.25, seed) val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) try { + simulator.apply(topology) simulator.run(traceReader) } finally { simulator.close() @@ -279,11 +274,11 @@ class CapelinIntegrationTest { } /** - * Obtain the environment reader for the test. + * Obtain the topology factory for the test. */ - private fun createTestEnvironmentReader(name: String = "topology"): EnvironmentReader { + private fun createTopology(name: String = "topology"): Topology { val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name.txt")) - return ClusterEnvironmentReader(stream) + return stream.use { clusterTopology(stream) } } class TestExperimentReporter : ComputeMonitor { diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json b/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json deleted file mode 100644 index 1be5852b..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/test/resources/perf-interference.json +++ /dev/null @@ -1,22 +0,0 @@ -[ - { - "vms": [ - "vm_a", - "vm_c", - "vm_x", - "vm_y" - ], - "minServerLoad": 0.0, - "performanceScore": 0.8830158730158756 - }, - { - "vms": [ - "vm_a", - "vm_b", - "vm_c", - "vm_d" - ], - "minServerLoad": 0.0, - "performanceScore": 0.7133055555552751 - } -] -- cgit v1.2.3 From b14df2a0924774c5aed15cedeb1027abf8ee5361 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 16 Sep 2021 16:52:00 +0200 Subject: refactor(capelin): Make workload sampling model extensible This change updates the workload sampling implementation to be more flexible in the way the workload is constructed. Users can now sample multiple workloads at the same time using multiple samplers and use them as a single workload to simulate. --- .../capelin/CompositeWorkloadPortfolio.kt | 28 ++- .../opendc/experiments/capelin/HorVerPortfolio.kt | 10 +- .../opendc/experiments/capelin/MoreHpcPortfolio.kt | 18 +- .../experiments/capelin/MoreVelocityPortfolio.kt | 10 +- .../capelin/OperationalPhenomenaPortfolio.kt | 10 +- .../org/opendc/experiments/capelin/Portfolio.kt | 26 +-- .../opendc/experiments/capelin/ReplayPortfolio.kt | 3 +- .../opendc/experiments/capelin/TestPortfolio.kt | 3 +- .../opendc/experiments/capelin/model/Workload.kt | 23 +-- .../capelin/trace/ParquetTraceReader.kt | 68 ------- .../experiments/capelin/trace/WorkloadSampler.kt | 199 --------------------- .../experiments/capelin/CapelinIntegrationTest.kt | 84 +++++---- .../resources/trace/bitbrains-small/meta.parquet | Bin 0 -> 2081 bytes .../resources/trace/bitbrains-small/trace.parquet | Bin 0 -> 1647189 bytes .../src/test/resources/trace/meta.parquet | Bin 2081 -> 0 bytes .../src/test/resources/trace/trace.parquet | Bin 1647189 -> 0 bytes .../opendc-experiments-radice/build.gradle.kts | 47 ----- 17 files changed, 95 insertions(+), 434 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet create mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet delete mode 100644 opendc-experiments/opendc-experiments-radice/build.gradle.kts (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CompositeWorkloadPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CompositeWorkloadPortfolio.kt index faabe5cb..31e8f961 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CompositeWorkloadPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CompositeWorkloadPortfolio.kt @@ -22,7 +22,8 @@ package org.opendc.experiments.capelin -import org.opendc.experiments.capelin.model.CompositeWorkload +import org.opendc.compute.workload.composite +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -42,30 +43,25 @@ public class CompositeWorkloadPortfolio : Portfolio("composite-workload") { ) override val workload: Workload by anyOf( - CompositeWorkload( + Workload( "all-azure", - listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)), - totalSampleLoad + composite(trace("solvinity-short") to 0.0, trace("azure") to 1.0) ), - CompositeWorkload( + Workload( "solvinity-25-azure-75", - listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)), - totalSampleLoad + composite(trace("solvinity-short") to 0.25, trace("azure") to 0.75) ), - CompositeWorkload( + Workload( "solvinity-50-azure-50", - listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)), - totalSampleLoad + composite(trace("solvinity-short") to 0.5, trace("azure") to 0.5) ), - CompositeWorkload( + Workload( "solvinity-75-azure-25", - listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)), - totalSampleLoad + composite(trace("solvinity-short") to 0.75, trace("azure") to 0.25) ), - CompositeWorkload( + Workload( "all-solvinity", - listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)), - totalSampleLoad + composite(trace("solvinity-short") to 1.0, trace("azure") to 0.0) ) ) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/HorVerPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/HorVerPortfolio.kt index e1cf8517..cd093e6c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/HorVerPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/HorVerPortfolio.kt @@ -22,6 +22,8 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.sampleByLoad +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -44,10 +46,10 @@ public class HorVerPortfolio : Portfolio("horizontal_vs_vertical") { ) override val workload: Workload by anyOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) + Workload("solvinity", trace("solvinity").sampleByLoad(0.1)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.25)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.5)), + Workload("solvinity", trace("solvinity").sampleByLoad(1.0)) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreHpcPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreHpcPortfolio.kt index a995e467..73e59a58 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreHpcPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreHpcPortfolio.kt @@ -22,8 +22,10 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.sampleByHpc +import org.opendc.compute.workload.sampleByHpcLoad +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena -import org.opendc.experiments.capelin.model.SamplingStrategy import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.harness.dsl.anyOf @@ -40,13 +42,13 @@ public class MoreHpcPortfolio : Portfolio("more_hpc") { ) override val workload: Workload by anyOf( - Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) + Workload("solvinity", trace("solvinity").sampleByHpc(0.0)), + Workload("solvinity", trace("solvinity").sampleByHpc(0.25)), + Workload("solvinity", trace("solvinity").sampleByHpc(0.5)), + Workload("solvinity", trace("solvinity").sampleByHpc(1.0)), + Workload("solvinity", trace("solvinity").sampleByHpcLoad(0.25)), + Workload("solvinity", trace("solvinity").sampleByHpcLoad(0.5)), + Workload("solvinity", trace("solvinity").sampleByHpcLoad(1.0)) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreVelocityPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreVelocityPortfolio.kt index 49559e0e..9d5717bb 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreVelocityPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/MoreVelocityPortfolio.kt @@ -22,6 +22,8 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.sampleByLoad +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -40,10 +42,10 @@ public class MoreVelocityPortfolio : Portfolio("more_velocity") { ) override val workload: Workload by anyOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) + Workload("solvinity", trace("solvinity").sampleByLoad(0.1)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.25)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.5)), + Workload("solvinity", trace("solvinity").sampleByLoad(1.0)) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/OperationalPhenomenaPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/OperationalPhenomenaPortfolio.kt index 1aac4f9e..7ab586b3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/OperationalPhenomenaPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/OperationalPhenomenaPortfolio.kt @@ -22,6 +22,8 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.sampleByLoad +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -36,10 +38,10 @@ public class OperationalPhenomenaPortfolio : Portfolio("operational_phenomena") ) override val workload: Workload by anyOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) + Workload("solvinity", trace("solvinity").sampleByLoad(0.1)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.25)), + Workload("solvinity", trace("solvinity").sampleByLoad(0.5)), + Workload("solvinity", trace("solvinity").sampleByLoad(1.0)) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 02811d83..630b76c4 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -24,18 +24,16 @@ package org.opendc.experiments.capelin import com.typesafe.config.ConfigFactory import mu.KotlinLogging +import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.ComputeWorkloadRunner import org.opendc.compute.workload.export.parquet.ParquetExportMonitor import org.opendc.compute.workload.grid5000 import org.opendc.compute.workload.topology.apply -import org.opendc.compute.workload.trace.RawParquetTraceReader import org.opendc.compute.workload.util.PerformanceInterferenceReader -import org.opendc.experiments.capelin.model.CompositeWorkload import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.topology.clusterTopology -import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.experiments.capelin.util.createComputeScheduler import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf @@ -47,7 +45,6 @@ import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File import java.time.Duration import java.util.* -import java.util.concurrent.ConcurrentHashMap import kotlin.math.roundToLong /** @@ -92,9 +89,9 @@ abstract class Portfolio(name: String) : Experiment(name) { abstract val allocationPolicy: String /** - * A map of trace readers. + * A helper class to load workload traces. */ - private val traceReaders = ConcurrentHashMap() + private val workloadLoader = ComputeWorkloadLoader(File(config.getString("trace-path"))) /** * Perform a single trial for this portfolio. @@ -102,19 +99,6 @@ abstract class Portfolio(name: String) : Experiment(name) { override fun doRun(repeat: Int): Unit = runBlockingSimulation { val seeder = Random(repeat.toLong()) - val workload = workload - val workloadNames = if (workload is CompositeWorkload) { - workload.workloads.map { it.name } - } else { - listOf(workload.name) - } - val rawReaders = workloadNames.map { workloadName -> - traceReaders.computeIfAbsent(workloadName) { - logger.info { "Loading trace $workloadName" } - RawParquetTraceReader(File(config.getString("trace-path"), workloadName)) - } - } - val trace = ParquetTraceReader(rawReaders, workload, seeder.nextInt()) val performanceInterferenceModel = if (operationalPhenomena.hasInterference) PerformanceInterferenceReader() .read(File(config.getString("interference-model"))) @@ -125,7 +109,7 @@ abstract class Portfolio(name: String) : Experiment(name) { val computeScheduler = createComputeScheduler(allocationPolicy, seeder, vmPlacements) val failureModel = if (operationalPhenomena.failureFrequency > 0) - grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong()), seeder.nextInt()) + grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong())) else null val runner = ComputeWorkloadRunner( @@ -149,7 +133,7 @@ abstract class Portfolio(name: String) : Experiment(name) { runner.apply(topology) // Run the workload trace - runner.run(trace) + runner.run(workload.source.resolve(workloadLoader, seeder), seeder.nextLong()) } finally { runner.close() metricReader.close() diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ReplayPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ReplayPortfolio.kt index b6d3b30c..17ec48d4 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ReplayPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ReplayPortfolio.kt @@ -22,6 +22,7 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -36,7 +37,7 @@ public class ReplayPortfolio : Portfolio("replay") { ) override val workload: Workload by anyOf( - Workload("solvinity", 1.0) + Workload("solvinity", trace("solvinity")) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/TestPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/TestPortfolio.kt index 90840db8..98eb989d 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/TestPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/TestPortfolio.kt @@ -22,6 +22,7 @@ package org.opendc.experiments.capelin +import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload @@ -36,7 +37,7 @@ public class TestPortfolio : Portfolio("test") { ) override val workload: Workload by anyOf( - Workload("solvinity", 1.0) + Workload("solvinity", trace("solvinity")) ) override val operationalPhenomena: OperationalPhenomena by anyOf( diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt index c4ddd158..a2e71243 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt @@ -22,23 +22,12 @@ package org.opendc.experiments.capelin.model -public enum class SamplingStrategy { - REGULAR, - HPC, - HPC_LOAD -} +import org.opendc.compute.workload.ComputeWorkload /** - * A workload that is considered for a scenario. - */ -public open class Workload( - public open val name: String, - public val fraction: Double, - public val samplingStrategy: SamplingStrategy = SamplingStrategy.REGULAR -) - -/** - * A workload that is composed of multiple workloads. + * A single workload originating from a trace. + * + * @param name the name of the workload. + * @param source The source of the workload data. */ -public class CompositeWorkload(override val name: String, public val workloads: List, public val totalLoad: Double) : - Workload(name, -1.0) +data class Workload(val name: String, val source: ComputeWorkload) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt deleted file mode 100644 index 498636ba..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import org.opendc.compute.workload.trace.RawParquetTraceReader -import org.opendc.compute.workload.trace.TraceEntry -import org.opendc.compute.workload.trace.TraceReader -import org.opendc.experiments.capelin.model.CompositeWorkload -import org.opendc.experiments.capelin.model.Workload -import org.opendc.simulator.compute.workload.SimWorkload - -/** - * A [TraceReader] for the internal VM workload trace format. - * - * @param rawReaders The internal raw trace readers to use. - * @param workload The workload to read. - * @param seed The seed to use for sampling. - */ -public class ParquetTraceReader( - rawReaders: List, - workload: Workload, - seed: Int -) : TraceReader { - /** - * The iterator over the actual trace. - */ - private val iterator: Iterator> = - rawReaders - .map { it.read() } - .run { - if (workload is CompositeWorkload) { - this.zip(workload.workloads) - } else { - this.zip(listOf(workload)) - } - } - .flatMap { - sampleWorkload(it.first, workload, it.second, seed) - .sortedBy(TraceEntry::start) - } - .iterator() - - override fun hasNext(): Boolean = iterator.hasNext() - - override fun next(): TraceEntry = iterator.next() - - override fun close() {} -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt deleted file mode 100644 index b42951df..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin.trace - -import mu.KotlinLogging -import org.opendc.compute.workload.trace.TraceEntry -import org.opendc.experiments.capelin.model.CompositeWorkload -import org.opendc.experiments.capelin.model.SamplingStrategy -import org.opendc.experiments.capelin.model.Workload -import org.opendc.simulator.compute.workload.SimWorkload -import java.util.* -import kotlin.random.Random - -private val logger = KotlinLogging.logger {} - -/** - * Sample the workload for the specified [run]. - */ -public fun sampleWorkload( - trace: List>, - workload: Workload, - subWorkload: Workload, - seed: Int -): List> { - return when { - workload is CompositeWorkload -> sampleRegularWorkload(trace, workload, subWorkload, seed) - workload.samplingStrategy == SamplingStrategy.HPC -> - sampleHpcWorkload(trace, workload, seed, sampleOnLoad = false) - workload.samplingStrategy == SamplingStrategy.HPC_LOAD -> - sampleHpcWorkload(trace, workload, seed, sampleOnLoad = true) - else -> - sampleRegularWorkload(trace, workload, workload, seed) - } -} - -/** - * Sample a regular (non-HPC) workload. - */ -public fun sampleRegularWorkload( - trace: List>, - workload: Workload, - subWorkload: Workload, - seed: Int -): List> { - val fraction = subWorkload.fraction - - val shuffled = trace.shuffled(Random(seed)) - val res = mutableListOf>() - val totalLoad = if (workload is CompositeWorkload) { - workload.totalLoad - } else { - shuffled.sumOf { it.meta.getValue("total-load") as Double } - } - var currentLoad = 0.0 - - for (entry in shuffled) { - val entryLoad = entry.meta.getValue("total-load") as Double - if ((currentLoad + entryLoad) / totalLoad > fraction) { - break - } - - currentLoad += entryLoad - res += entry - } - - logger.info { "Sampled ${trace.size} VMs (fraction $fraction) into subset of ${res.size} VMs" } - - return res -} - -/** - * Sample a HPC workload. - */ -public fun sampleHpcWorkload( - trace: List>, - workload: Workload, - seed: Int, - sampleOnLoad: Boolean -): List> { - val pattern = Regex("^vm__workload__(ComputeNode|cn).*") - val random = Random(seed) - - val fraction = workload.fraction - val (hpc, nonHpc) = trace.partition { entry -> - val name = entry.name - name.matches(pattern) - } - - val hpcSequence = generateSequence(0) { it + 1 } - .map { index -> - val res = mutableListOf>() - hpc.mapTo(res) { sample(it, index) } - res.shuffle(random) - res - } - .flatten() - - val nonHpcSequence = generateSequence(0) { it + 1 } - .map { index -> - val res = mutableListOf>() - nonHpc.mapTo(res) { sample(it, index) } - res.shuffle(random) - res - } - .flatten() - - logger.debug { "Found ${hpc.size} HPC workloads and ${nonHpc.size} non-HPC workloads" } - - val totalLoad = if (workload is CompositeWorkload) { - workload.totalLoad - } else { - trace.sumOf { it.meta.getValue("total-load") as Double } - } - - logger.debug { "Total trace load: $totalLoad" } - var hpcCount = 0 - var hpcLoad = 0.0 - var nonHpcCount = 0 - var nonHpcLoad = 0.0 - - val res = mutableListOf>() - - if (sampleOnLoad) { - var currentLoad = 0.0 - for (entry in hpcSequence) { - val entryLoad = entry.meta.getValue("total-load") as Double - if ((currentLoad + entryLoad) / totalLoad > fraction) { - break - } - - hpcLoad += entryLoad - hpcCount += 1 - currentLoad += entryLoad - res += entry - } - - for (entry in nonHpcSequence) { - val entryLoad = entry.meta.getValue("total-load") as Double - if ((currentLoad + entryLoad) / totalLoad > 1) { - break - } - - nonHpcLoad += entryLoad - nonHpcCount += 1 - currentLoad += entryLoad - res += entry - } - } else { - hpcSequence - .take((fraction * trace.size).toInt()) - .forEach { entry -> - hpcLoad += entry.meta.getValue("total-load") as Double - hpcCount += 1 - res.add(entry) - } - - nonHpcSequence - .take(((1 - fraction) * trace.size).toInt()) - .forEach { entry -> - nonHpcLoad += entry.meta.getValue("total-load") as Double - nonHpcCount += 1 - res.add(entry) - } - } - - logger.debug { "HPC $hpcCount (load $hpcLoad) and non-HPC $nonHpcCount (load $nonHpcLoad)" } - logger.debug { "Total sampled load: ${hpcLoad + nonHpcLoad}" } - logger.info { "Sampled ${trace.size} VMs (fraction $fraction) into subset of ${res.size} VMs" } - - return res -} - -/** - * Sample a random trace entry. - */ -private fun sample(entry: TraceEntry, i: Int): TraceEntry { - val uid = UUID.nameUUIDFromBytes("${entry.uid}-$i".toByteArray()) - return entry.copy(uid = uid) -} diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index c1386bfe..140a84db 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -31,18 +31,12 @@ import org.opendc.compute.service.scheduler.filters.ComputeFilter import org.opendc.compute.service.scheduler.filters.RamFilter import org.opendc.compute.service.scheduler.filters.VCpuFilter import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.workload.ComputeWorkloadRunner -import org.opendc.compute.workload.grid5000 +import org.opendc.compute.workload.* import org.opendc.compute.workload.topology.Topology import org.opendc.compute.workload.topology.apply -import org.opendc.compute.workload.trace.RawParquetTraceReader -import org.opendc.compute.workload.trace.TraceReader import org.opendc.compute.workload.util.PerformanceInterferenceReader -import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.topology.clusterTopology -import org.opendc.experiments.capelin.trace.ParquetTraceReader import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.core.runBlockingSimulation import org.opendc.telemetry.compute.ComputeMetricExporter import org.opendc.telemetry.compute.ComputeMonitor @@ -67,6 +61,11 @@ class CapelinIntegrationTest { */ private lateinit var computeScheduler: FilterScheduler + /** + * The [ComputeWorkloadLoader] responsible for loading the traces. + */ + private lateinit var workloadLoader: ComputeWorkloadLoader + /** * Setup the experimental environment. */ @@ -77,6 +76,7 @@ class CapelinIntegrationTest { filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) ) + workloadLoader = ComputeWorkloadLoader(File("src/test/resources/trace")) } /** @@ -84,24 +84,24 @@ class CapelinIntegrationTest { */ @Test fun testLarge() = runBlockingSimulation { - val traceReader = createTestTraceReader() - val simulator = ComputeWorkloadRunner( + val workload = createTestWorkload(1.0) + val runner = ComputeWorkloadRunner( coroutineContext, clock, computeScheduler ) val topology = createTopology() - val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, runner.producers, ComputeMetricExporter(clock, monitor)) try { - simulator.apply(topology) - simulator.run(traceReader) + runner.apply(topology) + runner.run(workload, 0) } finally { - simulator.close() + runner.close() metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(clock.instant(), runner.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -117,11 +117,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(223856043, monitor.idleTime) { "Incorrect idle time" } }, - { assertEquals(66481557, monitor.activeTime) { "Incorrect active time" } }, - { assertEquals(360441, monitor.stealTime) { "Incorrect steal time" } }, + { assertEquals(221949826, monitor.idleTime) { "Incorrect idle time" } }, + { assertEquals(68421374, monitor.activeTime) { "Incorrect active time" } }, + { assertEquals(947010, monitor.stealTime) { "Incorrect steal time" } }, { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.418336360461193E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, + { assertEquals(5.783711298639437E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -131,7 +131,7 @@ class CapelinIntegrationTest { @Test fun testSmall() = runBlockingSimulation { val seed = 1 - val traceReader = createTestTraceReader(0.25, seed) + val workload = createTestWorkload(0.25, seed) val simulator = ComputeWorkloadRunner( coroutineContext, @@ -143,7 +143,7 @@ class CapelinIntegrationTest { try { simulator.apply(topology) - simulator.run(traceReader) + simulator.run(workload, seed.toLong()) } finally { simulator.close() metricReader.close() @@ -161,9 +161,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(9597804, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(11140596, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(326138, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(8545158, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(12195642, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(941038, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -173,9 +173,8 @@ class CapelinIntegrationTest { */ @Test fun testInterference() = runBlockingSimulation { - val seed = 1 - val traceReader = createTestTraceReader(0.25, seed) - + val seed = 0 + val workload = createTestWorkload(1.0, seed) val perfInterferenceInput = checkNotNull(CapelinIntegrationTest::class.java.getResourceAsStream("/bitbrains-perf-interference.json")) val performanceInterferenceModel = PerformanceInterferenceReader() @@ -193,7 +192,7 @@ class CapelinIntegrationTest { try { simulator.apply(topology) - simulator.run(traceReader) + simulator.run(workload, seed.toLong()) } finally { simulator.close() metricReader.close() @@ -211,10 +210,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(9597804, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(11140596, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(326138, monitor.stealTime) { "Steal time incorrect" } }, - { assertEquals(925305, monitor.lostTime) { "Lost time incorrect" } } + { assertEquals(8545158, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(12195642, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(941038, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(3378, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -228,15 +227,15 @@ class CapelinIntegrationTest { coroutineContext, clock, computeScheduler, - grid5000(Duration.ofDays(7), seed) + grid5000(Duration.ofDays(7)) ) val topology = createTopology("single") - val traceReader = createTestTraceReader(0.25, seed) + val workload = createTestWorkload(0.25, seed) val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) try { simulator.apply(topology) - simulator.run(traceReader) + simulator.run(workload, seed.toLong()) } finally { simulator.close() metricReader.close() @@ -254,23 +253,20 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(9836315, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(10902085, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(306249, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(8640140, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(12100660, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(939456, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, - { assertEquals(2540877457, monitor.uptime) { "Uptime incorrect" } } + { assertEquals(2559305056, monitor.uptime) { "Uptime incorrect" } } ) } /** * Obtain the trace reader for the test. */ - private fun createTestTraceReader(fraction: Double = 1.0, seed: Int = 0): TraceReader { - return ParquetTraceReader( - listOf(RawParquetTraceReader(File("src/test/resources/trace"))), - Workload("test", fraction), - seed - ) + private fun createTestWorkload(fraction: Double, seed: Int = 0): List { + val source = trace("bitbrains-small").sampleByLoad(fraction) + return source.resolve(workloadLoader, Random(seed.toLong())) } /** diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet new file mode 100644 index 00000000..ee76d38f Binary files /dev/null and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet differ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet new file mode 100644 index 00000000..9b1cde13 Binary files /dev/null and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet differ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet deleted file mode 100644 index ee76d38f..00000000 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/meta.parquet and /dev/null differ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet deleted file mode 100644 index 9b1cde13..00000000 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/trace.parquet and /dev/null differ diff --git a/opendc-experiments/opendc-experiments-radice/build.gradle.kts b/opendc-experiments/opendc-experiments-radice/build.gradle.kts deleted file mode 100644 index 0c716183..00000000 --- a/opendc-experiments/opendc-experiments-radice/build.gradle.kts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -description = "Experiments for the Risk Analysis work" - -/* Build configuration */ -plugins { - `experiment-conventions` - `testing-conventions` -} - -dependencies { - api(platform(projects.opendcPlatform)) - api(projects.opendcHarness.opendcHarnessApi) - implementation(projects.opendcFormat) - implementation(projects.opendcSimulator.opendcSimulatorCore) - implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcCompute.opendcComputeSimulator) - implementation(projects.opendcTelemetry.opendcTelemetrySdk) - - implementation(libs.kotlin.logging) - implementation(libs.config) - implementation(libs.progressbar) - implementation(libs.clikt) - - implementation(libs.parquet) - testImplementation(libs.log4j.slf4j) -} -- cgit v1.2.3 From 474044649a67cfcc857615b6a0f8387a2954abbd Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 16 Sep 2021 12:34:53 +0200 Subject: feat(trace): Update OpenDC VM trace format This change optimizes the OpenDC VM trace format by removing unnecessary columns as well as optimizing the writer settings. The new implementation still supports reading the old trace format in case users run OpenDC with older workload traces. --- .../experiments/capelin/CapelinIntegrationTest.kt | 30 ++++++++++----------- .../resources/trace/bitbrains-small/meta.parquet | Bin 2081 -> 2099 bytes .../resources/trace/bitbrains-small/trace.parquet | Bin 1647189 -> 1125930 bytes 3 files changed, 15 insertions(+), 15 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 140a84db..ac2ea646 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -117,11 +117,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(221949826, monitor.idleTime) { "Incorrect idle time" } }, - { assertEquals(68421374, monitor.activeTime) { "Incorrect active time" } }, - { assertEquals(947010, monitor.stealTime) { "Incorrect steal time" } }, + { assertEquals(223331032, monitor.idleTime) { "Incorrect idle time" } }, + { assertEquals(67006568, monitor.activeTime) { "Incorrect active time" } }, + { assertEquals(3159379, monitor.stealTime) { "Incorrect steal time" } }, { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.783711298639437E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, + { assertEquals(5.841120890240688E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -161,9 +161,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(8545158, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(12195642, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(941038, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(10998110, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(9740290, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(0, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -210,10 +210,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(8545158, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(12195642, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(941038, monitor.stealTime) { "Steal time incorrect" } }, - { assertEquals(3378, monitor.lostTime) { "Lost time incorrect" } } + { assertEquals(6013899, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(14724501, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(12530742, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(473394, monitor.lostTime) { "Lost time incorrect" } } ) } @@ -253,11 +253,11 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(8640140, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(12100660, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(939456, monitor.stealTime) { "Steal time incorrect" } }, + { assertEquals(11134319, monitor.idleTime) { "Idle time incorrect" } }, + { assertEquals(9604081, monitor.activeTime) { "Active time incorrect" } }, + { assertEquals(0, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, - { assertEquals(2559305056, monitor.uptime) { "Uptime incorrect" } } + { assertEquals(2559005056, monitor.uptime) { "Uptime incorrect" } } ) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet index ee76d38f..da6e5330 100644 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/meta.parquet differ diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet index 9b1cde13..fe0a254c 100644 Binary files a/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet and b/opendc-experiments/opendc-experiments-capelin/src/test/resources/trace/bitbrains-small/trace.parquet differ -- cgit v1.2.3 From 68ef3700ed2f69bcf0118bb69eda71e6b1f4d54f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 21 Sep 2021 11:34:34 +0200 Subject: feat(trace): Add support for writing traces This change adds a new API for writing traces in a trace format. Currently, writing is only supported by the OpenDC VM format, but over time the other formats will also have support for writing added. --- .../opendc-experiments-capelin/build.gradle.kts | 4 +- .../org/opendc/experiments/capelin/Portfolio.kt | 2 +- .../experiments/capelin/util/ComputeSchedulers.kt | 86 ---------------------- 3 files changed, 3 insertions(+), 89 deletions(-) delete mode 100644 opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 4bcbaf61..23a3e4a7 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -33,8 +33,6 @@ dependencies { api(projects.opendcHarness.opendcHarnessApi) api(projects.opendcCompute.opendcComputeWorkload) - implementation(projects.opendcTrace.opendcTraceParquet) - implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcCompute.opendcComputeSimulator) @@ -49,5 +47,7 @@ dependencies { implementation(kotlin("reflect")) implementation(libs.opentelemetry.semconv) + runtimeOnly(projects.opendcTrace.opendcTraceOpendc) + testImplementation(libs.log4j.slf4j) } diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 630b76c4..2201a6b4 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -26,6 +26,7 @@ import com.typesafe.config.ConfigFactory import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.ComputeWorkloadRunner +import org.opendc.compute.workload.createComputeScheduler import org.opendc.compute.workload.export.parquet.ParquetExportMonitor import org.opendc.compute.workload.grid5000 import org.opendc.compute.workload.topology.apply @@ -34,7 +35,6 @@ import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload import org.opendc.experiments.capelin.topology.clusterTopology -import org.opendc.experiments.capelin.util.createComputeScheduler import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt deleted file mode 100644 index 3b7c3f0f..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/util/ComputeSchedulers.kt +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("ComputeSchedulers") -package org.opendc.experiments.capelin.util - -import org.opendc.compute.service.scheduler.ComputeScheduler -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.ReplayScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.compute.service.scheduler.weights.InstanceCountWeigher -import org.opendc.compute.service.scheduler.weights.RamWeigher -import org.opendc.compute.service.scheduler.weights.VCpuWeigher -import java.util.* - -/** - * Create a [ComputeScheduler] for the experiment. - */ -fun createComputeScheduler(allocationPolicy: String, seeder: Random, vmPlacements: Map = emptyMap()): ComputeScheduler { - val cpuAllocationRatio = 16.0 - val ramAllocationRatio = 1.5 - return when (allocationPolicy) { - "mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = 1.0)) - ) - "mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(RamWeigher(multiplier = -1.0)) - ) - "core-mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) - "core-mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(CoreRamWeigher(multiplier = -1.0)) - ) - "active-servers" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = -1.0)) - ) - "active-servers-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(InstanceCountWeigher(multiplier = 1.0)) - ) - "provisioned-cores" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = 1.0)) - ) - "provisioned-cores-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = listOf(VCpuWeigher(cpuAllocationRatio, multiplier = -1.0)) - ) - "random" -> FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), - weighers = emptyList(), - subsetSize = Int.MAX_VALUE, - random = Random(seeder.nextLong()) - ) - "replay" -> ReplayScheduler(vmPlacements) - else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") - } -} -- cgit v1.2.3 From 30cd010f1f98262aa7f264bb3c3eb6028b8495c5 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 22 Sep 2021 12:43:01 +0200 Subject: refactor(telemetry): Do not require clock for ComputeMetricExporter This change drops the requirement for a clock parameter when constructing a ComputeMetricExporter, since it will now derive the timestamp from the recorded metrics. --- .../org/opendc/experiments/capelin/Portfolio.kt | 10 ++-- .../experiments/capelin/CapelinIntegrationTest.kt | 59 +++++++++++----------- 2 files changed, 33 insertions(+), 36 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 2201a6b4..21ff3ab0 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -27,7 +27,7 @@ import mu.KotlinLogging import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.ComputeWorkloadRunner import org.opendc.compute.workload.createComputeScheduler -import org.opendc.compute.workload.export.parquet.ParquetExportMonitor +import org.opendc.compute.workload.export.parquet.ParquetComputeMetricExporter import org.opendc.compute.workload.grid5000 import org.opendc.compute.workload.topology.apply import org.opendc.compute.workload.util.PerformanceInterferenceReader @@ -39,7 +39,6 @@ import org.opendc.harness.dsl.Experiment import org.opendc.harness.dsl.anyOf import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.core.runBlockingSimulation -import org.opendc.telemetry.compute.ComputeMetricExporter import org.opendc.telemetry.compute.collectServiceMetrics import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader import java.io.File @@ -120,12 +119,12 @@ abstract class Portfolio(name: String) : Experiment(name) { performanceInterferenceModel ) - val monitor = ParquetExportMonitor( + val exporter = ParquetComputeMetricExporter( File(config.getString("output-path")), "portfolio_id=$name/scenario_id=$id/run_id=$repeat", 4096 ) - val metricReader = CoroutineMetricReader(this, runner.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, runner.producers, exporter) val topology = clusterTopology(File(config.getString("env-path"), "${topology.name}.txt")) try { @@ -137,10 +136,9 @@ abstract class Portfolio(name: String) : Experiment(name) { } finally { runner.close() metricReader.close() - monitor.close() } - val monitorResults = collectServiceMetrics(clock.instant(), runner.producers[0]) + val monitorResults = collectServiceMetrics(runner.producers[0]) logger.debug { "Scheduler " + "Success=${monitorResults.attemptsSuccess} " + diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index ac2ea646..30cc1466 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -39,7 +39,6 @@ import org.opendc.experiments.capelin.topology.clusterTopology import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel import org.opendc.simulator.core.runBlockingSimulation import org.opendc.telemetry.compute.ComputeMetricExporter -import org.opendc.telemetry.compute.ComputeMonitor import org.opendc.telemetry.compute.collectServiceMetrics import org.opendc.telemetry.compute.table.HostData import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader @@ -54,7 +53,7 @@ class CapelinIntegrationTest { /** * The monitor used to keep track of the metrics. */ - private lateinit var monitor: TestExperimentReporter + private lateinit var exporter: TestComputeMetricExporter /** * The [FilterScheduler] to use for all experiments. @@ -71,7 +70,7 @@ class CapelinIntegrationTest { */ @BeforeEach fun setUp() { - monitor = TestExperimentReporter() + exporter = TestComputeMetricExporter() computeScheduler = FilterScheduler( filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), weighers = listOf(CoreRamWeigher(multiplier = 1.0)) @@ -91,7 +90,7 @@ class CapelinIntegrationTest { computeScheduler ) val topology = createTopology() - val metricReader = CoroutineMetricReader(this, runner.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, runner.producers, exporter) try { runner.apply(topology) @@ -101,7 +100,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.instant(), runner.producers[0]) + val serviceMetrics = collectServiceMetrics(runner.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -117,11 +116,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(223331032, monitor.idleTime) { "Incorrect idle time" } }, - { assertEquals(67006568, monitor.activeTime) { "Incorrect active time" } }, - { assertEquals(3159379, monitor.stealTime) { "Incorrect steal time" } }, - { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.841120890240688E9, monitor.energyUsage, 0.01) { "Incorrect power draw" } }, + { assertEquals(223331032, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, + { assertEquals(67006568, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, + { assertEquals(3159379, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, + { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, + { assertEquals(5.841120890240688E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -139,7 +138,7 @@ class CapelinIntegrationTest { computeScheduler ) val topology = createTopology("single") - val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, simulator.producers, exporter) try { simulator.apply(topology) @@ -149,7 +148,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -161,10 +160,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(10998110, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(9740290, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(0, monitor.stealTime) { "Steal time incorrect" } }, - { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } } + { assertEquals(10998110, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9740290, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(0, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } @@ -188,7 +187,7 @@ class CapelinIntegrationTest { interferenceModel = performanceInterferenceModel ) val topology = createTopology("single") - val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, simulator.producers, exporter) try { simulator.apply(topology) @@ -198,7 +197,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -210,10 +209,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(6013899, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(14724501, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(12530742, monitor.stealTime) { "Steal time incorrect" } }, - { assertEquals(473394, monitor.lostTime) { "Lost time incorrect" } } + { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(473394, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } @@ -231,7 +230,7 @@ class CapelinIntegrationTest { ) val topology = createTopology("single") val workload = createTestWorkload(0.25, seed) - val metricReader = CoroutineMetricReader(this, simulator.producers, ComputeMetricExporter(clock, monitor)) + val metricReader = CoroutineMetricReader(this, simulator.producers, exporter) try { simulator.apply(topology) @@ -241,7 +240,7 @@ class CapelinIntegrationTest { metricReader.close() } - val serviceMetrics = collectServiceMetrics(clock.instant(), simulator.producers[0]) + val serviceMetrics = collectServiceMetrics(simulator.producers[0]) println( "Scheduler " + "Success=${serviceMetrics.attemptsSuccess} " + @@ -253,11 +252,11 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(11134319, monitor.idleTime) { "Idle time incorrect" } }, - { assertEquals(9604081, monitor.activeTime) { "Active time incorrect" } }, - { assertEquals(0, monitor.stealTime) { "Steal time incorrect" } }, - { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, - { assertEquals(2559005056, monitor.uptime) { "Uptime incorrect" } } + { assertEquals(11134319, exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9604081, exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(0, exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(0, exporter.lostTime) { "Lost time incorrect" } }, + { assertEquals(2559005056, exporter.uptime) { "Uptime incorrect" } } ) } @@ -277,7 +276,7 @@ class CapelinIntegrationTest { return stream.use { clusterTopology(stream) } } - class TestExperimentReporter : ComputeMonitor { + class TestComputeMetricExporter : ComputeMetricExporter() { var idleTime = 0L var activeTime = 0L var stealTime = 0L -- cgit v1.2.3 From d575bed5418be222e1d3ad39af862e2390596d61 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 26 Sep 2021 13:11:10 +0200 Subject: refactor(simulator): Combine work and deadline to duration This change removes the work and deadline properties from the SimResourceCommand.Consume class and introduces a new property duration. This property is now used in conjunction with the limit to compute the amount of work processed by a resource provider. Previously, we used both work and deadline to compute the duration and the amount of remaining work at the end of a consumption. However, with this change, we ensure that a resource consumption always runs at the same speed once establishing, drastically simplifying the computation for the amount of work processed during the consumption. --- .../opendc/experiments/capelin/CapelinIntegrationTest.kt | 6 +++--- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 30cc1466..ffc50ad8 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -118,7 +118,7 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, { assertEquals(223331032, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, { assertEquals(67006568, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, - { assertEquals(3159379, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, + { assertEquals(3088047, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, { assertEquals(5.841120890240688E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) @@ -211,8 +211,8 @@ class CapelinIntegrationTest { assertAll( { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(473394, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(12027839, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(477664, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 0873aac9..bfc5fc6f 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -41,6 +41,7 @@ import java.util.* import kotlin.coroutines.Continuation import kotlin.coroutines.CoroutineContext import kotlin.coroutines.resume +import kotlin.math.roundToLong /** * A [TFDevice] implementation using simulated components. @@ -127,13 +128,16 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext): SimResourceCommand { + override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): SimResourceCommand { + val consumedWork = ctx.speed * delta / 1000.0 + val activeWork = activeWork if (activeWork != null) { - if (activeWork.consume(activeWork.flops - ctx.remainingWork)) { + if (activeWork.consume(consumedWork)) { this.activeWork = null } else { - return SimResourceCommand.Consume(activeWork.flops, ctx.capacity) + val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() + return SimResourceCommand.Consume(ctx.capacity, duration) } } @@ -141,9 +145,10 @@ public class SimTFDevice( val head = queue.poll() return if (head != null) { this.activeWork = head - SimResourceCommand.Consume(head.flops, ctx.capacity) + val duration = (head.flops / ctx.capacity * 1000).roundToLong() + SimResourceCommand.Consume(ctx.capacity, duration) } else { - SimResourceCommand.Idle() + SimResourceCommand.Consume(0.0) } } -- cgit v1.2.3 From 02fa44c0b116ff51c4cbe2876d8b2a225ed68553 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 28 Sep 2021 11:58:19 +0200 Subject: refactor(simulator): Add support for pushing flow from context This change adds a new method to `SimResourceContext` called `push` which allows users to change the requested flow rate directly without having to interrupt the consumer. --- .../kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index bfc5fc6f..1fbd3b6a 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -128,7 +128,7 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): SimResourceCommand { + override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): Long { val consumedWork = ctx.speed * delta / 1000.0 val activeWork = activeWork @@ -137,7 +137,8 @@ public class SimTFDevice( this.activeWork = null } else { val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() - return SimResourceCommand.Consume(ctx.capacity, duration) + ctx.push(ctx.capacity) + return duration } } @@ -146,9 +147,11 @@ public class SimTFDevice( return if (head != null) { this.activeWork = head val duration = (head.flops / ctx.capacity * 1000).roundToLong() - SimResourceCommand.Consume(ctx.capacity, duration) + ctx.push(ctx.capacity) + duration } else { - SimResourceCommand.Consume(0.0) + ctx.push(0.0) + Long.MAX_VALUE } } -- cgit v1.2.3 From 657deac134f7b9ee30ed7e2b7667e30f3b17f79f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 29 Sep 2021 16:52:30 +0200 Subject: perf(simulator): Reduce memory allocations in SimResourceInterpreter This change removes unnecessary allocations in the SimResourceInterpreter caused by the way timers were allocated for the resource context. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index ffc50ad8..04413db5 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -212,7 +212,7 @@ class CapelinIntegrationTest { { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(12027839, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(477664, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(475891, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } -- cgit v1.2.3 From d031a70f8bea02a86df7840c5ce731185df86883 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 28 Sep 2021 22:10:12 +0200 Subject: refactor(simulator): Invoke consumer callback on every invalidation This change updates the simulator implementation to always invoke the `SimResourceConsumer.onNext` callback when the resource context is invalidated. This allows users to update the resource counter or do some other work if the context has changed. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 04413db5..12336308 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -212,7 +212,7 @@ class CapelinIntegrationTest { { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(12027839, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(475891, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(476163, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } -- cgit v1.2.3 From e07a5357013b92377a840b4d0d394d0ef6605b26 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 27 Sep 2021 14:22:02 +0200 Subject: refactor(simulator): Remove onUpdate callback This change removes the `onUpdate` callback from the `SimResourceProviderLogic` interface. Instead, users should now update counters using either `onConsume` or `onConverge`. --- .../org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 12336308..16085d82 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -118,7 +118,7 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, { assertEquals(223331032, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, { assertEquals(67006568, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, - { assertEquals(3088047, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, + { assertEquals(3159379, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, { assertEquals(5.841120890240688E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) @@ -211,7 +211,7 @@ class CapelinIntegrationTest { assertAll( { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(12027839, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(476163, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } @@ -252,8 +252,8 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(11134319, exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(9604081, exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(11132222, exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9606178, exporter.activeTime) { "Active time incorrect" } }, { assertEquals(0, exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(0, exporter.lostTime) { "Lost time incorrect" } }, { assertEquals(2559005056, exporter.uptime) { "Uptime incorrect" } } -- cgit v1.2.3 From d2f15fd7fd16922c11b0dcaa8807e8a321859773 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 29 Sep 2021 22:05:03 +0200 Subject: refactor(simulator): Merge distributor and aggregator into switch This change removes the distributor and aggregator interfaces in favour of a single switch interface. Since the switch interface is as powerful as both the distributor and aggregator, we don't need the latter two. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 16085d82..1bec2de5 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -212,7 +212,7 @@ class CapelinIntegrationTest { { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(476163, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(477279, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } -- cgit v1.2.3 From 4cc1d40d421c8736f8b21b360b61d6b065158b7a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 29 Sep 2021 23:56:16 +0200 Subject: refactor(simulator): Migrate to flow-based simulation This change renames the `opendc-simulator-resources` module into the `opendc-simulator-flow` module to indicate that the core simulation model of OpenDC is based around modelling and simulating flows. Previously, the distinction between resource consumer and provider, and input and output caused some confusion. By switching to a flow-based model, this distinction is now clear (as in, the water flows from source to consumer/sink). --- .../org/opendc/experiments/capelin/Portfolio.kt | 2 +- .../opendc/experiments/tf20/core/SimTFDevice.kt | 42 +++++++++++----------- .../opendc/experiments/tf20/distribute/Strategy.kt | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt index 21ff3ab0..4e855f82 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt @@ -131,7 +131,7 @@ abstract class Portfolio(name: String) : Experiment(name) { // Instantiate the desired topology runner.apply(topology) - // Run the workload trace + // Converge the workload trace runner.run(workload.source.resolve(workloadLoader, seeder), seeder.nextLong()) } finally { runner.close() diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 1fbd3b6a..2ba65e90 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -35,7 +35,7 @@ import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.* +import org.opendc.simulator.flow.* import java.time.Clock import java.util.* import kotlin.coroutines.Continuation @@ -65,7 +65,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - SimResourceInterpreter(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), + FlowEngine(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), SimplePowerDriver(powerModel) ) @@ -95,11 +95,11 @@ public class SimTFDevice( /** * The workload that will be run by the device. */ - private val workload = object : SimWorkload, SimResourceConsumer { + private val workload = object : SimWorkload, FlowSource { /** * The resource context to interrupt the workload with. */ - var ctx: SimResourceContext? = null + var ctx: FlowConnection? = null /** * The capacity of the device. @@ -128,16 +128,16 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): Long { - val consumedWork = ctx.speed * delta / 1000.0 + override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { + val consumedWork = conn.rate * delta / 1000.0 val activeWork = activeWork if (activeWork != null) { if (activeWork.consume(consumedWork)) { this.activeWork = null } else { - val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() - ctx.push(ctx.capacity) + val duration = (activeWork.flops / conn.capacity * 1000).roundToLong() + conn.push(conn.capacity) return duration } } @@ -146,27 +146,27 @@ public class SimTFDevice( val head = queue.poll() return if (head != null) { this.activeWork = head - val duration = (head.flops / ctx.capacity * 1000).roundToLong() - ctx.push(ctx.capacity) + val duration = (head.flops / conn.capacity * 1000).roundToLong() + conn.push(conn.capacity) duration } else { - ctx.push(0.0) + conn.push(0.0) Long.MAX_VALUE } } - override fun onEvent(ctx: SimResourceContext, event: SimResourceEvent) { + override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { when (event) { - SimResourceEvent.Start -> { - this.ctx = ctx - this.capacity = ctx.capacity + FlowEvent.Start -> { + this.ctx = conn + this.capacity = conn.capacity } - SimResourceEvent.Capacity -> { - this.capacity = ctx.capacity - ctx.interrupt() + FlowEvent.Capacity -> { + this.capacity = conn.capacity + conn.pull() } - SimResourceEvent.Run -> { - _usage.record(ctx.speed) + FlowEvent.Converge -> { + _usage.record(conn.rate) _power.record(machine.psu.powerDraw) } else -> {} @@ -188,7 +188,7 @@ public class SimTFDevice( override suspend fun compute(flops: Double) = suspendCancellableCoroutine { cont -> workload.queue.add(Work(flops, cont)) if (workload.isIdle) { - workload.ctx?.interrupt() + workload.ctx?.pull() } } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt index 5839c0df..3e755b56 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt @@ -27,7 +27,7 @@ package org.opendc.experiments.tf20.distribute */ public interface Strategy { /** - * Run the specified batch using the given strategy. + * Converge the specified batch using the given strategy. */ public suspend fun run(forward: Double, backward: Double, batchSize: Int) } -- cgit v1.2.3 From 7b2d03add3170b9142bf42c5a64aaa263773caf7 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 11:55:27 +0200 Subject: refactor(simulator): Separate push and pull flags This change separates the push and pull flags in FlowConsumerContextImpl, meaning that sources can now push directly without pulling and vice versa. --- .../experiments/capelin/CapelinIntegrationTest.kt | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 1bec2de5..67d39ffa 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -116,9 +116,9 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(223331032, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, - { assertEquals(67006568, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, - { assertEquals(3159379, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, + { assertEquals(223327751, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, + { assertEquals(67009849, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, + { assertEquals(3155964, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, { assertEquals(5.841120890240688E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) @@ -160,8 +160,8 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(10998110, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(9740290, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(10998184, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9740216, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) @@ -209,10 +209,10 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(6013899, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(14724501, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(477279, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(6009751, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(14728649, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(12526520, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(480866, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } @@ -252,9 +252,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(11132222, exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(9606178, exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(0, exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(11133606, exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9604794, exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(1311, exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(0, exporter.lostTime) { "Lost time incorrect" } }, { assertEquals(2559005056, exporter.uptime) { "Uptime incorrect" } } ) -- cgit v1.2.3 From 4f5a1f88d0c6aa19ce4cab0ec7b9b13a24c92fbe Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 14:45:42 +0200 Subject: refactor(simulator): Remove capacity event This change removes the Capacity entry from FlowEvent. Since the source is always pulled on a capacity change, we do not need a separate event for this. --- .../kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 2ba65e90..6f460ef7 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -131,6 +131,8 @@ public class SimTFDevice( override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { val consumedWork = conn.rate * delta / 1000.0 + capacity = conn.capacity + val activeWork = activeWork if (activeWork != null) { if (activeWork.consume(consumedWork)) { @@ -158,12 +160,8 @@ public class SimTFDevice( override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { when (event) { FlowEvent.Start -> { - this.ctx = conn - this.capacity = conn.capacity - } - FlowEvent.Capacity -> { - this.capacity = conn.capacity - conn.pull() + ctx = conn + capacity = conn.capacity } FlowEvent.Converge -> { _usage.record(conn.rate) -- cgit v1.2.3 From a2ce07026bf3ef17326e72f395dfa2dd9d9b17be Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 15:37:35 +0200 Subject: refactor(simulator): Create separate callbacks for remaining events This change creates separate callbacks for the remaining events: onStart, onStop and onConverge. --- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 6f460ef7..017bca59 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -128,6 +128,11 @@ public class SimTFDevice( } } + override fun onStart(conn: FlowConnection, now: Long) { + ctx = conn + capacity = conn.capacity + } + override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { val consumedWork = conn.rate * delta / 1000.0 @@ -157,18 +162,9 @@ public class SimTFDevice( } } - override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { - when (event) { - FlowEvent.Start -> { - ctx = conn - capacity = conn.capacity - } - FlowEvent.Converge -> { - _usage.record(conn.rate) - _power.record(machine.psu.powerDraw) - } - else -> {} - } + override fun onConverge(conn: FlowConnection, now: Long, delta: Long) { + _usage.record(conn.rate) + _power.record(machine.psu.powerDraw) } } -- cgit v1.2.3 From 559ac2327b8aa319fb8ab4558d4f4aa3382349f4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 16:27:45 +0200 Subject: perf(simulator): Make convergence callback optional This change adds two new properties for controlling whether the convergence callbacks of the source and consumer respectively should be invoked. This saves a lot of unnecessary calls for stages that do not have any implementation of the `onConvergence` method. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 2 ++ 1 file changed, 2 insertions(+) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 017bca59..fb36d2c7 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -131,6 +131,8 @@ public class SimTFDevice( override fun onStart(conn: FlowConnection, now: Long) { ctx = conn capacity = conn.capacity + + conn.shouldSourceConverge = true } override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { -- cgit v1.2.3 From 081221684fb826ab5a00c1d8cc5a9886b9e2203c Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 1 Oct 2021 22:04:35 +0200 Subject: feat(simulator): Expose CPU time counters directly on hypervisor This change adds a new interface to the SimHypervisor interface that exposes the CPU time counters directly. These are derived from the flow counters and will be used by SimHost to expose them via telemetry. --- .../experiments/capelin/CapelinIntegrationTest.kt | 27 +++++++++++----------- 1 file changed, 14 insertions(+), 13 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 67d39ffa..9d540118 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -116,11 +116,11 @@ class CapelinIntegrationTest { { assertEquals(0, serviceMetrics.serversActive, "All VMs should finish after a run") }, { assertEquals(0, serviceMetrics.attemptsFailure, "No VM should be unscheduled") }, { assertEquals(0, serviceMetrics.serversPending, "No VM should not be in the queue") }, - { assertEquals(223327751, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, - { assertEquals(67009849, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, - { assertEquals(3155964, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, + { assertEquals(223325655, this@CapelinIntegrationTest.exporter.idleTime) { "Incorrect idle time" } }, + { assertEquals(67006560, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, + { assertEquals(3159377, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.841120890240688E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, + { assertEquals(5.840207707767459E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -160,10 +160,11 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(10998184, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(9740216, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(10997726, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9740289, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } }, + { assertEquals(7.009945802750012E8, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } } ) } @@ -209,9 +210,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(6009751, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(14728649, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(12526520, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(6013515, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(14724500, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(480866, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } @@ -252,9 +253,9 @@ class CapelinIntegrationTest { // Note that these values have been verified beforehand assertAll( - { assertEquals(11133606, exporter.idleTime) { "Idle time incorrect" } }, - { assertEquals(9604794, exporter.activeTime) { "Active time incorrect" } }, - { assertEquals(1311, exporter.stealTime) { "Steal time incorrect" } }, + { assertEquals(10865478, exporter.idleTime) { "Idle time incorrect" } }, + { assertEquals(9606177, exporter.activeTime) { "Active time incorrect" } }, + { assertEquals(0, exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(0, exporter.lostTime) { "Lost time incorrect" } }, { assertEquals(2559005056, exporter.uptime) { "Uptime incorrect" } } ) -- cgit v1.2.3 From 557797c63c19e80c908eccc96472f215eab0c2f3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 4 Oct 2021 19:26:29 +0200 Subject: perf(experiments): Add benchmark for Capelin experiment --- .../opendc-experiments-capelin/build.gradle.kts | 1 + .../experiments/capelin/CapelinBenchmarks.kt | 83 ++++++++++++++++++++++ .../src/jmh/resources/log4j2.xml | 37 ++++++++++ 3 files changed, 121 insertions(+) create mode 100644 opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt create mode 100644 opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 23a3e4a7..c20556b5 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -26,6 +26,7 @@ description = "Experiments for the Capelin work" plugins { `experiment-conventions` `testing-conventions` + `benchmark-conventions` } dependencies { diff --git a/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt b/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt new file mode 100644 index 00000000..48a90985 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.capelin + +import kotlinx.coroutines.ExperimentalCoroutinesApi +import org.opendc.compute.service.scheduler.FilterScheduler +import org.opendc.compute.service.scheduler.filters.ComputeFilter +import org.opendc.compute.service.scheduler.filters.RamFilter +import org.opendc.compute.service.scheduler.filters.VCpuFilter +import org.opendc.compute.service.scheduler.weights.CoreRamWeigher +import org.opendc.compute.workload.* +import org.opendc.compute.workload.topology.Topology +import org.opendc.compute.workload.topology.apply +import org.opendc.experiments.capelin.topology.clusterTopology +import org.opendc.simulator.core.runBlockingSimulation +import org.openjdk.jmh.annotations.* +import java.io.File +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * Benchmark suite for the Capelin experiments. + */ +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@OptIn(ExperimentalCoroutinesApi::class) +class CapelinBenchmarks { + private lateinit var vms: List + private lateinit var topology: Topology + + @Param("true", "false") + private var isOptimized: Boolean = false + + @Setup + fun setUp() { + val loader = ComputeWorkloadLoader(File("src/test/resources/trace")) + val source = trace("bitbrains-small") + vms = source.resolve(loader, Random(1L)) + topology = checkNotNull(object {}.javaClass.getResourceAsStream("/env/topology.txt")).use { clusterTopology(it) } + } + + @Benchmark + fun benchmarkCapelin() = runBlockingSimulation { + val computeScheduler = FilterScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), + weighers = listOf(CoreRamWeigher(multiplier = 1.0)) + ) + val runner = ComputeWorkloadRunner( + coroutineContext, + clock, + computeScheduler + ) + + try { + runner.apply(topology, isOptimized) + runner.run(vms, 0) + } finally { + runner.close() + } + } +} diff --git a/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml b/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml new file mode 100644 index 00000000..c496dd75 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + -- cgit v1.2.3 From 7c260ab0b083488b8855f61648548a40401cf62e Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 4 Oct 2021 21:50:38 +0200 Subject: perf(simulator): Manage deadlines centrally in max min mux This change updates the MaxMinFlowMultiplexer implementation to centrally manage the deadlines of the `FlowSource`s as opposed to each source using its own timers. For large amounts of inputs, this is much faster as the multiplexer already needs to traverse each input on a timer expiration of an input. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 9d540118..337d68bf 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -120,7 +120,7 @@ class CapelinIntegrationTest { { assertEquals(67006560, this@CapelinIntegrationTest.exporter.activeTime) { "Incorrect active time" } }, { assertEquals(3159377, this@CapelinIntegrationTest.exporter.stealTime) { "Incorrect steal time" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.840207707767459E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, + { assertEquals(5.840212485920686E9, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } }, ) } @@ -164,7 +164,7 @@ class CapelinIntegrationTest { { assertEquals(9740289, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, { assertEquals(0, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } }, - { assertEquals(7.009945802750012E8, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } } + { assertEquals(7.0099453912813E8, this@CapelinIntegrationTest.exporter.energyUsage, 0.01) { "Incorrect power draw" } } ) } @@ -213,7 +213,7 @@ class CapelinIntegrationTest { { assertEquals(6013515, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724500, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(480866, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(481270, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } -- cgit v1.2.3 From 94fa3d33d4ef77aca5e70cc7f91ae9dca71d25e7 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 6 Oct 2021 13:12:18 +0200 Subject: perf(simulator): Optimize SimTraceWorkload This change improves the performance of the SimTraceWorkload class by changing the way trace fragments are read and processed by the CPU consumers. --- .../org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt index a119a219..bbe130e3 100644 --- a/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt +++ b/opendc-experiments/opendc-experiments-serverless20/src/main/kotlin/org/opendc/experiments/serverless/trace/FunctionTraceWorkload.kt @@ -23,12 +23,15 @@ package org.opendc.experiments.serverless.trace import org.opendc.faas.simulator.workload.SimFaaSWorkload +import org.opendc.simulator.compute.workload.SimTrace +import org.opendc.simulator.compute.workload.SimTraceFragment import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.compute.workload.SimWorkload /** * A [SimFaaSWorkload] for a [FunctionTrace]. */ -public class FunctionTraceWorkload(trace: FunctionTrace) : SimFaaSWorkload, SimWorkload by SimTraceWorkload(trace.samples.asSequence().map { SimTraceWorkload.Fragment(it.timestamp, it.duration, it.cpuUsage, 1) }) { +class FunctionTraceWorkload(trace: FunctionTrace) : + SimFaaSWorkload, SimWorkload by SimTraceWorkload(SimTrace.ofFragments(trace.samples.map { SimTraceFragment(it.timestamp, it.duration, it.cpuUsage, 1) })) { override suspend fun invoke() {} } -- cgit v1.2.3 From a0340a8752c4c4ed8413944b1dfb81b9481b6556 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 6 Oct 2021 14:29:31 +0200 Subject: perf(simulator): Skip fair-share algorithm if capacity remaining This change updates the MaxMinFlowMultiplexer implementation to skip the fair-share algorithm in case the total demand is lower than the available capacity. In this case, no re-division of capacity is necessary. --- .../kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments') diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 337d68bf..e34c5bdc 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -213,7 +213,7 @@ class CapelinIntegrationTest { { assertEquals(6013515, this@CapelinIntegrationTest.exporter.idleTime) { "Idle time incorrect" } }, { assertEquals(14724500, this@CapelinIntegrationTest.exporter.activeTime) { "Active time incorrect" } }, { assertEquals(12530742, this@CapelinIntegrationTest.exporter.stealTime) { "Steal time incorrect" } }, - { assertEquals(481270, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } + { assertEquals(481251, this@CapelinIntegrationTest.exporter.lostTime) { "Lost time incorrect" } } ) } -- cgit v1.2.3