From 6752b6d50faab447b3edc13bddf14f53401392f1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 2 Jul 2021 17:52:12 +0200 Subject: runner: Use public API for scheduling simulation jobs This change updates the web runner to not require direct database access for scheduling simulation jobs. Instead, the runner polls the public REST API for available jobs and reports its results through there. --- opendc-web/opendc-web-runner/build.gradle.kts | 14 +- .../src/main/kotlin/org/opendc/runner/web/Main.kt | 381 ------------------- .../org/opendc/runner/web/ScenarioManager.kt | 115 ------ .../kotlin/org/opendc/runner/web/TopologyParser.kt | 126 ------- .../org/opendc/runner/web/WebExperimentMonitor.kt | 189 ---------- .../main/kotlin/org/opendc/web/client/ApiClient.kt | 179 +++++++++ .../main/kotlin/org/opendc/web/client/ApiResult.kt | 43 +++ .../org/opendc/web/client/AuthConfiguration.kt | 32 ++ .../main/kotlin/org/opendc/web/client/model/Job.kt | 38 ++ .../kotlin/org/opendc/web/client/model/Machine.kt | 42 +++ .../org/opendc/web/client/model/MemoryUnit.kt | 37 ++ .../web/client/model/OperationalPhenomena.kt | 32 ++ .../org/opendc/web/client/model/Portfolio.kt | 38 ++ .../opendc/web/client/model/PortfolioTargets.kt | 28 ++ .../org/opendc/web/client/model/ProcessingUnit.kt | 37 ++ .../kotlin/org/opendc/web/client/model/Rack.kt | 39 ++ .../kotlin/org/opendc/web/client/model/Room.kt | 37 ++ .../kotlin/org/opendc/web/client/model/RoomTile.kt | 38 ++ .../kotlin/org/opendc/web/client/model/Scenario.kt | 39 ++ .../opendc/web/client/model/ScenarioTopology.kt | 28 ++ .../org/opendc/web/client/model/ScenarioTrace.kt | 28 ++ .../org/opendc/web/client/model/SimulationState.kt | 30 ++ .../kotlin/org/opendc/web/client/model/Topology.kt | 38 ++ .../src/main/kotlin/org/opendc/web/runner/Main.kt | 404 +++++++++++++++++++++ .../org/opendc/web/runner/ScenarioManager.kt | 86 +++++ .../org/opendc/web/runner/WebExperimentMonitor.kt | 189 ++++++++++ .../src/main/resources/log4j2.xml | 2 +- .../kotlin/org/opendc/web/client/ApiClientTest.kt | 264 ++++++++++++++ 28 files changed, 1735 insertions(+), 818 deletions(-) delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt delete mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt create mode 100644 opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt create mode 100644 opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt (limited to 'opendc-web') diff --git a/opendc-web/opendc-web-runner/build.gradle.kts b/opendc-web/opendc-web-runner/build.gradle.kts index f2b2ba23..1f705b79 100644 --- a/opendc-web/opendc-web-runner/build.gradle.kts +++ b/opendc-web/opendc-web-runner/build.gradle.kts @@ -25,11 +25,12 @@ description = "Experiment runner for OpenDC" /* Build configuration */ plugins { `kotlin-conventions` + `testing-conventions` application } application { - mainClass.set("org.opendc.runner.web.MainKt") + mainClass.set("org.opendc.web.runner.MainKt") } dependencies { @@ -42,12 +43,13 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.clikt) - implementation(libs.jackson.module.kotlin) { - exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") - } - implementation(kotlin("reflect")) implementation(libs.sentry.log4j2) - implementation(libs.mongodb) + implementation(libs.ktor.client.cio) + implementation(libs.ktor.client.auth) + implementation(libs.ktor.client.jackson) + implementation(libs.jackson.datatype.jsr310) runtimeOnly(libs.log4j.slf4j) + + testImplementation(libs.ktor.client.mock) } diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt deleted file mode 100644 index d0b97d90..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/Main.kt +++ /dev/null @@ -1,381 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.options.* -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.int -import com.github.ajalt.clikt.parameters.types.long -import com.mongodb.MongoClientSettings -import com.mongodb.MongoCredential -import com.mongodb.ServerAddress -import com.mongodb.client.MongoClients -import com.mongodb.client.MongoDatabase -import com.mongodb.client.model.Filters -import io.opentelemetry.api.metrics.MeterProvider -import io.opentelemetry.sdk.metrics.SdkMeterProvider -import io.opentelemetry.sdk.metrics.export.MetricProducer -import kotlinx.coroutines.* -import kotlinx.coroutines.channels.Channel -import mu.KotlinLogging -import org.bson.Document -import org.bson.types.ObjectId -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.weights.* -import org.opendc.experiments.capelin.* -import org.opendc.experiments.capelin.model.Workload -import org.opendc.experiments.capelin.trace.ParquetTraceReader -import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader -import org.opendc.experiments.capelin.trace.RawParquetTraceReader -import org.opendc.format.environment.EnvironmentReader -import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel -import org.opendc.simulator.core.runBlockingSimulation -import org.opendc.telemetry.sdk.toOtelClock -import java.io.File -import kotlin.random.Random -import kotlin.random.asJavaRandom - -private val logger = KotlinLogging.logger {} - -/** - * Represents the CLI command for starting the OpenDC web runner. - */ -@OptIn(ExperimentalCoroutinesApi::class) -class RunnerCli : CliktCommand(name = "runner") { - /** - * The name of the database to use. - */ - private val mongoDb by option( - "--mongo-db", - help = "name of the database to use", - envvar = "OPENDC_DB" - ) - .default("opendc") - - /** - * The database host to connect to. - */ - private val mongoHost by option( - "--mongo-host", - help = "database host to connect to", - envvar = "OPENDC_DB_HOST" - ) - .default("localhost") - - /** - * The database port to connect to. - */ - private val mongoPort by option( - "--mongo-port", - help = "database port to connect to", - envvar = "OPENDC_DB_PORT" - ) - .int() - .default(27017) - - /** - * The database user to connect with. - */ - private val mongoUser by option( - "--mongo-user", - help = "database user to connect with", - envvar = "OPENDC_DB_USER" - ) - .default("opendc") - - /** - * The database password to connect with. - */ - private val mongoPassword by option( - "--mongo-password", - help = "database password to connect with", - envvar = "OPENDC_DB_PASSWORD" - ) - .convert { it.toCharArray() } - .required() - - /** - * The path to the traces directory. - */ - private val tracePath by option( - "--traces", - help = "path to the directory containing the traces", - envvar = "OPENDC_TRACES" - ) - .file(canBeFile = false) - .defaultLazy { File("traces/") } - - /** - * The maximum duration of a single experiment run. - */ - private val runTimeout by option( - "--run-timeout", - help = "maximum duration of experiment in seconds", - envvar = "OPENDC_RUN_TIMEOUT" - ) - .long() - .default(60 * 3) // Experiment may run for a maximum of three minutes - - /** - * Connect to the user-specified database. - */ - private fun createDatabase(): MongoDatabase { - val credential = MongoCredential.createScramSha1Credential( - mongoUser, - mongoDb, - mongoPassword - ) - - val settings = MongoClientSettings.builder() - .credential(credential) - .applyToClusterSettings { it.hosts(listOf(ServerAddress(mongoHost, mongoPort))) } - .build() - val client = MongoClients.create(settings) - return client.getDatabase(mongoDb) - } - - /** - * Run a single scenario. - */ - private suspend fun runScenario(portfolio: Document, scenario: Document, topologyParser: TopologyParser): List { - val id = scenario.getObjectId("_id") - - logger.info { "Constructing performance interference model" } - - val traceDir = File( - tracePath, - scenario.getEmbedded(listOf("trace", "traceId"), String::class.java) - ) - val traceReader = RawParquetTraceReader(traceDir) - val interferenceGroups = let { - val path = File(traceDir, "performance-interference-model.json") - val operational = scenario.get("operational", Document::class.java) - val enabled = operational.getBoolean("performanceInterferenceEnabled") - - if (!enabled || !path.exists()) { - return@let null - } - - PerformanceInterferenceReader(path.inputStream()).use { reader -> reader.read() } - } - - val targets = portfolio.get("targets", Document::class.java) - val topologyId = scenario.getEmbedded(listOf("topology", "topologyId"), ObjectId::class.java) - val environment = topologyParser.read(topologyId) - - val results = (0 until targets.getInteger("repeatsPerScenario")).map { repeat -> - logger.info { "Starting repeat $repeat" } - withTimeout(runTimeout * 1000) { - val interferenceModel = interferenceGroups?.let { VmInterferenceModel(it, Random(repeat.toLong()).asJavaRandom()) } - runRepeat(scenario, repeat, environment, traceReader, interferenceModel) - } - } - - logger.info { "Finished simulation for scenario $id" } - - return results - } - - /** - * Run a single repeat. - */ - private suspend fun runRepeat( - scenario: Document, - repeat: Int, - environment: EnvironmentReader, - traceReader: RawParquetTraceReader, - interferenceModel: VmInterferenceModel? - ): WebExperimentMonitor.Result { - val monitor = WebExperimentMonitor() - - try { - runBlockingSimulation { - val seed = repeat - val traceDocument = scenario.get("trace", Document::class.java) - val workloadName = traceDocument.getString("traceId") - val workloadFraction = traceDocument.get("loadSamplingFraction", Number::class.java).toDouble() - - val seeder = Random(seed) - - val chan = Channel(Channel.CONFLATED) - - val meterProvider: MeterProvider = SdkMeterProvider - .builder() - .setClock(clock.toOtelClock()) - .build() - val metricProducer = meterProvider as MetricProducer - - val operational = scenario.get("operational", Document::class.java) - val allocationPolicy = - when (val policyName = operational.getString("schedulerName")) { - "mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to -1.0) - ) - "mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(MemoryWeigher() to 1.0) - ) - "core-mem" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to -1.0) - ) - "core-mem-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(CoreMemoryWeigher() to 1.0) - ) - "active-servers" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "active-servers-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(InstanceCountWeigher() to 1.0) - ) - "provisioned-cores" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to -1.0) - ) - "provisioned-cores-inv" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(ProvisionedCoresWeigher() to 1.0) - ) - "random" -> FilterScheduler( - filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), - weighers = listOf(RandomWeigher(java.util.Random(seeder.nextLong())) to 1.0) - ) - else -> throw IllegalArgumentException("Unknown policy $policyName") - } - - val trace = ParquetTraceReader( - listOf(traceReader), - Workload(workloadName, workloadFraction), - seed - ) - val failureFrequency = if (operational.getBoolean("failuresEnabled", false)) 24.0 * 7 else 0.0 - - withComputeService(clock, meterProvider, environment, allocationPolicy, interferenceModel) { scheduler -> - val failureDomain = if (failureFrequency > 0) { - logger.debug { "ENABLING failures" } - createFailureDomain( - this, - clock, - seeder.nextInt(), - failureFrequency, - scheduler, - chan - ) - } else { - null - } - - withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { - processTrace( - clock, - trace, - scheduler, - chan, - monitor - ) - } - - failureDomain?.cancel() - } - - val monitorResults = collectMetrics(metricProducer) - logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } - } - } catch (cause: Throwable) { - logger.warn(cause) { "Experiment failed" } - } - - return monitor.getResult() - } - - private val POLL_INTERVAL = 5000L // ms = 5 s - private val HEARTBEAT_INTERVAL = 60000L // ms = 1 min - - override fun run(): Unit = runBlocking(Dispatchers.Default) { - logger.info { "Starting OpenDC web runner" } - logger.info { "Connecting to MongoDB instance" } - val database = createDatabase() - val manager = ScenarioManager(database.getCollection("scenarios")) - val portfolios = database.getCollection("portfolios") - val topologies = database.getCollection("topologies") - val topologyParser = TopologyParser(topologies) - - logger.info { "Watching for queued scenarios" } - - while (true) { - val scenario = manager.findNext() - - if (scenario == null) { - delay(POLL_INTERVAL) - continue - } - - val id = scenario.getObjectId("_id") - - logger.info { "Found queued scenario $id: attempting to claim" } - - if (!manager.claim(id)) { - logger.info { "Failed to claim scenario" } - continue - } - - coroutineScope { - // Launch heartbeat process - val heartbeat = launch { - while (true) { - delay(HEARTBEAT_INTERVAL) - manager.heartbeat(id) - } - } - - try { - val portfolio = portfolios.find(Filters.eq("_id", scenario.getObjectId("portfolioId"))).first()!! - val results = runScenario(portfolio, scenario, topologyParser) - - logger.info { "Writing results to database" } - - manager.finish(id, results) - - logger.info { "Successfully finished scenario $id" } - } catch (e: Exception) { - logger.error(e) { "Scenario failed to finish" } - manager.fail(id) - } finally { - heartbeat.cancel() - } - } - } - } -} - -/** - * Main entry point of the runner. - */ -fun main(args: Array): Unit = RunnerCli().main(args) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt deleted file mode 100644 index a3907051..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/ScenarioManager.kt +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Updates -import org.bson.Document -import org.bson.types.ObjectId -import java.time.Instant - -/** - * Manages the queue of scenarios that need to be processed. - */ -public class ScenarioManager(private val collection: MongoCollection) { - /** - * Find the next scenario that the simulator needs to process. - */ - public fun findNext(): Document? { - return collection - .find(Filters.eq("simulation.state", "QUEUED")) - .first() - } - - /** - * Claim the scenario in the database with the specified id. - */ - public fun claim(id: ObjectId): Boolean { - val res = collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "QUEUED") - ), - Updates.combine( - Updates.set("simulation.state", "RUNNING"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - return res != null - } - - /** - * Update the heartbeat of the specified scenario. - */ - public fun heartbeat(id: ObjectId) { - collection.findOneAndUpdate( - Filters.and( - Filters.eq("_id", id), - Filters.eq("simulation.state", "RUNNING") - ), - Updates.set("simulation.heartbeat", Instant.now()) - ) - } - - /** - * Mark the scenario as failed. - */ - public fun fail(id: ObjectId) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FAILED"), - Updates.set("simulation.heartbeat", Instant.now()) - ) - ) - } - - /** - * Persist the specified results. - */ - public fun finish(id: ObjectId, results: List) { - collection.findOneAndUpdate( - Filters.eq("_id", id), - Updates.combine( - Updates.set("simulation.state", "FINISHED"), - Updates.unset("simulation.time"), - Updates.set("results.total_requested_burst", results.map { it.totalRequestedBurst }), - Updates.set("results.total_granted_burst", results.map { it.totalGrantedBurst }), - Updates.set("results.total_overcommitted_burst", results.map { it.totalOvercommittedBurst }), - Updates.set("results.total_interfered_burst", results.map { it.totalInterferedBurst }), - Updates.set("results.mean_cpu_usage", results.map { it.meanCpuUsage }), - Updates.set("results.mean_cpu_demand", results.map { it.meanCpuDemand }), - Updates.set("results.mean_num_deployed_images", results.map { it.meanNumDeployedImages }), - Updates.set("results.max_num_deployed_images", results.map { it.maxNumDeployedImages }), - Updates.set("results.total_power_draw", results.map { it.totalPowerDraw }), - Updates.set("results.total_failure_slices", results.map { it.totalFailureSlices }), - Updates.set("results.total_failure_vm_slices", results.map { it.totalFailureVmSlices }), - Updates.set("results.total_vms_submitted", results.map { it.totalVmsSubmitted }), - Updates.set("results.total_vms_queued", results.map { it.totalVmsQueued }), - Updates.set("results.total_vms_finished", results.map { it.totalVmsFinished }), - Updates.set("results.total_vms_failed", results.map { it.totalVmsFailed }) - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt deleted file mode 100644 index 2135ee1d..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import com.mongodb.client.AggregateIterable -import com.mongodb.client.MongoCollection -import com.mongodb.client.model.Aggregates -import com.mongodb.client.model.Field -import com.mongodb.client.model.Filters -import com.mongodb.client.model.Projections -import org.bson.Document -import org.bson.types.ObjectId -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef -import org.opendc.simulator.compute.model.MachineModel -import org.opendc.simulator.compute.model.MemoryUnit -import org.opendc.simulator.compute.model.ProcessingNode -import org.opendc.simulator.compute.model.ProcessingUnit -import org.opendc.simulator.compute.power.LinearPowerModel -import java.util.* - -/** - * A helper class that converts the MongoDB topology into an OpenDC environment. - */ -public class TopologyParser(private val collection: MongoCollection) { - - /** - * Parse the topology from the specified [id]. - */ - public fun read(id: ObjectId): EnvironmentReader { - val nodes = mutableListOf() - val random = Random(0) - - for (machine in fetchMachines(id)) { - val clusterId = machine.get("rack_id").toString() - val position = machine.getInteger("position") - - val processors = machine.getList("cpus", Document::class.java).flatMap { cpu -> - val cores = cpu.getInteger("numberOfCores") - val speed = cpu.get("clockRateMhz", Number::class.java).toDouble() - // TODO Remove hardcoding of vendor - val node = ProcessingNode("Intel", "amd64", cpu.getString("name"), cores) - List(cores) { coreId -> - ProcessingUnit(node, coreId, speed) - } - } - val memoryUnits = machine.getList("memories", Document::class.java).map { memory -> - MemoryUnit( - "Samsung", - memory.getString("name"), - memory.get("speedMbPerS", Number::class.java).toDouble(), - memory.get("sizeMb", Number::class.java).toLong() - ) - } - - val energyConsumptionW = machine.getList("cpus", Document::class.java).sumOf { it.getInteger("energyConsumptionW") }.toDouble() - - nodes.add( - MachineDef( - UUID(random.nextLong(), random.nextLong()), - "node-$clusterId-$position", - mapOf("cluster" to clusterId), - MachineModel(processors, memoryUnits), - LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) - ) - ) - } - - return object : EnvironmentReader { - override fun read(): List = nodes - override fun close() {} - } - } - - /** - * Fetch the metadata of the topology. - */ - private fun fetchName(id: ObjectId): String { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.include("name")) - ) - ) - .first()!! - .getString("name") - } - - /** - * Fetch a topology from the database with the specified [id]. - */ - private fun fetchMachines(id: ObjectId): AggregateIterable { - return collection.aggregate( - listOf( - Aggregates.match(Filters.eq("_id", id)), - Aggregates.project(Projections.fields(Document("racks", "\$rooms.tiles.rack"))), - Aggregates.unwind("\$racks"), - Aggregates.unwind("\$racks"), - Aggregates.replaceRoot("\$racks"), - Aggregates.addFields(Field("machines.rack_id", "\$_id")), - Aggregates.unwind("\$machines"), - Aggregates.replaceRoot("\$machines") - ) - ) - } -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt deleted file mode 100644 index c913f82f..00000000 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.runner.web - -import mu.KotlinLogging -import org.opendc.compute.api.Server -import org.opendc.compute.api.ServerState -import org.opendc.compute.service.driver.Host -import org.opendc.compute.service.driver.HostState -import org.opendc.experiments.capelin.monitor.ExperimentMonitor -import org.opendc.experiments.capelin.telemetry.HostEvent -import kotlin.math.max - -/** - * An [ExperimentMonitor] that tracks the aggregate metrics for each repeat. - */ -public class WebExperimentMonitor : ExperimentMonitor { - private val logger = KotlinLogging.logger {} - - override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} - - override fun reportHostStateChange(time: Long, host: Host, newState: HostState) { - logger.debug { "Host ${host.uid} changed state $newState [$time]" } - } - - override fun reportHostSlice( - time: Long, - requestedBurst: Long, - grantedBurst: Long, - overcommissionedBurst: Long, - interferedBurst: Long, - cpuUsage: Double, - cpuDemand: Double, - powerDraw: Double, - numberOfDeployedImages: Int, - host: Host, - ) { - processHostEvent( - HostEvent( - time, - 5 * 60 * 1000L, - host, - numberOfDeployedImages, - requestedBurst, - grantedBurst, - overcommissionedBurst, - interferedBurst, - cpuUsage, - cpuDemand, - powerDraw, - host.model.cpuCount - ) - ) - } - - private var hostAggregateMetrics: AggregateHostMetrics = AggregateHostMetrics() - private val hostMetrics: MutableMap = mutableMapOf() - - private fun processHostEvent(event: HostEvent) { - val slices = event.duration / SLICE_LENGTH - - hostAggregateMetrics = AggregateHostMetrics( - hostAggregateMetrics.totalRequestedBurst + event.requestedBurst, - hostAggregateMetrics.totalGrantedBurst + event.grantedBurst, - hostAggregateMetrics.totalOvercommittedBurst + event.overcommissionedBurst, - hostAggregateMetrics.totalInterferedBurst + event.interferedBurst, - hostAggregateMetrics.totalPowerDraw + (event.duration * event.powerDraw) / 3600, - hostAggregateMetrics.totalFailureSlices + if (event.host.state != HostState.UP) slices else 0, - hostAggregateMetrics.totalFailureVmSlices + if (event.host.state != HostState.UP) event.vmCount * slices else 0 - ) - - hostMetrics.compute(event.host) { _, prev -> - HostMetrics( - (event.cpuUsage.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuUsage ?: 0.0), - (event.cpuDemand.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuDemand ?: 0.0), - event.vmCount + (prev?.vmCount ?: 0), - 1 + (prev?.count ?: 0) - ) - } - } - - private val SLICE_LENGTH: Long = 5 * 60 * 1000 - - public data class AggregateHostMetrics( - val totalRequestedBurst: Long = 0, - val totalGrantedBurst: Long = 0, - val totalOvercommittedBurst: Long = 0, - val totalInterferedBurst: Long = 0, - val totalPowerDraw: Double = 0.0, - val totalFailureSlices: Long = 0, - val totalFailureVmSlices: Long = 0, - ) - - public data class HostMetrics( - val cpuUsage: Double, - val cpuDemand: Double, - val vmCount: Long, - val count: Long - ) - - private var provisionerMetrics: AggregateProvisionerMetrics = AggregateProvisionerMetrics() - - override fun reportProvisionerMetrics( - time: Long, - totalHostCount: Int, - availableHostCount: Int, - totalVmCount: Int, - activeVmCount: Int, - inactiveVmCount: Int, - waitingVmCount: Int, - failedVmCount: Int - ) { - provisionerMetrics = AggregateProvisionerMetrics( - max(totalVmCount, provisionerMetrics.vmTotalCount), - max(waitingVmCount, provisionerMetrics.vmWaitingCount), - max(activeVmCount, provisionerMetrics.vmActiveCount), - max(inactiveVmCount, provisionerMetrics.vmInactiveCount), - max(failedVmCount, provisionerMetrics.vmFailedCount), - ) - } - - public data class AggregateProvisionerMetrics( - val vmTotalCount: Int = 0, - val vmWaitingCount: Int = 0, - val vmActiveCount: Int = 0, - val vmInactiveCount: Int = 0, - val vmFailedCount: Int = 0 - ) - - override fun close() {} - - public fun getResult(): Result { - return Result( - hostAggregateMetrics.totalRequestedBurst, - hostAggregateMetrics.totalGrantedBurst, - hostAggregateMetrics.totalOvercommittedBurst, - hostAggregateMetrics.totalInterferedBurst, - hostMetrics.map { it.value.cpuUsage / it.value.count }.average(), - hostMetrics.map { it.value.cpuDemand / it.value.count }.average(), - hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.average(), - hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.maxOrNull() ?: 0.0, - hostAggregateMetrics.totalPowerDraw, - hostAggregateMetrics.totalFailureSlices, - hostAggregateMetrics.totalFailureVmSlices, - provisionerMetrics.vmTotalCount, - provisionerMetrics.vmWaitingCount, - provisionerMetrics.vmInactiveCount, - provisionerMetrics.vmFailedCount, - ) - } - - public data class Result( - public val totalRequestedBurst: Long, - public val totalGrantedBurst: Long, - public val totalOvercommittedBurst: Long, - public val totalInterferedBurst: Long, - public val meanCpuUsage: Double, - public val meanCpuDemand: Double, - public val meanNumDeployedImages: Double, - public val maxNumDeployedImages: Double, - public val totalPowerDraw: Double, - public val totalFailureSlices: Long, - public val totalFailureVmSlices: Long, - public val totalVmsSubmitted: Int, - public val totalVmsQueued: Int, - public val totalVmsFinished: Int, - public val totalVmsFailed: Int - ) -} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt new file mode 100644 index 00000000..9f2656c4 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiClient.kt @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import io.ktor.client.* +import io.ktor.client.features.auth.* +import io.ktor.client.features.auth.providers.* +import io.ktor.client.features.json.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import org.opendc.web.client.model.* +import java.net.URI + +/** + * Client implementation for the OpenDC REST API (version 2). + * + * @param baseUrl The base url of the API. + * @param auth The authentication configuration for the client. + * @param client The HTTP client to use. + */ +public class ApiClient( + private val baseUrl: URI, + private val auth: AuthConfiguration, + private val audience: String = "https://api.opendc.org/v2/", + client: HttpClient = HttpClient {} +) : AutoCloseable { + /** + * The Ktor [HttpClient] that is used to communicate with the REST API. + */ + private val client = client.config { + install(JsonFeature) { + serializer = JacksonSerializer { + registerModule(JavaTimeModule()) + } + } + install(Auth) { + bearer { + loadTokens { requestToken() } + refreshTokens { requestToken() } + } + } + expectSuccess = false + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getPortfolio(id: String): Portfolio? { + val url = URLBuilder(Url(baseUrl)) + .path("portfolios", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the scenario with the specified [id]. + */ + public suspend fun getScenario(id: String): Scenario? { + val url = URLBuilder(Url(baseUrl)) + .path("scenarios", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the topology with the specified [id]. + */ + public suspend fun getTopology(id: String): Topology? { + val url = URLBuilder(Url(baseUrl)) + .path("topologies", id) + .build() + return when (val result = client.get>(url)) { + is ApiResult.Success -> result.data + else -> null + } + } + + /** + * Retrieve the available jobs. + */ + public suspend fun getJobs(): List { + val url = URLBuilder(Url(baseUrl)) + .path("jobs") + .build() + return when (val result = client.get>>(url)) { + is ApiResult.Success -> result.data + else -> emptyList() + } + } + + /** + * Update the specified job. + * + * @param id The identifier of the job. + * @param state The new state of the job. + * @param results The results of the job. + */ + public suspend fun updateJob(id: String, state: SimulationState, results: Map = emptyMap()): Boolean { + val url = URLBuilder(Url(baseUrl)) + .path("jobs", id) + .build() + + data class Request( + val state: SimulationState, + val results: Map + ) + + val res = client.post { + url(url) + contentType(ContentType.Application.Json) + body = Request(state, results) + } + return res.status.isSuccess() + } + + /** + * Request the auth token for the API. + */ + private suspend fun requestToken(): BearerTokens { + data class Request( + val audience: String, + @JsonProperty("grant_type") + val grantType: String, + @JsonProperty("client_id") + val clientId: String, + @JsonProperty("client_secret") + val clientSecret: String + ) + + data class Response( + @JsonProperty("access_token") + val accessToken: String, + @JsonProperty("token_type") + val tokenType: String, + val scope: String = "", + @JsonProperty("expires_in") + val expiresIn: Long + ) + + val result = client.post { + url(Url("https://${auth.domain}/oauth/token")) + contentType(ContentType.Application.Json) + body = Request(audience, "client_credentials", auth.clientId, auth.clientSecret) + } + + return BearerTokens(result.accessToken, "") + } + + override fun close() = client.close() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt new file mode 100644 index 00000000..a3df01c5 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/ApiResult.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import com.fasterxml.jackson.annotation.JsonSubTypes +import com.fasterxml.jackson.annotation.JsonTypeInfo + +/** + * Generic response model for the OpenDC API. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.DEDUCTION) +@JsonSubTypes(JsonSubTypes.Type(ApiResult.Success::class), JsonSubTypes.Type(ApiResult.Failure::class)) +public sealed class ApiResult { + /** + * A response indicating everything is okay. + */ + public data class Success(val data: T) : ApiResult() + + /** + * A response indicating a failure. + */ + public data class Failure(val message: String, val errors: List = emptyList()) : ApiResult() +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt new file mode 100644 index 00000000..5dbf2f59 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/AuthConfiguration.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +/** + * The authentication configuration for the API client. + */ +public data class AuthConfiguration( + val domain: String, + val clientId: String, + val clientSecret: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt new file mode 100644 index 00000000..eeb65e49 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Job.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty +import java.time.LocalDateTime + +/** + * A description of a simulation job. + */ +public data class Job( + @JsonProperty("_id") + val id: String, + val scenarioId: String, + val state: SimulationState, + val heartbeat: LocalDateTime, + val results: Map +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt new file mode 100644 index 00000000..c6757c5c --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Machine.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A machine in a rack. + */ +@JsonIgnoreProperties("id_legacy") +public data class Machine( + @JsonProperty("_id") + val id: String, + val position: Int, + val cpus: List = emptyList(), + val gpus: List = emptyList(), + @JsonProperty("memories") + val memory: List = emptyList(), + @JsonProperty("storages") + val storage: List = emptyList() +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt new file mode 100644 index 00000000..11e794e8 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/MemoryUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A memory unit in a system. + */ +public data class MemoryUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val speedMbPerS: Double, + val sizeMb: Double, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt new file mode 100644 index 00000000..ef5b4902 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/OperationalPhenomena.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * Object describing the enabled operational phenomena for a scenario. + */ +public data class OperationalPhenomena( + val failuresEnabled: Boolean, + val performanceInterferenceEnabled: Boolean, + val schedulerName: String +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt new file mode 100644 index 00000000..6904920b --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Portfolio.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A portfolio in OpenDC. + */ +public data class Portfolio( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + @JsonProperty("scenarioIds") + val scenarios: Set, + val targets: PortfolioTargets +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt new file mode 100644 index 00000000..07c11c19 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/PortfolioTargets.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The targets of a portfolio. + */ +public data class PortfolioTargets(val enabledMetrics: Set, val repeatsPerScenario: Int) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt new file mode 100644 index 00000000..449b5c43 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ProcessingUnit.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A CPU model. + */ +public data class ProcessingUnit( + @JsonProperty("_id") + val id: String, + val name: String, + val clockRateMhz: Double, + val numberOfCores: Int, + val energyConsumptionW: Double +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt new file mode 100644 index 00000000..a0464388 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Rack.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A rack in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public class Rack( + @JsonProperty("_id") + val id: String, + val name: String, + val capacity: Int, + val powerCapacityW: Double, + val machines: List +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt new file mode 100644 index 00000000..e961d6db --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Room.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room in a datacenter. + */ +@JsonIgnoreProperties("id_legacy") +public data class Room( + @JsonProperty("_id") + val id: String, + val name: String, + val tiles: Set +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt new file mode 100644 index 00000000..3bee3204 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/RoomTile.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A room tile. + */ +@JsonIgnoreProperties("id_legacy") +public data class RoomTile( + @JsonProperty("_id") + val id: String, + val positionX: Double, + val positionY: Double, + val rack: Rack? = null +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt new file mode 100644 index 00000000..851ff980 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Scenario.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * A simulation scenario. + */ +public data class Scenario( + @JsonProperty("_id") + val id: String, + val portfolioId: String, + val name: String, + val trace: ScenarioTrace, + val topology: ScenarioTopology, + @JsonProperty("operational") + val operationalPhenomena: OperationalPhenomena +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt new file mode 100644 index 00000000..2b90f7ef --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTopology.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The topology details for a scenario. + */ +public data class ScenarioTopology(val topologyId: String) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt new file mode 100644 index 00000000..adff6d97 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/ScenarioTrace.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The trace details of a scenario. + */ +public data class ScenarioTrace(val traceId: String, val loadSamplingFraction: Double) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt new file mode 100644 index 00000000..2eadd747 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/SimulationState.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +/** + * The state of a simulation job. + */ +public enum class SimulationState { + QUEUED, CLAIMED, RUNNING, FINISHED, FAILED +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt new file mode 100644 index 00000000..b59aba42 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/client/model/Topology.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client.model + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty + +/** + * Model for an OpenDC topology. + */ +@JsonIgnoreProperties("id_legacy", "datacenter_id_legacy", "datetimeLastUpdated", "datetimeLastEdited") +public data class Topology( + @JsonProperty("_id") + val id: String, + val projectId: String, + val name: String, + val rooms: Set, +) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt new file mode 100644 index 00000000..5b5ef802 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/Main.kt @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.* +import com.github.ajalt.clikt.parameters.types.file +import com.github.ajalt.clikt.parameters.types.long +import io.opentelemetry.api.metrics.MeterProvider +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.export.MetricProducer +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import mu.KotlinLogging +import org.opendc.compute.service.scheduler.FilterScheduler +import org.opendc.compute.service.scheduler.filters.ComputeCapabilitiesFilter +import org.opendc.compute.service.scheduler.filters.ComputeFilter +import org.opendc.compute.service.scheduler.weights.* +import org.opendc.experiments.capelin.* +import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.capelin.trace.ParquetTraceReader +import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader +import org.opendc.experiments.capelin.trace.RawParquetTraceReader +import org.opendc.format.environment.EnvironmentReader +import org.opendc.format.environment.MachineDef +import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.model.MemoryUnit +import org.opendc.simulator.compute.model.ProcessingNode +import org.opendc.simulator.compute.model.ProcessingUnit +import org.opendc.simulator.compute.power.LinearPowerModel +import org.opendc.simulator.core.runBlockingSimulation +import org.opendc.telemetry.sdk.toOtelClock +import org.opendc.web.client.ApiClient +import org.opendc.web.client.AuthConfiguration +import org.opendc.web.client.model.Scenario +import org.opendc.web.client.model.Topology +import java.io.File +import java.net.URI +import java.util.* +import kotlin.random.Random +import kotlin.random.asJavaRandom +import org.opendc.web.client.model.Portfolio as ClientPortfolio + +private val logger = KotlinLogging.logger {} + +/** + * Represents the CLI command for starting the OpenDC web runner. + */ +class RunnerCli : CliktCommand(name = "runner") { + /** + * The URL to the OpenDC API. + */ + private val apiUrl by option( + "--api-url", + help = "url to the OpenDC API", + envvar = "OPENDC_API_URL" + ) + .convert { URI(it) } + .default(URI("https://api.opendc.org/v2")) + + /** + * The auth domain to use. + */ + private val authDomain by option( + "--auth-domain", + help = "auth domain of the OpenDC API", + envvar = "AUTH0_DOMAIN" + ) + .required() + + /** + * The auth client ID to use. + */ + private val authClientId by option( + "--auth-id", + help = "auth client id of the OpenDC API", + envvar = "AUTH0_CLIENT_ID" + ) + .required() + + /** + * The auth client secret to use. + */ + private val authClientSecret by option( + "--auth-secret", + help = "auth client secret of the OpenDC API", + envvar = "AUTH0_CLIENT_SECRET" + ) + .required() + + /** + * The path to the traces directory. + */ + private val tracePath by option( + "--traces", + help = "path to the directory containing the traces", + envvar = "OPENDC_TRACES" + ) + .file(canBeFile = false) + .defaultLazy { File("traces/") } + + /** + * The maximum duration of a single experiment run. + */ + private val runTimeout by option( + "--run-timeout", + help = "maximum duration of experiment in seconds", + envvar = "OPENDC_RUN_TIMEOUT" + ) + .long() + .default(60L * 3) // Experiment may run for a maximum of three minutes + + /** + * Run a single scenario. + */ + private suspend fun runScenario(portfolio: ClientPortfolio, scenario: Scenario, environment: EnvironmentReader): List { + val id = scenario.id + + logger.info { "Constructing performance interference model" } + + val traceDir = File( + tracePath, + scenario.trace.traceId + ) + val traceReader = RawParquetTraceReader(traceDir) + val interferenceGroups = let { + val path = File(traceDir, "performance-interference-model.json") + val operational = scenario.operationalPhenomena + val enabled = operational.performanceInterferenceEnabled + + if (!enabled || !path.exists()) { + return@let null + } + + PerformanceInterferenceReader(path.inputStream()).use { reader -> reader.read() } + } + + val targets = portfolio.targets + val results = (0 until targets.repeatsPerScenario).map { repeat -> + logger.info { "Starting repeat $repeat" } + withTimeout(runTimeout * 1000) { + val interferenceModel = interferenceGroups?.let { VmInterferenceModel(it, Random(repeat.toLong()).asJavaRandom()) } + runRepeat(scenario, repeat, environment, traceReader, interferenceModel) + } + } + + logger.info { "Finished simulation for scenario $id" } + + return results + } + + /** + * Run a single repeat. + */ + private suspend fun runRepeat( + scenario: Scenario, + repeat: Int, + environment: EnvironmentReader, + traceReader: RawParquetTraceReader, + interferenceModel: VmInterferenceModel? + ): WebExperimentMonitor.Result { + val monitor = WebExperimentMonitor() + + try { + runBlockingSimulation { + val seed = repeat + val workloadName = scenario.trace.traceId + val workloadFraction = scenario.trace.loadSamplingFraction + + val seeder = Random(seed) + + val chan = Channel(Channel.CONFLATED) + + val meterProvider: MeterProvider = SdkMeterProvider + .builder() + .setClock(clock.toOtelClock()) + .build() + val metricProducer = meterProvider as MetricProducer + + val operational = scenario.operationalPhenomena + val allocationPolicy = + when (val policyName = operational.schedulerName) { + "mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(MemoryWeigher() to -1.0) + ) + "mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(MemoryWeigher() to 1.0) + ) + "core-mem" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(CoreMemoryWeigher() to -1.0) + ) + "core-mem-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(CoreMemoryWeigher() to 1.0) + ) + "active-servers" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to -1.0) + ) + "active-servers-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(InstanceCountWeigher() to 1.0) + ) + "provisioned-cores" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to -1.0) + ) + "provisioned-cores-inv" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(ProvisionedCoresWeigher() to 1.0) + ) + "random" -> FilterScheduler( + filters = listOf(ComputeFilter(), ComputeCapabilitiesFilter()), + weighers = listOf(RandomWeigher(java.util.Random(seeder.nextLong())) to 1.0) + ) + else -> throw IllegalArgumentException("Unknown policy $policyName") + } + + val trace = ParquetTraceReader( + listOf(traceReader), + Workload(workloadName, workloadFraction), + seed + ) + val failureFrequency = if (operational.failuresEnabled) 24.0 * 7 else 0.0 + + withComputeService(clock, meterProvider, environment, allocationPolicy, interferenceModel) { scheduler -> + val failureDomain = if (failureFrequency > 0) { + logger.debug { "ENABLING failures" } + createFailureDomain( + this, + clock, + seeder.nextInt(), + failureFrequency, + scheduler, + chan + ) + } else { + null + } + + withMonitor(monitor, clock, meterProvider as MetricProducer, scheduler) { + processTrace( + clock, + trace, + scheduler, + chan, + monitor + ) + } + + failureDomain?.cancel() + } + + val monitorResults = collectMetrics(metricProducer) + logger.debug { "Finish SUBMIT=${monitorResults.submittedVms} FAIL=${monitorResults.unscheduledVms} QUEUE=${monitorResults.queuedVms} RUNNING=${monitorResults.runningVms}" } + } + } catch (cause: Throwable) { + logger.warn(cause) { "Experiment failed" } + } + + return monitor.getResult() + } + + private val POLL_INTERVAL = 30000L // ms = 30 s + private val HEARTBEAT_INTERVAL = 60000L // ms = 1 min + + override fun run(): Unit = runBlocking(Dispatchers.Default) { + logger.info { "Starting OpenDC web runner" } + + val client = ApiClient(baseUrl = apiUrl, AuthConfiguration(authDomain, authClientId, authClientSecret)) + val manager = ScenarioManager(client) + + logger.info { "Watching for queued scenarios" } + + while (true) { + val scenario = manager.findNext() + + if (scenario == null) { + delay(POLL_INTERVAL) + continue + } + + val id = scenario.id + + logger.info { "Found queued scenario $id: attempting to claim" } + + if (!manager.claim(id)) { + logger.info { "Failed to claim scenario" } + continue + } + + coroutineScope { + // Launch heartbeat process + val heartbeat = launch { + while (true) { + manager.heartbeat(id) + delay(HEARTBEAT_INTERVAL) + } + } + + try { + val scenarioModel = client.getScenario(id)!! + val portfolio = client.getPortfolio(scenarioModel.portfolioId)!! + val environment = convert(client.getTopology(scenarioModel.topology.topologyId)!!) + val results = runScenario(portfolio, scenarioModel, environment) + + logger.info { "Writing results to database" } + + manager.finish(id, results) + + logger.info { "Successfully finished scenario $id" } + } catch (e: Exception) { + logger.error(e) { "Scenario failed to finish" } + manager.fail(id) + } finally { + heartbeat.cancel() + } + } + } + } + + /** + * Convert the specified [topology] into an [EnvironmentReader] understood by Capelin. + */ + private fun convert(topology: Topology): EnvironmentReader { + val nodes = mutableListOf() + val random = Random(0) + + val machines = topology.rooms.asSequence() + .flatMap { room -> + room.tiles.flatMap { tile -> + tile.rack?.machines?.map { machine -> tile.rack to machine } ?: emptyList() + } + } + for ((rack, machine) in machines) { + val clusterId = rack.id + val position = machine.position + + val processors = machine.cpus.flatMap { cpu -> + val cores = cpu.numberOfCores + val speed = cpu.clockRateMhz + // TODO Remove hard coding of vendor + val node = ProcessingNode("Intel", "amd64", cpu.name, cores) + List(cores) { coreId -> + ProcessingUnit(node, coreId, speed) + } + } + val memoryUnits = machine.memory.map { memory -> + MemoryUnit( + "Samsung", + memory.name, + memory.speedMbPerS, + memory.sizeMb.toLong() + ) + } + + val energyConsumptionW = machine.cpus.sumOf { it.energyConsumptionW } + + nodes.add( + MachineDef( + UUID(random.nextLong(), random.nextLong()), + "node-$clusterId-$position", + mapOf("cluster" to clusterId), + MachineModel(processors, memoryUnits), + LinearPowerModel(2 * energyConsumptionW, energyConsumptionW * 0.5) + ) + ) + } + + return object : EnvironmentReader { + override fun read(): List = nodes + override fun close() {} + } + } +} + +/** + * Main entry point of the runner. + */ +fun main(args: Array): Unit = RunnerCli().main(args) diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt new file mode 100644 index 00000000..4044cec9 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/ScenarioManager.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import org.opendc.web.client.ApiClient +import org.opendc.web.client.model.Job +import org.opendc.web.client.model.SimulationState + +/** + * Manages the queue of scenarios that need to be processed. + */ +public class ScenarioManager(private val client: ApiClient) { + /** + * Find the next job that the simulator needs to process. + */ + public suspend fun findNext(): Job? { + return client.getJobs().firstOrNull() + } + + /** + * Claim the simulation job with the specified id. + */ + public suspend fun claim(id: String): Boolean { + return client.updateJob(id, SimulationState.CLAIMED) + } + + /** + * Update the heartbeat of the specified scenario. + */ + public suspend fun heartbeat(id: String) { + client.updateJob(id, SimulationState.RUNNING) + } + + /** + * Mark the scenario as failed. + */ + public suspend fun fail(id: String) { + client.updateJob(id, SimulationState.FAILED) + } + + /** + * Persist the specified results. + */ + public suspend fun finish(id: String, results: List) { + client.updateJob( + id, SimulationState.FINISHED, + mapOf( + "total_requested_burst" to results.map { it.totalRequestedBurst }, + "total_granted_burst" to results.map { it.totalGrantedBurst }, + "total_overcommitted_burst" to results.map { it.totalOvercommittedBurst }, + "total_interfered_burst" to results.map { it.totalInterferedBurst }, + "mean_cpu_usage" to results.map { it.meanCpuUsage }, + "mean_cpu_demand" to results.map { it.meanCpuDemand }, + "mean_num_deployed_images" to results.map { it.meanNumDeployedImages }, + "max_num_deployed_images" to results.map { it.maxNumDeployedImages }, + "total_power_draw" to results.map { it.totalPowerDraw }, + "total_failure_slices" to results.map { it.totalFailureSlices }, + "total_failure_vm_slices" to results.map { it.totalFailureVmSlices }, + "total_vms_submitted" to results.map { it.totalVmsSubmitted }, + "total_vms_queued" to results.map { it.totalVmsQueued }, + "total_vms_finished" to results.map { it.totalVmsFinished }, + "total_vms_failed" to results.map { it.totalVmsFailed } + ) + ) + } +} diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt new file mode 100644 index 00000000..d4445810 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/WebExperimentMonitor.kt @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.runner + +import mu.KotlinLogging +import org.opendc.compute.api.Server +import org.opendc.compute.api.ServerState +import org.opendc.compute.service.driver.Host +import org.opendc.compute.service.driver.HostState +import org.opendc.experiments.capelin.monitor.ExperimentMonitor +import org.opendc.experiments.capelin.telemetry.HostEvent +import kotlin.math.max + +/** + * An [ExperimentMonitor] that tracks the aggregate metrics for each repeat. + */ +public class WebExperimentMonitor : ExperimentMonitor { + private val logger = KotlinLogging.logger {} + + override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) {} + + override fun reportHostStateChange(time: Long, host: Host, newState: HostState) { + logger.debug { "Host ${host.uid} changed state $newState [$time]" } + } + + override fun reportHostSlice( + time: Long, + requestedBurst: Long, + grantedBurst: Long, + overcommissionedBurst: Long, + interferedBurst: Long, + cpuUsage: Double, + cpuDemand: Double, + powerDraw: Double, + numberOfDeployedImages: Int, + host: Host, + ) { + processHostEvent( + HostEvent( + time, + 5 * 60 * 1000L, + host, + numberOfDeployedImages, + requestedBurst, + grantedBurst, + overcommissionedBurst, + interferedBurst, + cpuUsage, + cpuDemand, + powerDraw, + host.model.cpuCount + ) + ) + } + + private var hostAggregateMetrics: AggregateHostMetrics = AggregateHostMetrics() + private val hostMetrics: MutableMap = mutableMapOf() + + private fun processHostEvent(event: HostEvent) { + val slices = event.duration / SLICE_LENGTH + + hostAggregateMetrics = AggregateHostMetrics( + hostAggregateMetrics.totalRequestedBurst + event.requestedBurst, + hostAggregateMetrics.totalGrantedBurst + event.grantedBurst, + hostAggregateMetrics.totalOvercommittedBurst + event.overcommissionedBurst, + hostAggregateMetrics.totalInterferedBurst + event.interferedBurst, + hostAggregateMetrics.totalPowerDraw + (event.duration * event.powerDraw) / 3600, + hostAggregateMetrics.totalFailureSlices + if (event.host.state != HostState.UP) slices else 0, + hostAggregateMetrics.totalFailureVmSlices + if (event.host.state != HostState.UP) event.vmCount * slices else 0 + ) + + hostMetrics.compute(event.host) { _, prev -> + HostMetrics( + (event.cpuUsage.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuUsage ?: 0.0), + (event.cpuDemand.takeIf { event.host.state == HostState.UP } ?: 0.0) + (prev?.cpuDemand ?: 0.0), + event.vmCount + (prev?.vmCount ?: 0), + 1 + (prev?.count ?: 0) + ) + } + } + + private val SLICE_LENGTH: Long = 5 * 60 * 1000 + + public data class AggregateHostMetrics( + val totalRequestedBurst: Long = 0, + val totalGrantedBurst: Long = 0, + val totalOvercommittedBurst: Long = 0, + val totalInterferedBurst: Long = 0, + val totalPowerDraw: Double = 0.0, + val totalFailureSlices: Long = 0, + val totalFailureVmSlices: Long = 0, + ) + + public data class HostMetrics( + val cpuUsage: Double, + val cpuDemand: Double, + val vmCount: Long, + val count: Long + ) + + private var provisionerMetrics: AggregateProvisionerMetrics = AggregateProvisionerMetrics() + + override fun reportProvisionerMetrics( + time: Long, + totalHostCount: Int, + availableHostCount: Int, + totalVmCount: Int, + activeVmCount: Int, + inactiveVmCount: Int, + waitingVmCount: Int, + failedVmCount: Int + ) { + provisionerMetrics = AggregateProvisionerMetrics( + max(totalVmCount, provisionerMetrics.vmTotalCount), + max(waitingVmCount, provisionerMetrics.vmWaitingCount), + max(activeVmCount, provisionerMetrics.vmActiveCount), + max(inactiveVmCount, provisionerMetrics.vmInactiveCount), + max(failedVmCount, provisionerMetrics.vmFailedCount), + ) + } + + public data class AggregateProvisionerMetrics( + val vmTotalCount: Int = 0, + val vmWaitingCount: Int = 0, + val vmActiveCount: Int = 0, + val vmInactiveCount: Int = 0, + val vmFailedCount: Int = 0 + ) + + override fun close() {} + + public fun getResult(): Result { + return Result( + hostAggregateMetrics.totalRequestedBurst, + hostAggregateMetrics.totalGrantedBurst, + hostAggregateMetrics.totalOvercommittedBurst, + hostAggregateMetrics.totalInterferedBurst, + hostMetrics.map { it.value.cpuUsage / it.value.count }.average(), + hostMetrics.map { it.value.cpuDemand / it.value.count }.average(), + hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.average(), + hostMetrics.map { it.value.vmCount.toDouble() / it.value.count }.maxOrNull() ?: 0.0, + hostAggregateMetrics.totalPowerDraw, + hostAggregateMetrics.totalFailureSlices, + hostAggregateMetrics.totalFailureVmSlices, + provisionerMetrics.vmTotalCount, + provisionerMetrics.vmWaitingCount, + provisionerMetrics.vmInactiveCount, + provisionerMetrics.vmFailedCount, + ) + } + + public data class Result( + public val totalRequestedBurst: Long, + public val totalGrantedBurst: Long, + public val totalOvercommittedBurst: Long, + public val totalInterferedBurst: Long, + public val meanCpuUsage: Double, + public val meanCpuDemand: Double, + public val meanNumDeployedImages: Double, + public val maxNumDeployedImages: Double, + public val totalPowerDraw: Double, + public val totalFailureSlices: Long, + public val totalFailureVmSlices: Long, + public val totalVmsSubmitted: Int, + public val totalVmsQueued: Int, + public val totalVmsFinished: Int, + public val totalVmsFailed: Int + ) +} diff --git a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml index 503bc5dc..ad99cc00 100644 --- a/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml +++ b/opendc-web/opendc-web-runner/src/main/resources/log4j2.xml @@ -36,7 +36,7 @@ - + diff --git a/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt new file mode 100644 index 00000000..3a0730a6 --- /dev/null +++ b/opendc-web/opendc-web-runner/src/test/kotlin/org/opendc/web/client/ApiClientTest.kt @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.web.client + +import io.ktor.client.* +import io.ktor.client.engine.mock.* +import io.ktor.http.* +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test +import java.net.URI + +/** + * Test suite for the [ApiClient] class. + */ +class ApiClientTest { + /** + * The Ktor [HttpClient] instance. + */ + private val ktor = HttpClient(MockEngine) { + engine { + addHandler { request -> + when (request.url.fullPath) { + "/oauth/token" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "access_token": "eyJz93a...k4laUWw", + "token_type": "Bearer", + "expires_in": 86400 + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/5fda5daa97dca438e7cb0a4c" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "scenarioIds": [ + "string" + ], + "targets": { + "enabledMetrics": [ + "string" + ], + "repeatsPerScenario": 0 + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/portfolios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/scenarios/5fda5db297dca438e7cb0a4d" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "portfolioId": "string", + "name": "string", + "trace": { + "traceId": "string", + "loadSamplingFraction": 0 + }, + "topology": { + "topologyId": "string" + }, + "operational": { + "failuresEnabled": true, + "performanceInterferenceEnabled": true, + "schedulerName": "string" + } + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/scenarios/x" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + "/topologies/5f9825a6cf6e4c24e380b86f" -> { + val responseHeaders = headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "data": { + "_id": "string", + "projectId": "string", + "name": "string", + "rooms": [ + { + "_id": "string", + "name": "string", + "tiles": [ + { + "_id": "string", + "positionX": 0, + "positionY": 0, + "rack": { + "_id": "string", + "name": "string", + "capacity": 0, + "powerCapacityW": 0, + "machines": [ + { + "_id": "string", + "position": 0, + "cpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "gpus": [ + { + "_id": "string", + "name": "string", + "clockRateMhz": 0, + "numberOfCores": 0 + } + ], + "memories": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ], + "storages": [ + { + "_id": "string", + "name": "string", + "speedMbPerS": 0, + "sizeMb": 0 + } + ] + } + ] + } + } + ] + } + ] + } + } + """.trimIndent(), + headers = responseHeaders + ) + } + "/topologies/x" -> { + val responseHeaders = + headersOf("Content-Type" to listOf(ContentType.Application.Json.toString())) + respond( + """ + { + "message": "Not Found" + } + """.trimIndent(), + headers = responseHeaders, status = HttpStatusCode.NotFound + ) + } + else -> error("Unhandled ${request.url}") + } + } + } + } + + private val auth = AuthConfiguration("auth.opendc.org", "a", "b") + + @Test + fun testPortfolioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("5fda5daa97dca438e7cb0a4c") + assertNotNull(portfolio) + } + + @Test + fun testPortfolioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val portfolio = client.getPortfolio("x") + assertNull(portfolio) + } + + @Test + fun testScenarioExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("5fda5db297dca438e7cb0a4d") + assertNotNull(scenario) + } + + @Test + fun testScenarioDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val scenario = client.getScenario("x") + assertNull(scenario) + } + + @Test + fun testTopologyExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("5f9825a6cf6e4c24e380b86f") + assertNotNull(topology) + } + + @Test + fun testTopologyDoesNotExists(): Unit = runBlocking { + val client = ApiClient(URI("http://localhost:8081"), auth, client = ktor) + val topology = client.getTopology("x") + assertNull(topology) + } +} -- cgit v1.2.3