From 0bbb0adb97ba4783bbd0073f845781725e6212e8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 25 Mar 2021 10:23:47 +0100 Subject: compute: Add test suite for ComputeService This change adds a test suite for the OpenDC compute service. --- .../opendc-compute-service/build.gradle.kts | 3 +- .../compute/service/internal/ClientFlavor.kt | 6 + .../opendc/compute/service/internal/ClientImage.kt | 6 + .../compute/service/internal/ClientServer.kt | 6 + .../compute/service/internal/ComputeServiceImpl.kt | 274 ++++++++------- .../opendc/compute/service/internal/HostView.kt | 2 + .../compute/service/internal/InternalFlavor.kt | 4 +- .../compute/service/internal/InternalImage.kt | 4 +- .../compute/service/internal/InternalServer.kt | 46 ++- .../service/scheduler/ReplayAllocationPolicy.kt | 7 +- .../opendc/compute/service/ComputeServiceTest.kt | 390 +++++++++++++++++++++ .../opendc/compute/service/InternalFlavorTest.kt | 80 +++++ .../opendc/compute/service/InternalImageTest.kt | 81 +++++ .../opendc/compute/service/InternalServerTest.kt | 285 +++++++++++++++ .../service/scheduler/AllocationPolicyTest.kt | 219 ++++++++++++ .../src/test/resources/log4j2.xml | 38 ++ 16 files changed, 1297 insertions(+), 154 deletions(-) create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalFlavorTest.kt create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalImageTest.kt create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/AllocationPolicyTest.kt create mode 100644 simulator/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts index 1b09ef6d..1825e989 100644 --- a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts +++ b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts @@ -26,6 +26,7 @@ description = "OpenDC Compute Service implementation" plugins { `kotlin-library-conventions` `testing-conventions` + `jacoco-conventions` } dependencies { @@ -36,5 +37,5 @@ dependencies { implementation("io.github.microutils:kotlin-logging") testImplementation(project(":opendc-simulator:opendc-simulator-core")) - testRuntimeOnly("org.slf4j:slf4j-simple:${versions.slf4j}") + testRuntimeOnly("org.apache.logging.log4j:log4j-slf4j-impl") } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientFlavor.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientFlavor.kt index 29f10e27..4a8d3046 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientFlavor.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientFlavor.kt @@ -59,4 +59,10 @@ internal class ClientFlavor(private val delegate: Flavor) : Flavor { labels = delegate.labels meta = delegate.meta } + + override fun equals(other: Any?): Boolean = other is Flavor && other.uid == uid + + override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Flavor[uid=$uid,name=$name]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientImage.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientImage.kt index 6c5b2ab0..e0b5c171 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientImage.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientImage.kt @@ -52,4 +52,10 @@ internal class ClientImage(private val delegate: Image) : Image { labels = delegate.labels meta = delegate.meta } + + override fun equals(other: Any?): Boolean = other is Image && other.uid == uid + + override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Image[uid=$uid,name=$name]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientServer.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientServer.kt index ae4cee3b..f2929bf3 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientServer.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ClientServer.kt @@ -104,4 +104,10 @@ internal class ClientServer(private val delegate: Server) : Server, ServerWatche watcher.onStateChanged(this, newState) } } + + override fun equals(other: Any?): Boolean = other is Server && other.uid == uid + + override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Server[uid=$uid,name=$name,state=$state]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt index aa7e0aa1..62808b4d 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt @@ -57,7 +57,7 @@ public class ComputeServiceImpl( /** * The [CoroutineScope] of the service bounded by the lifecycle of the service. */ - private val scope = CoroutineScope(context) + private val scope = CoroutineScope(context + Job()) /** * The logger instance of this server. @@ -133,130 +133,133 @@ public class ComputeServiceImpl( override val hostCount: Int get() = hostToView.size - override fun newClient(): ComputeClient = object : ComputeClient { - private var isClosed: Boolean = false + override fun newClient(): ComputeClient { + check(scope.isActive) { "Service is already closed" } + return object : ComputeClient { + private var isClosed: Boolean = false - override suspend fun queryFlavors(): List { - check(!isClosed) { "Client is already closed" } + override suspend fun queryFlavors(): List { + check(!isClosed) { "Client is already closed" } - return flavors.values.map { ClientFlavor(it) } - } + return flavors.values.map { ClientFlavor(it) } + } - override suspend fun findFlavor(id: UUID): Flavor? { - check(!isClosed) { "Client is already closed" } + override suspend fun findFlavor(id: UUID): Flavor? { + check(!isClosed) { "Client is already closed" } - return flavors[id]?.let { ClientFlavor(it) } - } + return flavors[id]?.let { ClientFlavor(it) } + } - override suspend fun newFlavor( - name: String, - cpuCount: Int, - memorySize: Long, - labels: Map, - meta: Map - ): Flavor { - check(!isClosed) { "Client is already closed" } - - val uid = UUID(clock.millis(), random.nextLong()) - val flavor = InternalFlavor( - this@ComputeServiceImpl, - uid, - name, - cpuCount, - memorySize, - labels, - meta - ) + override suspend fun newFlavor( + name: String, + cpuCount: Int, + memorySize: Long, + labels: Map, + meta: Map + ): Flavor { + check(!isClosed) { "Client is already closed" } + + val uid = UUID(clock.millis(), random.nextLong()) + val flavor = InternalFlavor( + this@ComputeServiceImpl, + uid, + name, + cpuCount, + memorySize, + labels, + meta + ) - flavors[uid] = flavor + flavors[uid] = flavor - return ClientFlavor(flavor) - } + return ClientFlavor(flavor) + } - override suspend fun queryImages(): List { - check(!isClosed) { "Client is already closed" } + override suspend fun queryImages(): List { + check(!isClosed) { "Client is already closed" } - return images.values.map { ClientImage(it) } - } + return images.values.map { ClientImage(it) } + } - override suspend fun findImage(id: UUID): Image? { - check(!isClosed) { "Client is already closed" } + override suspend fun findImage(id: UUID): Image? { + check(!isClosed) { "Client is already closed" } - return images[id]?.let { ClientImage(it) } - } + return images[id]?.let { ClientImage(it) } + } - override suspend fun newImage(name: String, labels: Map, meta: Map): Image { - check(!isClosed) { "Client is already closed" } + override suspend fun newImage(name: String, labels: Map, meta: Map): Image { + check(!isClosed) { "Client is already closed" } - val uid = UUID(clock.millis(), random.nextLong()) - val image = InternalImage(this@ComputeServiceImpl, uid, name, labels, meta) + val uid = UUID(clock.millis(), random.nextLong()) + val image = InternalImage(this@ComputeServiceImpl, uid, name, labels, meta) - images[uid] = image + images[uid] = image - return ClientImage(image) - } + return ClientImage(image) + } - override suspend fun newServer( - name: String, - image: Image, - flavor: Flavor, - labels: Map, - meta: Map, - start: Boolean - ): Server { - check(!isClosed) { "Client is closed" } - tracer.commit(VmSubmissionEvent(name, image, flavor)) + override suspend fun newServer( + name: String, + image: Image, + flavor: Flavor, + labels: Map, + meta: Map, + start: Boolean + ): Server { + check(!isClosed) { "Client is closed" } + tracer.commit(VmSubmissionEvent(name, image, flavor)) - _events.emit( - ComputeServiceEvent.MetricsAvailable( + _events.emit( + ComputeServiceEvent.MetricsAvailable( + this@ComputeServiceImpl, + hostCount, + availableHosts.size, + ++submittedVms, + runningVms, + finishedVms, + ++queuedVms, + unscheduledVms + ) + ) + + val uid = UUID(clock.millis(), random.nextLong()) + val server = InternalServer( this@ComputeServiceImpl, - hostCount, - availableHosts.size, - ++submittedVms, - runningVms, - finishedVms, - ++queuedVms, - unscheduledVms + uid, + name, + requireNotNull(flavors[flavor.uid]) { "Unknown flavor" }, + requireNotNull(images[image.uid]) { "Unknown image" }, + labels.toMutableMap(), + meta.toMutableMap() ) - ) - val uid = UUID(clock.millis(), random.nextLong()) - val server = InternalServer( - this@ComputeServiceImpl, - uid, - name, - flavor, - image, - labels.toMutableMap(), - meta.toMutableMap() - ) + servers[uid] = server - servers[uid] = server + if (start) { + server.start() + } - if (start) { - server.start() + return ClientServer(server) } - return ClientServer(server) - } + override suspend fun findServer(id: UUID): Server? { + check(!isClosed) { "Client is already closed" } - override suspend fun findServer(id: UUID): Server? { - check(!isClosed) { "Client is already closed" } + return servers[id]?.let { ClientServer(it) } + } - return servers[id]?.let { ClientServer(it) } - } + override suspend fun queryServers(): List { + check(!isClosed) { "Client is already closed" } - override suspend fun queryServers(): List { - check(!isClosed) { "Client is already closed" } + return servers.values.map { ClientServer(it) } + } - return servers.values.map { ClientServer(it) } - } + override fun close() { + isClosed = true + } - override fun close() { - isClosed = true + override fun toString(): String = "ComputeClient" } - - override fun toString(): String = "ComputeClient" } override fun addHost(host: Host) { @@ -285,23 +288,25 @@ public class ComputeServiceImpl( scope.cancel() } - internal fun schedule(server: InternalServer) { + internal fun schedule(server: InternalServer): SchedulingRequest { logger.debug { "Enqueueing server ${server.uid} to be assigned to host." } - queue.add(SchedulingRequest(server)) + val request = SchedulingRequest(server) + queue.add(request) requestSchedulingCycle() + return request } internal fun delete(flavor: InternalFlavor) { - checkNotNull(flavors.remove(flavor.uid)) { "Flavor was not known" } + flavors.remove(flavor.uid) } internal fun delete(image: InternalImage) { - checkNotNull(images.remove(image.uid)) { "Image was not known" } + images.remove(image.uid) } internal fun delete(server: InternalServer) { - checkNotNull(servers.remove(server.uid)) { "Server was not known" } + servers.remove(server.uid) } /** @@ -338,7 +343,7 @@ public class ComputeServiceImpl( val server = request.server val hv = allocationLogic.select(availableHosts, request.server) if (hv == null || !hv.host.canFit(server)) { - logger.trace { "Server $server selected for scheduling but no capacity available for it." } + logger.trace { "Server $server selected for scheduling but no capacity available for it at the moment" } if (server.flavor.memorySize > maxMemory || server.flavor.cpuCount > maxCores) { tracer.commit(VmSubmissionInvalidEvent(server.name)) @@ -360,6 +365,8 @@ public class ComputeServiceImpl( queue.poll() logger.warn("Failed to spawn $server: does not fit [${clock.millis()}]") + + server.state = ServerState.ERROR continue } else { break @@ -372,42 +379,39 @@ public class ComputeServiceImpl( queue.poll() logger.info { "Assigned server $server to host $host." } - try { - // Speculatively update the hypervisor view information to prevent other images in the queue from - // deciding on stale values. - hv.numberOfActiveServers++ - hv.provisionedCores += server.flavor.cpuCount - hv.availableMemory -= server.flavor.memorySize // XXX Temporary hack - - scope.launch { - try { - server.assignHost(host) - host.spawn(server) - activeServers[server] = host - - tracer.commit(VmScheduledEvent(server.name)) - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - ++runningVms, - finishedVms, - --queuedVms, - unscheduledVms - ) + + // Speculatively update the hypervisor view information to prevent other images in the queue from + // deciding on stale values. + hv.numberOfActiveServers++ + hv.provisionedCores += server.flavor.cpuCount + hv.availableMemory -= server.flavor.memorySize // XXX Temporary hack + + scope.launch { + try { + server.host = host + host.spawn(server) + activeServers[server] = host + + tracer.commit(VmScheduledEvent(server.name)) + _events.emit( + ComputeServiceEvent.MetricsAvailable( + this@ComputeServiceImpl, + hostCount, + availableHosts.size, + submittedVms, + ++runningVms, + finishedVms, + --queuedVms, + unscheduledVms ) - } catch (e: Throwable) { - logger.error("Failed to deploy VM", e) + ) + } catch (e: Throwable) { + logger.error("Failed to deploy VM", e) - hv.numberOfActiveServers-- - hv.provisionedCores -= server.flavor.cpuCount - hv.availableMemory += server.flavor.memorySize - } + hv.numberOfActiveServers-- + hv.provisionedCores -= server.flavor.cpuCount + hv.availableMemory += server.flavor.memorySize } - } catch (e: Exception) { - logger.warn(e) { "Failed to assign server $server to $host. " } } } } @@ -415,7 +419,7 @@ public class ComputeServiceImpl( /** * A request to schedule an [InternalServer] onto one of the [Host]s. */ - private data class SchedulingRequest(val server: InternalServer) { + internal data class SchedulingRequest(val server: InternalServer) { /** * A flag to indicate that the request is cancelled. */ diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/HostView.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/HostView.kt index 1bdfdf1a..5793541f 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/HostView.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/HostView.kt @@ -32,4 +32,6 @@ public class HostView(public val host: Host) { public var numberOfActiveServers: Int = 0 public var availableMemory: Long = host.model.memorySize public var provisionedCores: Int = 0 + + override fun toString(): String = "HostView[host=$host]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalFlavor.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalFlavor.kt index 95e280df..b8fb6279 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalFlavor.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalFlavor.kt @@ -58,7 +58,9 @@ internal class InternalFlavor( service.delete(this) } - override fun equals(other: Any?): Boolean = other is InternalFlavor && uid == other.uid + override fun equals(other: Any?): Boolean = other is Flavor && uid == other.uid override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Flavor[uid=$uid,name=$name]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalImage.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalImage.kt index 86f2f6b9..d9ed5896 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalImage.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalImage.kt @@ -48,7 +48,9 @@ internal class InternalImage( service.delete(this) } - override fun equals(other: Any?): Boolean = other is InternalImage && uid == other.uid + override fun equals(other: Any?): Boolean = other is Image && uid == other.uid override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Image[uid=$uid,name=$name]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt index ff7c1d15..d9d0f3fc 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt @@ -34,8 +34,8 @@ internal class InternalServer( private val service: ComputeServiceImpl, override val uid: UUID, override val name: String, - override val flavor: Flavor, - override val image: Image, + override val flavor: InternalFlavor, + override val image: InternalImage, override val labels: MutableMap, override val meta: MutableMap ) : Server { @@ -54,6 +54,11 @@ internal class InternalServer( */ internal var host: Host? = null + /** + * The current scheduling request. + */ + private var request: ComputeServiceImpl.SchedulingRequest? = null + override suspend fun start() { when (state) { ServerState.RUNNING -> { @@ -66,35 +71,43 @@ internal class InternalServer( } ServerState.DELETED -> { logger.warn { "User tried to start terminated server" } - throw IllegalArgumentException("Server is terminated") + throw IllegalStateException("Server is terminated") } else -> { logger.info { "User requested to start server $uid" } state = ServerState.PROVISIONING - service.schedule(this) + assert(request == null) { "Scheduling request already active" } + request = service.schedule(this) } } } override suspend fun stop() { when (state) { - ServerState.PROVISIONING -> {} // TODO Find way to interrupt these + ServerState.PROVISIONING -> { + cancelProvisioningRequest() + state = ServerState.TERMINATED + } ServerState.RUNNING, ServerState.ERROR -> { val host = checkNotNull(host) { "Server not running" } host.stop(this) } - ServerState.TERMINATED -> {} // No work needed - ServerState.DELETED -> throw IllegalStateException("Server is terminated") + ServerState.TERMINATED, ServerState.DELETED -> {} // No work needed } } override suspend fun delete() { when (state) { - ServerState.PROVISIONING -> {} // TODO Find way to interrupt these - ServerState.RUNNING -> { + ServerState.PROVISIONING, ServerState.TERMINATED -> { + cancelProvisioningRequest() + service.delete(this) + state = ServerState.DELETED + } + ServerState.RUNNING, ServerState.ERROR -> { val host = checkNotNull(host) { "Server not running" } host.delete(this) service.delete(this) + state = ServerState.DELETED } else -> {} // No work needed } @@ -121,11 +134,20 @@ internal class InternalServer( field = value } - internal fun assignHost(host: Host) { - this.host = host + /** + * Cancel the provisioning request if active. + */ + private fun cancelProvisioningRequest() { + val request = request + if (request != null) { + this.request = null + request.isCancelled = true + } } - override fun equals(other: Any?): Boolean = other is InternalServer && uid == other.uid + override fun equals(other: Any?): Boolean = other is Server && uid == other.uid override fun hashCode(): Int = uid.hashCode() + + override fun toString(): String = "Server[uid=$uid,state=$state]" } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayAllocationPolicy.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayAllocationPolicy.kt index ed1dc662..2c953f8b 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayAllocationPolicy.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/scheduler/ReplayAllocationPolicy.kt @@ -20,14 +20,11 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.allocation +package org.opendc.compute.service.scheduler import mu.KotlinLogging import org.opendc.compute.api.Server import org.opendc.compute.service.internal.HostView -import org.opendc.compute.service.scheduler.AllocationPolicy - -private val logger = KotlinLogging.logger {} /** * Policy replaying VM-cluster assignment. @@ -36,6 +33,8 @@ private val logger = KotlinLogging.logger {} * assigned the VM image. */ public class ReplayAllocationPolicy(private val vmPlacements: Map) : AllocationPolicy { + private val logger = KotlinLogging.logger {} + override fun invoke(): AllocationPolicy.Logic = object : AllocationPolicy.Logic { override fun select( hypervisors: Set, diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt new file mode 100644 index 00000000..ffec92ea --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt @@ -0,0 +1,390 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.service + +import io.mockk.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.TestCoroutineScope +import kotlinx.coroutines.test.runBlockingTest +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.opendc.compute.api.* +import org.opendc.compute.service.driver.Host +import org.opendc.compute.service.driver.HostListener +import org.opendc.compute.service.driver.HostModel +import org.opendc.compute.service.driver.HostState +import org.opendc.compute.service.scheduler.AvailableMemoryAllocationPolicy +import org.opendc.simulator.utils.DelayControllerClockAdapter +import org.opendc.trace.core.EventTracer +import java.util.* + +/** + * Test suite for the [ComputeService] interface. + */ +@OptIn(ExperimentalCoroutinesApi::class) +internal class ComputeServiceTest { + lateinit var scope: TestCoroutineScope + lateinit var service: ComputeService + + @BeforeEach + fun setUp() { + scope = TestCoroutineScope() + val clock = DelayControllerClockAdapter(scope) + val tracer = EventTracer(clock) + val policy = AvailableMemoryAllocationPolicy() + service = ComputeService(scope.coroutineContext, clock, tracer, policy) + } + + @AfterEach + fun tearDown() { + scope.cleanupTestCoroutines() + } + + @Test + fun testClientClose() = scope.runBlockingTest { + val client = service.newClient() + + assertEquals(emptyList(), client.queryFlavors()) + assertEquals(emptyList(), client.queryImages()) + assertEquals(emptyList(), client.queryServers()) + + client.close() + + assertThrows { client.queryFlavors() } + assertThrows { client.queryImages() } + assertThrows { client.queryServers() } + + assertThrows { client.findFlavor(UUID.randomUUID()) } + assertThrows { client.findImage(UUID.randomUUID()) } + assertThrows { client.findServer(UUID.randomUUID()) } + + assertThrows { client.newFlavor("test", 1, 2) } + assertThrows { client.newImage("test") } + assertThrows { client.newServer("test", mockk(), mockk()) } + } + + @Test + fun testClientCreate() = scope.runBlockingTest { + val client = service.newClient() + + val flavor = client.newFlavor("test", 1, 1024) + assertEquals(listOf(flavor), client.queryFlavors()) + assertEquals(flavor, client.findFlavor(flavor.uid)) + val image = client.newImage("test") + assertEquals(listOf(image), client.queryImages()) + assertEquals(image, client.findImage(image.uid)) + val server = client.newServer("test", image, flavor, start = false) + assertEquals(listOf(server), client.queryServers()) + assertEquals(server, client.findServer(server.uid)) + + server.delete() + assertNull(client.findServer(server.uid)) + + image.delete() + assertNull(client.findImage(image.uid)) + + flavor.delete() + assertNull(client.findFlavor(flavor.uid)) + + assertThrows { server.start() } + } + + @Test + fun testClientOnClose() = scope.runBlockingTest { + service.close() + assertThrows { + service.newClient() + } + } + + @Test + fun testAddHost() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + + assertEquals(0, service.hostCount) + assertEquals(emptySet(), service.hosts) + + service.addHost(host) + + verify(exactly = 1) { host.addListener(any()) } + + assertEquals(1, service.hostCount) + assertEquals(1, service.hosts.size) + + service.removeHost(host) + + verify(exactly = 1) { host.removeListener(any()) } + } + + @Test + fun testAddHostDouble() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.DOWN + + assertEquals(0, service.hostCount) + assertEquals(emptySet(), service.hosts) + + service.addHost(host) + service.addHost(host) + + verify(exactly = 1) { host.addListener(any()) } + } + + @Test + fun testServerStartWithoutEnoughCpus() = scope.runBlockingTest { + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 0) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.ERROR, server.state) + } + + @Test + fun testServerStartWithoutEnoughMemory() = scope.runBlockingTest { + val client = service.newClient() + val flavor = client.newFlavor("test", 0, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.ERROR, server.state) + } + + @Test + fun testServerStartWithoutEnoughResources() = scope.runBlockingTest { + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.ERROR, server.state) + } + + @Test + fun testServerCancelRequest() = scope.runBlockingTest { + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + server.stop() + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.TERMINATED, server.state) + } + + @Test + fun testServerCannotFitOnHost() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + every { host.canFit(any()) } returns false + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(10 * 60 * 1000) + server.refresh() + assertEquals(ServerState.PROVISIONING, server.state) + + verify { host.canFit(server) } + } + + @Test + fun testHostAvailableAfterSomeTime() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + val listeners = mutableListOf() + + every { host.uid } returns UUID.randomUUID() + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.DOWN + every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } + every { host.canFit(any()) } returns false + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(5 * 60 * 1000) + + listeners.forEach { it.onStateChanged(host, HostState.UP) } + + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.PROVISIONING, server.state) + + verify { host.canFit(server) } + } + + @Test + fun testHostUnavailableAfterSomeTime() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + val listeners = mutableListOf() + + every { host.uid } returns UUID.randomUUID() + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } + every { host.canFit(any()) } returns false + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + delay(5 * 60 * 1000) + + listeners.forEach { it.onStateChanged(host, HostState.DOWN) } + + server.start() + delay(5 * 60 * 1000) + server.refresh() + assertEquals(ServerState.PROVISIONING, server.state) + + verify(exactly = 0) { host.canFit(server) } + } + + @Test + fun testServerInvalidType() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + val listeners = mutableListOf() + + every { host.uid } returns UUID.randomUUID() + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + every { host.canFit(any()) } returns true + every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + assertThrows { + listeners.forEach { it.onStateChanged(host, server, ServerState.RUNNING) } + } + } + + @Test + fun testServerDeploy() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + val listeners = mutableListOf() + + every { host.uid } returns UUID.randomUUID() + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + every { host.canFit(any()) } returns true + every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + val slot = slot() + + val watcher = mockk(relaxUnitFun = true) + server.watch(watcher) + + // Start server + server.start() + delay(5 * 60 * 1000) + coVerify { host.spawn(capture(slot), true) } + + listeners.forEach { it.onStateChanged(host, slot.captured, ServerState.RUNNING) } + + server.refresh() + assertEquals(ServerState.RUNNING, server.state) + + verify { watcher.onStateChanged(server, ServerState.RUNNING) } + + // Stop server + listeners.forEach { it.onStateChanged(host, slot.captured, ServerState.TERMINATED) } + + server.refresh() + assertEquals(ServerState.TERMINATED, server.state) + + verify { watcher.onStateChanged(server, ServerState.TERMINATED) } + } + + @Test + fun testServerDeployFailure() = scope.runBlockingTest { + val host = mockk(relaxUnitFun = true) + val listeners = mutableListOf() + + every { host.uid } returns UUID.randomUUID() + every { host.model } returns HostModel(4, 2048) + every { host.state } returns HostState.UP + every { host.canFit(any()) } returns true + every { host.addListener(any()) } answers { listeners.add(it.invocation.args[0] as HostListener) } + coEvery { host.spawn(any(), true) } throws IllegalStateException() + + service.addHost(host) + + val client = service.newClient() + val flavor = client.newFlavor("test", 1, 1024) + val image = client.newImage("test") + val server = client.newServer("test", image, flavor, start = false) + + server.start() + delay(5 * 60 * 1000) + + server.refresh() + assertEquals(ServerState.PROVISIONING, server.state) + } +} diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalFlavorTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalFlavorTest.kt new file mode 100644 index 00000000..18d698c6 --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalFlavorTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.service + +import io.mockk.* +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import org.opendc.compute.api.Flavor +import org.opendc.compute.service.internal.ComputeServiceImpl +import org.opendc.compute.service.internal.InternalFlavor +import java.util.* + +/** + * Test suite for the [InternalFlavor] implementation. + */ +class InternalFlavorTest { + @Test + fun testEquality() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) + val b = InternalFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) + + assertEquals(a, b) + } + + @Test + fun testEqualityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns uid + + assertEquals(a, b) + } + + @Test + fun testInequalityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns UUID.randomUUID() + + assertNotEquals(a, b) + } + + @Test + fun testInequalityWithIncorrectType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalFlavor(service, uid, "test", 1, 1024, mutableMapOf(), mutableMapOf()) + + assertNotEquals(a, Unit) + } +} diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalImageTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalImageTest.kt new file mode 100644 index 00000000..e1cb0128 --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalImageTest.kt @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.service + +import io.mockk.* +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Test +import org.opendc.compute.api.Image +import org.opendc.compute.service.internal.ComputeServiceImpl +import org.opendc.compute.service.internal.InternalFlavor +import org.opendc.compute.service.internal.InternalImage +import java.util.* + +/** + * Test suite for the [InternalFlavor] implementation. + */ +class InternalImageTest { + @Test + fun testEquality() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalImage(service, uid, "test", mutableMapOf(), mutableMapOf()) + val b = InternalImage(service, uid, "test", mutableMapOf(), mutableMapOf()) + + assertEquals(a, b) + } + + @Test + fun testEqualityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalImage(service, uid, "test", mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns uid + + assertEquals(a, b) + } + + @Test + fun testInequalityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalImage(service, uid, "test", mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns UUID.randomUUID() + + assertNotEquals(a, b) + } + + @Test + fun testInequalityWithIncorrectType() { + val service = mockk() + val uid = UUID.randomUUID() + val a = InternalImage(service, uid, "test", mutableMapOf(), mutableMapOf()) + + assertNotEquals(a, Unit) + } +} diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt new file mode 100644 index 00000000..81cb45df --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt @@ -0,0 +1,285 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.service + +import io.mockk.* +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runBlockingTest +import kotlinx.coroutines.yield +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import org.opendc.compute.api.Server +import org.opendc.compute.api.ServerState +import org.opendc.compute.service.driver.Host +import org.opendc.compute.service.internal.ComputeServiceImpl +import org.opendc.compute.service.internal.InternalFlavor +import org.opendc.compute.service.internal.InternalImage +import org.opendc.compute.service.internal.InternalServer +import java.util.* + +/** + * Test suite for the [InternalServer] implementation. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class InternalServerTest { + @Test + fun testEquality() { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + val b = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + assertEquals(a, b) + } + + @Test + fun testEqualityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns uid + + assertEquals(a, b) + } + + @Test + fun testInequalityWithDifferentType() { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + val b = mockk(relaxUnitFun = true) + every { b.uid } returns UUID.randomUUID() + + assertNotEquals(a, b) + } + + @Test + fun testInequalityWithIncorrectType() { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + assertNotEquals(a, Unit) + } + + @Test + fun testStartTerminatedServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + every { service.schedule(any()) } answers { ComputeServiceImpl.SchedulingRequest(it.invocation.args[0] as InternalServer) } + + server.start() + + verify(exactly = 1) { service.schedule(server) } + assertEquals(ServerState.PROVISIONING, server.state) + } + + @Test + fun testStartDeletedServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.DELETED + + assertThrows { server.start() } + } + + @Test + fun testStartProvisioningServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.PROVISIONING + + server.start() + + assertEquals(ServerState.PROVISIONING, server.state) + } + + @Test + fun testStartRunningServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.RUNNING + + server.start() + + assertEquals(ServerState.RUNNING, server.state) + } + + @Test + fun testStopProvisioningServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + val request = ComputeServiceImpl.SchedulingRequest(server) + + every { service.schedule(any()) } returns request + + server.start() + server.stop() + + assertTrue(request.isCancelled) + assertEquals(ServerState.TERMINATED, server.state) + } + + @Test + fun testStopTerminatedServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.TERMINATED + server.stop() + + assertEquals(ServerState.TERMINATED, server.state) + } + + @Test + fun testStopDeletedServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.DELETED + server.stop() + + assertEquals(ServerState.DELETED, server.state) + } + + @Test + fun testStopRunningServer() = runBlockingTest { + val service = mockk() + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + val host = mockk(relaxUnitFun = true) + + server.state = ServerState.RUNNING + server.host = host + server.stop() + yield() + + coVerify { host.stop(server) } + } + + @Test + fun testDeleteProvisioningServer() = runBlockingTest { + val service = mockk(relaxUnitFun = true) + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + val request = ComputeServiceImpl.SchedulingRequest(server) + + every { service.schedule(any()) } returns request + + server.start() + server.delete() + + assertTrue(request.isCancelled) + assertEquals(ServerState.DELETED, server.state) + verify { service.delete(server) } + } + + @Test + fun testDeleteTerminatedServer() = runBlockingTest { + val service = mockk(relaxUnitFun = true) + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.TERMINATED + server.delete() + + assertEquals(ServerState.DELETED, server.state) + + verify { service.delete(server) } + } + + @Test + fun testDeleteDeletedServer() = runBlockingTest { + val service = mockk(relaxUnitFun = true) + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + + server.state = ServerState.DELETED + server.delete() + + assertEquals(ServerState.DELETED, server.state) + } + + @Test + fun testDeleteRunningServer() = runBlockingTest { + val service = mockk(relaxUnitFun = true) + val uid = UUID.randomUUID() + val flavor = mockk() + val image = mockk() + val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf()) + val host = mockk(relaxUnitFun = true) + + server.state = ServerState.RUNNING + server.host = host + server.delete() + yield() + + coVerify { host.delete(server) } + verify { service.delete(server) } + } +} diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/AllocationPolicyTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/AllocationPolicyTest.kt new file mode 100644 index 00000000..db377914 --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/scheduler/AllocationPolicyTest.kt @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.service.scheduler + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import org.opendc.compute.api.Server +import org.opendc.compute.service.internal.HostView +import java.util.* +import java.util.stream.Stream +import kotlin.random.Random + +/** + * Test suite for the [AllocationPolicy] interface. + */ +internal class AllocationPolicyTest { + @ParameterizedTest + @MethodSource("activeServersArgs") + fun testActiveServersPolicy( + reversed: Boolean, + hosts: Set, + server: Server, + expectedHost: HostView? + ) { + val policy = NumberOfActiveServersAllocationPolicy(reversed) + assertEquals(expectedHost, policy.invoke().select(hosts, server)) + } + + @ParameterizedTest + @MethodSource("availableMemoryArgs") + fun testAvailableMemoryPolicy( + reversed: Boolean, + hosts: Set, + server: Server, + expectedHost: HostView? + ) { + val policy = AvailableMemoryAllocationPolicy(reversed) + assertEquals(expectedHost, policy.invoke().select(hosts, server)) + } + + @ParameterizedTest + @MethodSource("availableCoreMemoryArgs") + fun testAvailableCoreMemoryPolicy( + reversed: Boolean, + hosts: Set, + server: Server, + expectedHost: HostView? + ) { + val policy = AvailableMemoryAllocationPolicy(reversed) + assertEquals(expectedHost, policy.invoke().select(hosts, server)) + } + + @ParameterizedTest + @MethodSource("provisionedCoresArgs") + fun testProvisionedPolicy( + reversed: Boolean, + hosts: Set, + server: Server, + expectedHost: HostView? + ) { + val policy = ProvisionedCoresAllocationPolicy(reversed) + assertEquals(expectedHost, policy.invoke().select(hosts, server)) + } + + @Suppress("unused") + private companion object { + /** + * Test arguments for the [NumberOfActiveServersAllocationPolicy]. + */ + @JvmStatic + fun activeServersArgs(): Stream { + val random = Random(1) + val hosts = List(4) { i -> + val view = mockk() + every { view.host.uid } returns UUID(0, i.toLong()) + every { view.host.model.cpuCount } returns random.nextInt(1, 16) + every { view.host.model.memorySize } returns random.nextLong(1024, 1024 * 1024) + every { view.availableMemory } returns random.nextLong(0, view.host.model.memorySize) + every { view.numberOfActiveServers } returns random.nextInt(0, 6) + every { view.provisionedCores } returns random.nextInt(0, view.host.model.cpuCount) + every { view.toString() } returns "HostView[$i,numberOfActiveServers=${view.numberOfActiveServers}]" + view + } + + val servers = List(2) { + val server = mockk() + every { server.flavor.cpuCount } returns random.nextInt(1, 8) + every { server.flavor.memorySize } returns random.nextLong(1024, 1024 * 512) + server + } + + return Stream.of( + Arguments.of(false, hosts.toSet(), servers[0], hosts[2]), + Arguments.of(false, hosts.toSet(), servers[1], hosts[1]), + Arguments.of(true, hosts.toSet(), servers[1], hosts[0]), + ) + } + + /** + * Test arguments for the [AvailableCoreMemoryAllocationPolicy]. + */ + @JvmStatic + fun availableCoreMemoryArgs(): Stream { + val random = Random(1) + val hosts = List(4) { i -> + val view = mockk() + every { view.host.uid } returns UUID(0, i.toLong()) + every { view.host.model.cpuCount } returns random.nextInt(1, 16) + every { view.host.model.memorySize } returns random.nextLong(1024, 1024 * 1024) + every { view.availableMemory } returns random.nextLong(0, view.host.model.memorySize) + every { view.numberOfActiveServers } returns random.nextInt(0, 6) + every { view.provisionedCores } returns random.nextInt(0, view.host.model.cpuCount) + every { view.toString() } returns "HostView[$i,availableMemory=${view.availableMemory}]" + view + } + + val servers = List(2) { + val server = mockk() + every { server.flavor.cpuCount } returns random.nextInt(1, 8) + every { server.flavor.memorySize } returns random.nextLong(1024, 1024 * 512) + server + } + + return Stream.of( + Arguments.of(false, hosts.toSet(), servers[0], hosts[2]), + Arguments.of(false, hosts.toSet(), servers[1], hosts[2]), + Arguments.of(true, hosts.toSet(), servers[1], hosts[1]), + ) + } + + /** + * Test arguments for the [AvailableMemoryAllocationPolicy]. + */ + @JvmStatic + fun availableMemoryArgs(): Stream { + val random = Random(1) + val hosts = List(4) { i -> + val view = mockk() + every { view.host.uid } returns UUID(0, i.toLong()) + every { view.host.model.cpuCount } returns random.nextInt(1, 16) + every { view.host.model.memorySize } returns random.nextLong(1024, 1024 * 1024) + every { view.availableMemory } returns random.nextLong(0, view.host.model.memorySize) + every { view.numberOfActiveServers } returns random.nextInt(0, 6) + every { view.provisionedCores } returns random.nextInt(0, view.host.model.cpuCount) + every { view.toString() } returns "HostView[$i,availableMemory=${view.availableMemory}]" + view + } + + val servers = List(2) { + val server = mockk() + every { server.flavor.cpuCount } returns random.nextInt(1, 8) + every { server.flavor.memorySize } returns random.nextLong(1024, 1024 * 512) + server + } + + return Stream.of( + Arguments.of(false, hosts.toSet(), servers[0], hosts[2]), + Arguments.of(false, hosts.toSet(), servers[1], hosts[2]), + Arguments.of(true, hosts.toSet(), servers[1], hosts[1]), + ) + } + + /** + * Test arguments for the [ProvisionedCoresAllocationPolicy]. + */ + @JvmStatic + fun provisionedCoresArgs(): Stream { + val random = Random(1) + val hosts = List(4) { i -> + val view = mockk() + every { view.host.uid } returns UUID(0, i.toLong()) + every { view.host.model.cpuCount } returns random.nextInt(1, 16) + every { view.host.model.memorySize } returns random.nextLong(1024, 1024 * 1024) + every { view.availableMemory } returns random.nextLong(0, view.host.model.memorySize) + every { view.numberOfActiveServers } returns random.nextInt(0, 6) + every { view.provisionedCores } returns random.nextInt(0, view.host.model.cpuCount) + every { view.toString() } returns "HostView[$i,provisionedCores=${view.provisionedCores}]" + view + } + + val servers = List(2) { + val server = mockk() + every { server.flavor.cpuCount } returns random.nextInt(1, 8) + every { server.flavor.memorySize } returns random.nextLong(1024, 1024 * 512) + server + } + + return Stream.of( + Arguments.of(false, hosts.toSet(), servers[0], hosts[2]), + Arguments.of(false, hosts.toSet(), servers[1], hosts[0]), + Arguments.of(true, hosts.toSet(), servers[1], hosts[0]), + ) + } + } +} diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml b/simulator/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml new file mode 100644 index 00000000..0dfb75f2 --- /dev/null +++ b/simulator/opendc-compute/opendc-compute-service/src/test/resources/log4j2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + -- cgit v1.2.3 From 0c5f0a07f87f85119be83b923ffe60b3863ebb9d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 25 Mar 2021 14:33:56 +0100 Subject: compute: Hide internals of compute service implementation This change changes the compute service and users of the compute service to not rely on the internals of `ComputeServiceImpl` and instead use its public API. --- .../opendc/compute/service/internal/ComputeServiceImpl.kt | 12 ++++++------ .../src/main/kotlin/org/opendc/compute/simulator/SimHost.kt | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt index 62808b4d..18255922 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt @@ -47,7 +47,7 @@ import kotlin.math.max * @param context The [CoroutineContext] to use. * @param clock The clock instance to keep track of time. */ -public class ComputeServiceImpl( +internal class ComputeServiceImpl( private val context: CoroutineContext, private val clock: Clock, private val tracer: EventTracer, @@ -104,11 +104,11 @@ public class ComputeServiceImpl( */ private val servers = mutableMapOf() - public var submittedVms: Int = 0 - public var queuedVms: Int = 0 - public var runningVms: Int = 0 - public var finishedVms: Int = 0 - public var unscheduledVms: Int = 0 + private var submittedVms: Int = 0 + private var queuedVms: Int = 0 + private var runningVms: Int = 0 + private var finishedVms: Int = 0 + private var unscheduledVms: Int = 0 private var maxCores = 0 private var maxMemory = 0L diff --git a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt index 3c4b4410..2e4191cc 100644 --- a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt @@ -207,6 +207,8 @@ public class SimHost( _state = HostState.DOWN } + override fun toString(): String = "SimHost[uid=$uid,name=$name,model=$model]" + /** * Convert flavor to machine model. */ -- cgit v1.2.3 From 0d66ef47d6e1ec0861b4939800c5070f96600ca0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 25 Mar 2021 14:49:07 +0100 Subject: compute: Remove dependency on event tracer This change removes the dependency on the event tracer in `opendc-trace`, since we are in the process of migrating OpenDC to use OpenTelemetry for distributed tracing and metrics. --- .../opendc-compute-service/build.gradle.kts | 1 - .../org/opendc/compute/service/ComputeService.kt | 5 +--- .../service/events/HypervisorAvailableEvent.kt | 31 --------------------- .../service/events/HypervisorUnavailableEvent.kt | 31 --------------------- .../compute/service/events/VmScheduledEvent.kt | 30 -------------------- .../compute/service/events/VmStoppedEvent.kt | 30 -------------------- .../compute/service/events/VmSubmissionEvent.kt | 32 ---------------------- .../service/events/VmSubmissionInvalidEvent.kt | 30 -------------------- .../compute/service/internal/ComputeServiceImpl.kt | 14 ---------- .../opendc/compute/service/ComputeServiceTest.kt | 4 +-- 10 files changed, 2 insertions(+), 206 deletions(-) delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorAvailableEvent.kt delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorUnavailableEvent.kt delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmScheduledEvent.kt delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmStoppedEvent.kt delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionEvent.kt delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionInvalidEvent.kt (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts index 1825e989..41b506b2 100644 --- a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts +++ b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts @@ -32,7 +32,6 @@ plugins { dependencies { api(platform(project(":opendc-platform"))) api(project(":opendc-compute:opendc-compute-api")) - api(project(":opendc-trace:opendc-trace-core")) implementation(project(":opendc-utils")) implementation("io.github.microutils:kotlin-logging") diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt index 593e4b56..28cef83a 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt @@ -27,7 +27,6 @@ import org.opendc.compute.api.ComputeClient import org.opendc.compute.service.driver.Host import org.opendc.compute.service.internal.ComputeServiceImpl import org.opendc.compute.service.scheduler.AllocationPolicy -import org.opendc.trace.core.EventTracer import java.time.Clock import kotlin.coroutines.CoroutineContext @@ -76,17 +75,15 @@ public interface ComputeService : AutoCloseable { * * @param context The [CoroutineContext] to use in the service. * @param clock The clock instance to use. - * @param tracer The event tracer to use. * @param allocationPolicy The allocation policy to use. */ public operator fun invoke( context: CoroutineContext, clock: Clock, - tracer: EventTracer, allocationPolicy: AllocationPolicy, schedulingQuantum: Long = 300000, ): ComputeService { - return ComputeServiceImpl(context, clock, tracer, allocationPolicy, schedulingQuantum) + return ComputeServiceImpl(context, clock, allocationPolicy, schedulingQuantum) } } } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorAvailableEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorAvailableEvent.kt deleted file mode 100644 index a7974062..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorAvailableEvent.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.trace.core.Event -import java.util.* - -/** - * This event is emitted when a hypervisor has become available. - */ -public class HypervisorAvailableEvent(public val uid: UUID) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorUnavailableEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorUnavailableEvent.kt deleted file mode 100644 index 75bb09ed..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/HypervisorUnavailableEvent.kt +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.trace.core.Event -import java.util.* - -/** - * This event is emitted when a hypervisor has become unavailable. - */ -public class HypervisorUnavailableEvent(public val uid: UUID) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmScheduledEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmScheduledEvent.kt deleted file mode 100644 index f59c74b7..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmScheduledEvent.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.trace.core.Event - -/** - * This event is emitted when a virtual machine has successfully been scheduled on a hypervisor. - */ -public class VmScheduledEvent(public val name: String) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmStoppedEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmStoppedEvent.kt deleted file mode 100644 index eaf0736b..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmStoppedEvent.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.trace.core.Event - -/** - * This event is emitted when a virtual machine has stopped running. - */ -public class VmStoppedEvent(public val name: String) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionEvent.kt deleted file mode 100644 index fa0a8a13..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionEvent.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.compute.api.Flavor -import org.opendc.compute.api.Image -import org.opendc.trace.core.Event - -/** - * This event is emitted when a virtual machine is submitted to the provisioning service. - */ -public class VmSubmissionEvent(public val name: String, public val image: Image, public val flavor: Flavor) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionInvalidEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionInvalidEvent.kt deleted file mode 100644 index 52b91616..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/events/VmSubmissionInvalidEvent.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.events - -import org.opendc.trace.core.Event - -/** - * An event that is emitted when the submission is deemed to be invalid. - */ -public class VmSubmissionInvalidEvent(public val name: String) : Event() diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt index 18255922..f9bd7fbc 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt @@ -31,9 +31,7 @@ import org.opendc.compute.service.ComputeServiceEvent import org.opendc.compute.service.driver.Host import org.opendc.compute.service.driver.HostListener import org.opendc.compute.service.driver.HostState -import org.opendc.compute.service.events.* import org.opendc.compute.service.scheduler.AllocationPolicy -import org.opendc.trace.core.EventTracer import org.opendc.utils.TimerScheduler import org.opendc.utils.flow.EventFlow import java.time.Clock @@ -50,7 +48,6 @@ import kotlin.math.max internal class ComputeServiceImpl( private val context: CoroutineContext, private val clock: Clock, - private val tracer: EventTracer, private val allocationPolicy: AllocationPolicy, private val schedulingQuantum: Long ) : ComputeService, HostListener { @@ -207,8 +204,6 @@ internal class ComputeServiceImpl( start: Boolean ): Server { check(!isClosed) { "Client is closed" } - tracer.commit(VmSubmissionEvent(name, image, flavor)) - _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, @@ -346,8 +341,6 @@ internal class ComputeServiceImpl( logger.trace { "Server $server selected for scheduling but no capacity available for it at the moment" } if (server.flavor.memorySize > maxMemory || server.flavor.cpuCount > maxCores) { - tracer.commit(VmSubmissionInvalidEvent(server.name)) - _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, @@ -392,7 +385,6 @@ internal class ComputeServiceImpl( host.spawn(server) activeServers[server] = host - tracer.commit(VmScheduledEvent(server.name)) _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, @@ -437,8 +429,6 @@ internal class ComputeServiceImpl( availableHosts += hv } - tracer.commit(HypervisorAvailableEvent(host.uid)) - _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, @@ -461,8 +451,6 @@ internal class ComputeServiceImpl( val hv = hostToView[host] ?: return availableHosts -= hv - tracer.commit(HypervisorUnavailableEvent(hv.uid)) - _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, @@ -495,8 +483,6 @@ internal class ComputeServiceImpl( if (newState == ServerState.TERMINATED || newState == ServerState.DELETED) { logger.info { "[${clock.millis()}] Server ${server.uid} ${server.name} ${server.flavor} finished." } - tracer.commit(VmStoppedEvent(server.name)) - _events.emit( ComputeServiceEvent.MetricsAvailable( this@ComputeServiceImpl, diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt index ffec92ea..e1482152 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt @@ -40,7 +40,6 @@ import org.opendc.compute.service.driver.HostModel import org.opendc.compute.service.driver.HostState import org.opendc.compute.service.scheduler.AvailableMemoryAllocationPolicy import org.opendc.simulator.utils.DelayControllerClockAdapter -import org.opendc.trace.core.EventTracer import java.util.* /** @@ -55,9 +54,8 @@ internal class ComputeServiceTest { fun setUp() { scope = TestCoroutineScope() val clock = DelayControllerClockAdapter(scope) - val tracer = EventTracer(clock) val policy = AvailableMemoryAllocationPolicy() - service = ComputeService(scope.coroutineContext, clock, tracer, policy) + service = ComputeService(scope.coroutineContext, clock, policy) } @AfterEach -- cgit v1.2.3 From 608ff59b2d7e8ce696fe6f7271d80b5efc9c4b87 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 25 Mar 2021 21:50:45 +0100 Subject: compute: Integrate OpenTelemetry Metrics in OpenDC Compute This change integrates the OpenTelemetry Metrics API in the OpenDC Compute Service implementation. This replaces the old infrastructure for gathering metrics. --- .../opendc-compute-service/build.gradle.kts | 1 + .../org/opendc/compute/service/ComputeService.kt | 9 +- .../opendc/compute/service/ComputeServiceEvent.kt | 47 ------ .../compute/service/internal/ComputeServiceImpl.kt | 174 ++++++++++----------- .../opendc/compute/service/ComputeServiceTest.kt | 4 +- .../kotlin/org/opendc/compute/simulator/SimHost.kt | 2 +- 6 files changed, 90 insertions(+), 147 deletions(-) delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeServiceEvent.kt (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts index 41b506b2..909e2dcd 100644 --- a/simulator/opendc-compute/opendc-compute-service/build.gradle.kts +++ b/simulator/opendc-compute/opendc-compute-service/build.gradle.kts @@ -32,6 +32,7 @@ plugins { dependencies { api(platform(project(":opendc-platform"))) api(project(":opendc-compute:opendc-compute-api")) + api(project(":opendc-telemetry:opendc-telemetry-api")) implementation(project(":opendc-utils")) implementation("io.github.microutils:kotlin-logging") diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt index 28cef83a..4bc0ba78 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt @@ -22,6 +22,7 @@ package org.opendc.compute.service +import io.opentelemetry.api.metrics.Meter import kotlinx.coroutines.flow.Flow import org.opendc.compute.api.ComputeClient import org.opendc.compute.service.driver.Host @@ -34,11 +35,6 @@ import kotlin.coroutines.CoroutineContext * The [ComputeService] hosts the API implementation of the OpenDC Compute service. */ public interface ComputeService : AutoCloseable { - /** - * The events emitted by the service. - */ - public val events: Flow - /** * The hosts that are used by the compute service. */ @@ -80,10 +76,11 @@ public interface ComputeService : AutoCloseable { public operator fun invoke( context: CoroutineContext, clock: Clock, + meter: Meter, allocationPolicy: AllocationPolicy, schedulingQuantum: Long = 300000, ): ComputeService { - return ComputeServiceImpl(context, clock, allocationPolicy, schedulingQuantum) + return ComputeServiceImpl(context, clock, meter, allocationPolicy, schedulingQuantum) } } } diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeServiceEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeServiceEvent.kt deleted file mode 100644 index 193008a7..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeServiceEvent.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service - -/** - * An event that is emitted by the [ComputeService]. - */ -public sealed class ComputeServiceEvent { - /** - * The service that has emitted the event. - */ - public abstract val provisioner: ComputeService - - /** - * An event emitted for writing metrics. - */ - public data class MetricsAvailable( - override val provisioner: ComputeService, - public val totalHostCount: Int, - public val availableHostCount: Int, - public val totalVmCount: Int, - public val activeVmCount: Int, - public val inactiveVmCount: Int, - public val waitingVmCount: Int, - public val failedVmCount: Int - ) : ComputeServiceEvent() -} diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt index f9bd7fbc..26a34ad9 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt @@ -22,18 +22,16 @@ package org.opendc.compute.service.internal +import io.opentelemetry.api.metrics.Meter import kotlinx.coroutines.* -import kotlinx.coroutines.flow.Flow import mu.KotlinLogging import org.opendc.compute.api.* import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.ComputeServiceEvent import org.opendc.compute.service.driver.Host import org.opendc.compute.service.driver.HostListener import org.opendc.compute.service.driver.HostState import org.opendc.compute.service.scheduler.AllocationPolicy import org.opendc.utils.TimerScheduler -import org.opendc.utils.flow.EventFlow import java.time.Clock import java.util.* import kotlin.coroutines.CoroutineContext @@ -48,6 +46,7 @@ import kotlin.math.max internal class ComputeServiceImpl( private val context: CoroutineContext, private val clock: Clock, + private val meter: Meter, private val allocationPolicy: AllocationPolicy, private val schedulingQuantum: Long ) : ComputeService, HostListener { @@ -101,24 +100,70 @@ internal class ComputeServiceImpl( */ private val servers = mutableMapOf() - private var submittedVms: Int = 0 - private var queuedVms: Int = 0 - private var runningVms: Int = 0 - private var finishedVms: Int = 0 - private var unscheduledVms: Int = 0 - private var maxCores = 0 private var maxMemory = 0L + /** + * The number of servers that have been submitted to the service for provisioning. + */ + private val _submittedServers = meter.longCounterBuilder("servers.submitted") + .setDescription("Number of start requests") + .setUnit("1") + .build() + + /** + * The number of servers that failed to be scheduled. + */ + private val _unscheduledServers = meter.longCounterBuilder("servers.unscheduled") + .setDescription("Number of unscheduled servers") + .setUnit("1") + .build() + + /** + * The number of servers that are waiting to be provisioned. + */ + private val _waitingServers = meter.longUpDownCounterBuilder("servers.waiting") + .setDescription("Number of servers waiting to be provisioned") + .setUnit("1") + .build() + + /** + * The number of servers that are waiting to be provisioned. + */ + private val _runningServers = meter.longUpDownCounterBuilder("servers.active") + .setDescription("Number of servers currently running") + .setUnit("1") + .build() + + /** + * The number of servers that have finished running. + */ + private val _finishedServers = meter.longCounterBuilder("servers.finished") + .setDescription("Number of servers that finished running") + .setUnit("1") + .build() + + /** + * The number of hosts registered at the compute service. + */ + private val _hostCount = meter.longUpDownCounterBuilder("hosts.total") + .setDescription("Number of hosts") + .setUnit("1") + .build() + + /** + * The number of available hosts registered at the compute service. + */ + private val _availableHostCount = meter.longUpDownCounterBuilder("hosts.available") + .setDescription("Number of available hosts") + .setUnit("1") + .build() + /** * The allocation logic to use. */ private val allocationLogic = allocationPolicy() - override val events: Flow - get() = _events - private val _events = EventFlow() - /** * The [TimerScheduler] to use for scheduling the scheduler cycles. */ @@ -204,18 +249,6 @@ internal class ComputeServiceImpl( start: Boolean ): Server { check(!isClosed) { "Client is closed" } - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - ++submittedVms, - runningVms, - finishedVms, - ++queuedVms, - unscheduledVms - ) - ) val uid = UUID(clock.millis(), random.nextLong()) val server = InternalServer( @@ -269,14 +302,23 @@ internal class ComputeServiceImpl( hostToView[host] = hv if (host.state == HostState.UP) { + _availableHostCount.add(1) availableHosts += hv } + _hostCount.add(1) host.addListener(this) } override fun removeHost(host: Host) { - host.removeListener(this) + val view = hostToView.remove(host) + if (view != null) { + if (availableHosts.remove(view)) { + _availableHostCount.add(-1) + } + host.removeListener(this) + _hostCount.add(-1) + } } override fun close() { @@ -288,6 +330,8 @@ internal class ComputeServiceImpl( val request = SchedulingRequest(server) queue.add(request) + _submittedServers.add(1) + _waitingServers.add(1) requestSchedulingCycle() return request } @@ -332,6 +376,7 @@ internal class ComputeServiceImpl( if (request.isCancelled) { queue.poll() + _waitingServers.add(-1) continue } @@ -341,21 +386,10 @@ internal class ComputeServiceImpl( logger.trace { "Server $server selected for scheduling but no capacity available for it at the moment" } if (server.flavor.memorySize > maxMemory || server.flavor.cpuCount > maxCores) { - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - runningVms, - finishedVms, - --queuedVms, - ++unscheduledVms - ) - ) - // Remove the incoming image queue.poll() + _waitingServers.add(-1) + _unscheduledServers.add(1) logger.warn("Failed to spawn $server: does not fit [${clock.millis()}]") @@ -370,6 +404,7 @@ internal class ComputeServiceImpl( // Remove request from queue queue.poll() + _waitingServers.add(-1) logger.info { "Assigned server $server to host $host." } @@ -384,19 +419,6 @@ internal class ComputeServiceImpl( server.host = host host.spawn(server) activeServers[server] = host - - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - ++runningVms, - finishedVms, - --queuedVms, - unscheduledVms - ) - ) } catch (e: Throwable) { logger.error("Failed to deploy VM", e) @@ -427,21 +449,9 @@ internal class ComputeServiceImpl( if (hv != null) { // Corner case for when the hypervisor already exists availableHosts += hv + _availableHostCount.add(1) } - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - runningVms, - finishedVms, - queuedVms, - unscheduledVms - ) - ) - // Re-schedule on the new machine requestSchedulingCycle() } @@ -450,19 +460,7 @@ internal class ComputeServiceImpl( val hv = hostToView[host] ?: return availableHosts -= hv - - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - runningVms, - finishedVms, - queuedVms, - unscheduledVms - ) - ) + _availableHostCount.add(-1) requestSchedulingCycle() } @@ -480,23 +478,15 @@ internal class ComputeServiceImpl( server.state = newState - if (newState == ServerState.TERMINATED || newState == ServerState.DELETED) { + if (newState == ServerState.RUNNING) { + _runningServers.add(1) + } else if (newState == ServerState.TERMINATED || newState == ServerState.DELETED) { logger.info { "[${clock.millis()}] Server ${server.uid} ${server.name} ${server.flavor} finished." } - _events.emit( - ComputeServiceEvent.MetricsAvailable( - this@ComputeServiceImpl, - hostCount, - availableHosts.size, - submittedVms, - --runningVms, - ++finishedVms, - queuedVms, - unscheduledVms - ) - ) - activeServers -= server + _runningServers.add(-1) + _finishedServers.add(1) + val hv = hostToView[host] if (hv != null) { hv.provisionedCores -= server.flavor.cpuCount diff --git a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt index e1482152..45a306aa 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt @@ -23,6 +23,7 @@ package org.opendc.compute.service import io.mockk.* +import io.opentelemetry.api.metrics.MeterProvider import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.delay import kotlinx.coroutines.test.TestCoroutineScope @@ -55,7 +56,8 @@ internal class ComputeServiceTest { scope = TestCoroutineScope() val clock = DelayControllerClockAdapter(scope) val policy = AvailableMemoryAllocationPolicy() - service = ComputeService(scope.coroutineContext, clock, policy) + val meter = MeterProvider.noop().get("opendc-compute") + service = ComputeService(scope.coroutineContext, clock, meter, policy) } @AfterEach diff --git a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt index 2e4191cc..89784803 100644 --- a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt @@ -59,7 +59,7 @@ public class SimHost( /** * The [CoroutineScope] of the host bounded by the lifecycle of the host. */ - override val scope: CoroutineScope = CoroutineScope(context) + override val scope: CoroutineScope = CoroutineScope(context + Job()) /** * The logger instance of this server. -- cgit v1.2.3 From be95e0b10b648e256c4eb8c2190ad6ed61f8afdd Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 26 Mar 2021 15:36:44 +0100 Subject: workflow: Remove event tracer from workflow service This change removes the event tracer from the OpenDC Workflow service as we start migrating to the industry standard OpenTelemetry. --- .../src/main/kotlin/org/opendc/compute/service/ComputeService.kt | 1 - 1 file changed, 1 deletion(-) (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt index 4bc0ba78..98566da3 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt @@ -23,7 +23,6 @@ package org.opendc.compute.service import io.opentelemetry.api.metrics.Meter -import kotlinx.coroutines.flow.Flow import org.opendc.compute.api.ComputeClient import org.opendc.compute.service.driver.Host import org.opendc.compute.service.internal.ComputeServiceImpl -- cgit v1.2.3 From 5b0eaf76ec00192c755b268b7655f6463c5bc62f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 27 Mar 2021 12:33:13 +0100 Subject: compute: Migrate compute service simulator to OpenTelemetry This change updates the compute service simulator to use OpenTelemetry for reporting metrics of the (simulated) hosts as opposed to using custom event flows. This approach is more generic, flexible and possibly offers better performance as we can collect metrics of all services in a single sweep, as opposed to listening to several services and each invoking the handlers. --- .../org/opendc/compute/service/driver/Host.kt | 6 - .../org/opendc/compute/service/driver/HostEvent.kt | 72 ----------- .../opendc-compute-simulator/build.gradle.kts | 1 + .../kotlin/org/opendc/compute/simulator/SimHost.kt | 117 +++++++++++++---- .../org/opendc/compute/simulator/SimHostTest.kt | 142 ++++++++++++--------- 5 files changed, 180 insertions(+), 158 deletions(-) delete mode 100644 simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/HostEvent.kt (limited to 'simulator/opendc-compute') diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/Host.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/Host.kt index c3c39572..bed15dfd 100644 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/Host.kt +++ b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/Host.kt @@ -22,7 +22,6 @@ package org.opendc.compute.service.driver -import kotlinx.coroutines.flow.Flow import org.opendc.compute.api.Server import java.util.* @@ -55,11 +54,6 @@ public interface Host { */ public val meta: Map - /** - * The events emitted by the driver. - */ - public val events: Flow - /** * Determine whether the specified [instance][server] can still fit on this host. */ diff --git a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/HostEvent.kt b/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/HostEvent.kt deleted file mode 100644 index 97350679..00000000 --- a/simulator/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/driver/HostEvent.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.service.driver - -/** - * An event that is emitted by a [Host]. - */ -public sealed class HostEvent { - /** - * The driver that emitted the event. - */ - public abstract val driver: Host - - /** - * This event is emitted when the number of active servers on the server managed by this driver is updated. - * - * @property driver The driver that emitted the event. - * @property numberOfActiveServers The number of active servers. - * @property availableMemory The available memory, in MB. - */ - public data class VmsUpdated( - override val driver: Host, - public val numberOfActiveServers: Int, - public val availableMemory: Long - ) : HostEvent() - - /** - * This event is emitted when a slice is finished. - * - * @property driver The driver that emitted the event. - * @property requestedBurst The total requested CPU time (can be above capacity). - * @property grantedBurst The actual total granted capacity, which might be lower than the requested burst due to - * the hypervisor being interrupted during a slice. - * @property overcommissionedBurst The CPU time that the hypervisor could not grant to the virtual machine since - * it did not have the capacity. - * @property interferedBurst The sum of CPU time that virtual machines could not utilize due to performance - * interference. - * @property cpuUsage CPU use in megahertz. - * @property cpuDemand CPU demand in megahertz. - * @property numberOfDeployedImages The number of images deployed on this hypervisor. - */ - public data class SliceFinished( - override val driver: Host, - public val requestedBurst: Long, - public val grantedBurst: Long, - public val overcommissionedBurst: Long, - public val interferedBurst: Long, - public val cpuUsage: Double, - public val cpuDemand: Double, - public val numberOfDeployedImages: Int, - ) : HostEvent() -} diff --git a/simulator/opendc-compute/opendc-compute-simulator/build.gradle.kts b/simulator/opendc-compute/opendc-compute-simulator/build.gradle.kts index 1ad3f1c6..3bf8a114 100644 --- a/simulator/opendc-compute/opendc-compute-simulator/build.gradle.kts +++ b/simulator/opendc-compute/opendc-compute-simulator/build.gradle.kts @@ -38,5 +38,6 @@ dependencies { implementation("io.github.microutils:kotlin-logging") testImplementation(project(":opendc-simulator:opendc-simulator-core")) + testImplementation(project(":opendc-telemetry:opendc-telemetry-sdk")) testRuntimeOnly("org.slf4j:slf4j-simple:${versions.slf4j}") } diff --git a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt index 89784803..6d81aa7d 100644 --- a/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ b/simulator/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt @@ -22,8 +22,9 @@ package org.opendc.compute.simulator +import io.opentelemetry.api.metrics.Meter +import io.opentelemetry.api.metrics.common.Labels import kotlinx.coroutines.* -import kotlinx.coroutines.flow.Flow import mu.KotlinLogging import org.opendc.compute.api.Flavor import org.opendc.compute.api.Server @@ -36,7 +37,6 @@ import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.power.ConstantPowerModel import org.opendc.simulator.compute.power.MachinePowerModel import org.opendc.simulator.failures.FailureDomain -import org.opendc.utils.flow.EventFlow import java.time.Clock import java.util.* import kotlin.coroutines.CoroutineContext @@ -52,6 +52,7 @@ public class SimHost( override val meta: Map, context: CoroutineContext, clock: Clock, + meter: Meter, hypervisor: SimHypervisorProvider, powerModel: MachinePowerModel = ConstantPowerModel(0.0), private val mapper: SimWorkloadMapper = SimMetaWorkloadMapper(), @@ -66,10 +67,6 @@ public class SimHost( */ private val logger = KotlinLogging.logger {} - override val events: Flow - get() = _events - internal val _events = EventFlow() - /** * The event listeners registered with this host. */ @@ -99,18 +96,13 @@ public class SimHost( cpuUsage: Double, cpuDemand: Double ) { - _events.emit( - HostEvent.SliceFinished( - this@SimHost, - requestedWork, - grantedWork, - overcommittedWork, - interferedWork, - cpuUsage, - cpuDemand, - guests.size - ) - ) + _cpuWork.record(requestedWork.toDouble()) + _cpuWorkGranted.record(grantedWork.toDouble()) + _cpuWorkOvercommit.record(overcommittedWork.toDouble()) + _cpuWorkInterference.record(interferedWork.toDouble()) + _cpuUsage.record(cpuUsage) + _cpuDemand.record(cpuDemand) + _cpuPower.record(machine.powerDraw.value) } } ) @@ -132,6 +124,87 @@ public class SimHost( override val model: HostModel = HostModel(model.cpus.size, model.memory.map { it.size }.sum()) + /** + * The number of guests on the host. + */ + private val _guests = meter.longUpDownCounterBuilder("guests.total") + .setDescription("Number of guests") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The number of active guests on the host. + */ + private val _activeGuests = meter.longUpDownCounterBuilder("guests.active") + .setDescription("Number of active guests") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The CPU usage on the host. + */ + private val _cpuUsage = meter.doubleValueRecorderBuilder("cpu.usage") + .setDescription("The amount of CPU resources used by the host") + .setUnit("MHz") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The CPU demand on the host. + */ + private val _cpuDemand = meter.doubleValueRecorderBuilder("cpu.demand") + .setDescription("The amount of CPU resources the guests would use if there were no CPU contention or CPU limits") + .setUnit("MHz") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The requested work for the CPU. + */ + private val _cpuPower = meter.doubleValueRecorderBuilder("power.usage") + .setDescription("The amount of power used by the CPU") + .setUnit("W") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The requested work for the CPU. + */ + private val _cpuWork = meter.doubleValueRecorderBuilder("cpu.work.total") + .setDescription("The amount of work supplied to the CPU") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The work actually performed by the CPU. + */ + private val _cpuWorkGranted = meter.doubleValueRecorderBuilder("cpu.work.granted") + .setDescription("The amount of work performed by the CPU") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The work that could not be performed by the CPU due to overcommitting resource. + */ + private val _cpuWorkOvercommit = meter.doubleValueRecorderBuilder("cpu.work.overcommit") + .setDescription("The amount of work not performed by the CPU due to overcommitment") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + + /** + * The work that could not be performed by the CPU due to interference. + */ + private val _cpuWorkInterference = meter.doubleValueRecorderBuilder("cpu.work.interference") + .setDescription("The amount of work not performed by the CPU due to interference") + .setUnit("1") + .build() + .bind(Labels.of("host", uid.toString())) + init { // Launch hypervisor onto machine scope.launch { @@ -166,12 +239,11 @@ public class SimHost( require(canFit(server)) { "Server does not fit" } val guest = Guest(server, hypervisor.createMachine(server.flavor.toMachineModel())) guests[server] = guest + _guests.add(1) if (start) { guest.start() } - - _events.emit(HostEvent.VmsUpdated(this, guests.count { it.value.state == ServerState.RUNNING }, availableMemory)) } override fun contains(server: Server): Boolean { @@ -191,6 +263,7 @@ public class SimHost( override suspend fun delete(server: Server) { val guest = guests.remove(server) ?: return guest.terminate() + _guests.add(-1) } override fun addListener(listener: HostListener) { @@ -228,6 +301,7 @@ public class SimHost( } } + _activeGuests.add(1) listeners.forEach { it.onStateChanged(this, vm.server, vm.state) } } @@ -238,9 +312,8 @@ public class SimHost( } } + _activeGuests.add(-1) listeners.forEach { it.onStateChanged(this, vm.server, vm.state) } - - _events.emit(HostEvent.VmsUpdated(this@SimHost, guests.count { it.value.state == ServerState.RUNNING }, availableMemory)) } override suspend fun fail() { diff --git a/simulator/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt b/simulator/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt index e311cd21..830fc868 100644 --- a/simulator/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt +++ b/simulator/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/SimHostTest.kt @@ -22,12 +22,14 @@ package org.opendc.compute.simulator -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.launch -import kotlinx.coroutines.test.TestCoroutineScope +import io.opentelemetry.api.metrics.MeterProvider +import io.opentelemetry.sdk.common.CompletableResultCode +import io.opentelemetry.sdk.metrics.SdkMeterProvider +import io.opentelemetry.sdk.metrics.data.MetricData +import io.opentelemetry.sdk.metrics.export.MetricExporter +import io.opentelemetry.sdk.metrics.export.MetricProducer +import kotlinx.coroutines.* +import kotlinx.coroutines.test.runBlockingTest import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -37,7 +39,8 @@ import org.opendc.compute.api.Image import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState import org.opendc.compute.api.ServerWatcher -import org.opendc.compute.service.driver.HostEvent +import org.opendc.compute.service.driver.Host +import org.opendc.compute.service.driver.HostListener import org.opendc.simulator.compute.SimFairShareHypervisorProvider import org.opendc.simulator.compute.SimMachineModel import org.opendc.simulator.compute.model.MemoryUnit @@ -45,23 +48,20 @@ import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.workload.SimTraceWorkload import org.opendc.simulator.utils.DelayControllerClockAdapter -import java.time.Clock +import org.opendc.telemetry.sdk.metrics.export.CoroutineMetricReader +import org.opendc.telemetry.sdk.toOtelClock import java.util.UUID +import kotlin.coroutines.resume /** * Basic test-suite for the hypervisor. */ @OptIn(ExperimentalCoroutinesApi::class) internal class SimHostTest { - private lateinit var scope: TestCoroutineScope - private lateinit var clock: Clock private lateinit var machineModel: SimMachineModel @BeforeEach fun setUp() { - scope = TestCoroutineScope() - clock = DelayControllerClockAdapter(scope) - val cpuNode = ProcessingNode("Intel", "Xeon", "amd64", 2) machineModel = SimMachineModel( @@ -74,72 +74,98 @@ internal class SimHostTest { * Test overcommitting of resources by the hypervisor. */ @Test - fun testOvercommitted() { + fun testOvercommitted() = runBlockingTest { + val clock = DelayControllerClockAdapter(this) var requestedWork = 0L var grantedWork = 0L var overcommittedWork = 0L - scope.launch { - val virtDriver = SimHost(UUID.randomUUID(), "test", machineModel, emptyMap(), coroutineContext, clock, SimFairShareHypervisorProvider()) - val duration = 5 * 60L - val vmImageA = MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to SimTraceWorkload( - sequenceOf( - SimTraceWorkload.Fragment(duration * 1000, 28.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 3500.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 0.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 183.0, 2) - ), - ) + val meterProvider: MeterProvider = SdkMeterProvider + .builder() + .setClock(clock.toOtelClock()) + .build() + + val virtDriver = SimHost(UUID.randomUUID(), "test", machineModel, emptyMap(), coroutineContext, clock, meterProvider.get("opendc-compute-simulator"), SimFairShareHypervisorProvider()) + val duration = 5 * 60L + val vmImageA = MockImage( + UUID.randomUUID(), + "", + emptyMap(), + mapOf( + "workload" to SimTraceWorkload( + sequenceOf( + SimTraceWorkload.Fragment(duration * 1000, 28.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 3500.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 0.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 183.0, 2) + ), ) ) - val vmImageB = MockImage( - UUID.randomUUID(), - "", - emptyMap(), - mapOf( - "workload" to SimTraceWorkload( - sequenceOf( - SimTraceWorkload.Fragment(duration * 1000, 28.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 3100.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 0.0, 2), - SimTraceWorkload.Fragment(duration * 1000, 73.0, 2) - ) + ) + val vmImageB = MockImage( + UUID.randomUUID(), + "", + emptyMap(), + mapOf( + "workload" to SimTraceWorkload( + sequenceOf( + SimTraceWorkload.Fragment(duration * 1000, 28.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 3100.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 0.0, 2), + SimTraceWorkload.Fragment(duration * 1000, 73.0, 2) ) ) ) + ) - delay(5) - - val flavor = MockFlavor(2, 0) - virtDriver.events - .onEach { event -> - when (event) { - is HostEvent.SliceFinished -> { - requestedWork += event.requestedBurst - grantedWork += event.grantedBurst - overcommittedWork += event.overcommissionedBurst - } - } + val flavor = MockFlavor(2, 0) + + // Setup metric reader + val reader = CoroutineMetricReader( + this, listOf(meterProvider as MetricProducer), + object : MetricExporter { + override fun export(metrics: Collection): CompletableResultCode { + val metricsByName = metrics.associateBy { it.name } + requestedWork += metricsByName.getValue("cpu.work.total").doubleSummaryData.points.first().sum.toLong() + grantedWork += metricsByName.getValue("cpu.work.granted").doubleSummaryData.points.first().sum.toLong() + overcommittedWork += metricsByName.getValue("cpu.work.overcommit").doubleSummaryData.points.first().sum.toLong() + return CompletableResultCode.ofSuccess() } - .launchIn(this) + override fun flush(): CompletableResultCode = CompletableResultCode.ofSuccess() + + override fun shutdown(): CompletableResultCode = CompletableResultCode.ofSuccess() + }, + exportInterval = duration * 1000 + ) + + coroutineScope { launch { virtDriver.spawn(MockServer(UUID.randomUUID(), "a", flavor, vmImageA)) } launch { virtDriver.spawn(MockServer(UUID.randomUUID(), "b", flavor, vmImageB)) } + + suspendCancellableCoroutine { cont -> + virtDriver.addListener(object : HostListener { + private var finished = 0 + + override fun onStateChanged(host: Host, server: Server, newState: ServerState) { + if (newState == ServerState.TERMINATED && ++finished == 2) { + cont.resume(Unit) + } + } + }) + } } - scope.advanceUntilIdle() + // Ensure last cycle is collected + delay(1000 * duration) + virtDriver.close() + reader.close() assertAll( - { assertEquals(emptyList(), scope.uncaughtExceptions, "No errors") }, { assertEquals(4197600, requestedWork, "Requested work does not match") }, { assertEquals(2157600, grantedWork, "Granted work does not match") }, { assertEquals(2040000, overcommittedWork, "Overcommitted work does not match") }, - { assertEquals(1200006, scope.currentTime) } + { assertEquals(1500001, currentTime) } ) } -- cgit v1.2.3