From c9cd6bb12eee73562ed9078f01aa041c7f5ed8ae Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 15 Mar 2020 17:20:08 +0100 Subject: feat: Make FaultInjector more generic --- .../compute/metal/driver/SimpleBareMetalDriver.kt | 5 +- .../metal/driver/SimpleBareMetalDriverTest.kt | 7 ++- .../atlarge/opendc/core/failure/FailureDomain.kt | 7 +++ .../atlarge/opendc/core/failure/FailureInjector.kt | 60 ---------------------- .../atlarge/opendc/core/failure/FaultInjector.kt | 35 +++++++++++++ .../core/failure/UncorrelatedFaultInjector.kt | 48 +++++++++++++++++ 6 files changed, 97 insertions(+), 65 deletions(-) delete mode 100644 opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureInjector.kt create mode 100644 opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FaultInjector.kt create mode 100644 opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt (limited to 'opendc') diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt index fbc5c0ce..4fe8d740 100644 --- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt @@ -43,8 +43,8 @@ import com.atlarge.opendc.compute.metal.monitor.NodeMonitor import com.atlarge.opendc.compute.metal.power.ConstantPowerModel import com.atlarge.opendc.core.power.PowerModel import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.launch @@ -275,6 +275,9 @@ public class SimpleBareMetalDriver( } } + override val scope: CoroutineScope + get() = domain + override suspend fun fail() { withContext(domain.coroutineContext) { serverContext?.cancel(fail = true) diff --git a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt index 1a2440c2..9378b5d7 100644 --- a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt +++ b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt @@ -34,8 +34,7 @@ import com.atlarge.opendc.compute.core.image.FlopsApplicationImage import com.atlarge.opendc.compute.metal.Node import com.atlarge.opendc.compute.metal.NodeState import com.atlarge.opendc.compute.metal.monitor.NodeMonitor -import com.atlarge.opendc.core.failure.FailureInjector -import kotlinx.coroutines.channels.Channel +import com.atlarge.opendc.core.failure.FaultInjector import kotlinx.coroutines.launch import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext @@ -80,8 +79,8 @@ internal class SimpleBareMetalDriverTest { } - val injector = FailureInjector(listOf(driver)) - injector() + val injector = FaultInjector() + injector.enqueue(driver) } runBlocking { diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt index b1c7ccd3..91ca9b83 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt @@ -24,10 +24,17 @@ package com.atlarge.opendc.core.failure +import kotlinx.coroutines.CoroutineScope + /** * A logical or physical component in a computing environment which may fail. */ public interface FailureDomain { + /** + * The lifecycle of the failure domain to which a [FaultInjector] will attach. + */ + public val scope: CoroutineScope + /** * Fail the domain externally. */ diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureInjector.kt deleted file mode 100644 index 456e18bb..00000000 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureInjector.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.atlarge.opendc.core.failure - -import kotlinx.coroutines.delay -import kotlinx.coroutines.isActive -import java.util.Random -import kotlin.coroutines.coroutineContext -import kotlin.math.ln -import kotlin.random.asKotlinRandom - -/** - * An entity that injects failures into a system. - * - * @param failureDomains The failure domains to be included. - */ -public class FailureInjector(private val failureDomains: List) { - /** - * The [Random] instance to generate the failures. - */ - private val random = Random() - - /** - * Start the failure injector process. - */ - public suspend operator fun invoke() { - val targets = HashSet(failureDomains) - val mu = 20.0 - while (targets.isNotEmpty() && coroutineContext.isActive) { - delay(random.expovariate(mu)) - val target = targets.random(random.asKotlinRandom()) - targets -= target - target.fail() - } - } - - private fun Random.expovariate(mu: Double) = (-mu * ln(1 - nextDouble())).toLong() -} diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FaultInjector.kt new file mode 100644 index 00000000..ac7a08de --- /dev/null +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FaultInjector.kt @@ -0,0 +1,35 @@ +/* + * MIT License + * + * Copyright (c) 2020 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.core.failure + +/** + * An interface for stochastically injecting faults into a running system. + */ +public interface FaultInjector { + /** + * Enqueue the specified [FailureDomain] into the queue as candidate for failure injection in the future. + */ + public fun enqueue(domain: FailureDomain) +} diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt new file mode 100644 index 00000000..95127deb --- /dev/null +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2020 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.core.failure + +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlin.math.ln +import kotlin.random.Random + +/** + * A [FaultInjector] that injects uncorrelated faults into the system, meaning that failures of the subsystems are + * independent. + */ +public class UncorrelatedFaultInjector(private val mu: Double = 256.0, private val random: Random = Random.Default) : FaultInjector { + /** + * Enqueue the specified [FailureDomain] to fail some time in the future. + */ + override fun enqueue(domain: FailureDomain) { + domain.scope.launch { + delay(random.expovariate(mu)) + domain.fail() + } + } + + private fun Random.expovariate(mu: Double) = (-mu * ln(1 - nextDouble())).toLong() +} -- cgit v1.2.3