diff options
| author | Dante Niewenhuis <d.niewenhuis@hotmail.com> | 2024-05-07 12:33:39 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-05-07 12:33:39 +0200 |
| commit | ad20465a5df47b49561bb0afbdda5cd65c5da4b8 (patch) | |
| tree | 268f0fde5924b71ca2750dbbbba4cbe24c361f12 /opendc-compute/opendc-compute-simulator/src | |
| parent | 7c0691eb6c348d2e49da3ef354b652cf26604905 (diff) | |
Revamped failure models (#228)
Diffstat (limited to 'opendc-compute/opendc-compute-simulator/src')
13 files changed, 16 insertions, 622 deletions
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt index 84799123..00f2acb3 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/SimHost.kt @@ -190,6 +190,9 @@ public class SimHost( override fun delete(server: Server) { val guest = guests[server] ?: return guest.delete() + + guests.remove(server) + localGuests.remove(guest) } override fun addListener(listener: HostListener) { diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/VictimSelector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt index b5610284..781c2025 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/VictimSelector.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/checkpoints/CheckpointModel.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 AtLarge Research + * Copyright (c) 2024 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,16 +20,9 @@ * SOFTWARE. */ -package org.opendc.compute.simulator.failure +package org.opendc.compute.simulator.checkpoints -import org.opendc.compute.simulator.SimHost - -/** - * Interface responsible for selecting the victim(s) for fault injection. - */ -public interface VictimSelector { - /** - * Select the hosts from [hosts] where a fault will be injected. - */ - public fun select(hosts: Set<SimHost>): List<SimHost> -} +public data class CheckpointModel( + val checkpointWait: Long = 60 * 60 * 1000, + val checkpointTime: Long = 5 * 60 * 1000, +) diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt deleted file mode 100644 index 9511017f..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModel.kt +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.opendc.compute.service.ComputeService -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext - -/** - * Factory interface for constructing [HostFaultInjector] for modeling failures of compute service hosts. - */ -public interface FailureModel { - /** - * Construct a [HostFaultInjector] for the specified [service]. - */ - public fun createInjector( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - ): HostFaultInjector -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt deleted file mode 100644 index 406665c7..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/FailureModelFactory.kt +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2024 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.opendc.compute.simulator.failure.models.Grid5000 -import java.time.Duration -import kotlin.math.roundToLong - -/** - * Get failure model - * - * @param failureInterval The interval of failures occurring in s - * @return - */ -public fun getFailureModel(failureInterval: Double): FailureModel? { - return if (failureInterval > 0) { - Grid5000(Duration.ofSeconds(failureInterval.roundToLong())) - } else { - null - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt deleted file mode 100644 index faf536ad..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFault.kt +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.opendc.compute.simulator.SimHost -import java.time.InstantSource - -/** - * Interface responsible for applying the fault to a host. - */ -public interface HostFault { - /** - * Apply the fault to the specified [victims]. - */ - public suspend fun apply( - clock: InstantSource, - victims: List<SimHost>, - ) -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt deleted file mode 100644 index 26084a1b..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/HostFaultInjector.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.internal.HostFaultInjectorImpl -import java.time.Clock -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext - -/** - * An interface for stochastically injecting faults into a set of hosts. - */ -public interface HostFaultInjector : AutoCloseable { - /** - * Start fault injection. - */ - public fun start() - - /** - * Stop fault injection into the system. - */ - public override fun close() - - public companion object { - /** - * Construct a new [HostFaultInjector]. - * - * @param context The scope to run the fault injector in. - * @param clock The [Clock] to keep track of simulation time. - * @param hosts The hosts to inject the faults into. - * @param iat The inter-arrival time distribution of the failures (in hours). - * @param selector The [VictimSelector] to select the host victims. - * @param fault The type of [HostFault] to inject. - */ - public operator fun invoke( - context: CoroutineContext, - clock: InstantSource, - hosts: Set<SimHost>, - iat: RealDistribution, - selector: VictimSelector, - fault: HostFault, - ): HostFaultInjector = HostFaultInjectorImpl(context, clock, hosts, iat, selector, fault) - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt deleted file mode 100644 index 45545f3b..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StartStopHostFault.kt +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import kotlinx.coroutines.delay -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import java.time.InstantSource -import kotlin.math.roundToLong - -/** - * A type of [HostFault] where the hosts are stopped and recover after some random amount of time. - */ -public class StartStopHostFault(private val duration: RealDistribution) : HostFault { - override suspend fun apply( - clock: InstantSource, - victims: List<SimHost>, - ) { - for (host in victims) { - host.fail() - } - - val df = (duration.sample() * 1000).roundToLong() // seconds to milliseconds - - // Handle long overflow - if (clock.millis() + df <= 0) { - return - } - - delay(df) - - for (host in victims) { - host.recover() - } - } - - override fun toString(): String = "StartStopHostFault[$duration]" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt deleted file mode 100644 index 93463cdb..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/StochasticVictimSelector.kt +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import java.util.ArrayList -import java.util.SplittableRandom -import java.util.random.RandomGenerator -import kotlin.math.roundToInt - -/** - * A [VictimSelector] that stochastically selects a set of hosts to be failed. - */ -public class StochasticVictimSelector( - private val size: RealDistribution, - private val random: RandomGenerator = SplittableRandom(0), -) : VictimSelector { - override fun select(hosts: Set<SimHost>): List<SimHost> { - val n = size.sample().roundToInt() - val result = ArrayList<SimHost>(n) - - val random = random - var samplesNeeded = n - var remainingHosts = hosts.size - val iterator = hosts.iterator() - - while (iterator.hasNext() && samplesNeeded > 0) { - val host = iterator.next() - - if (random.nextInt(remainingHosts) < samplesNeeded) { - result.add(host) - samplesNeeded-- - } - - remainingHosts-- - } - - return result - } - - override fun toString(): String = "StochasticVictimSelector[$size]" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt deleted file mode 100644 index 8aacc49d..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/failure/models/Grid5000.kt +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -@file:JvmName("FailureModels") - -package org.opendc.compute.simulator.failure.models - -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.opendc.compute.service.ComputeService -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.FailureModel -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.StartStopHostFault -import org.opendc.compute.simulator.failure.StochasticVictimSelector -import java.time.Duration -import java.time.InstantSource -import java.util.random.RandomGenerator -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln - -/** - * A [FailureModel] based on the GRID'5000 failure trace. - * - * This fault injector uses parameters from the GRID'5000 failure trace as described in - * "A Framework for the Study of Grid Inter-Operation Mechanisms", A. Iosup, 2009. - */ -public class Grid5000(private val failureInterval: Duration) : FailureModel { - override fun createInjector( - context: CoroutineContext, - clock: InstantSource, - service: ComputeService, - random: RandomGenerator, - ): HostFaultInjector { - val rng = Well19937c(random.nextLong()) - val hosts = service.hosts.map { it as SimHost }.toSet() - - // Parameters from A. Iosup, A Framework for the Study of Grid Inter-Operation Mechanisms, 2009 - // GRID'5000 - return HostFaultInjector( - context, - clock, - hosts, - iat = LogNormalDistribution(rng, ln(failureInterval.toHours().toDouble()), 1.03), - selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25), random), - fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)), - ) - } - - override fun toString(): String = "Grid5000FailureModel" -} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt index c5293a8d..7d4b7302 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/DefaultWorkloadMapper.kt @@ -38,7 +38,8 @@ internal object DefaultWorkloadMapper : SimWorkloadMapper { override fun createWorkload(server: Server): SimWorkload { val workload = delegate.createWorkload(server) - val bootWorkload = SimWorkloads.runtime(Duration.ofMillis(1), 0.8) + // FIXME: look at connecting this to frontend. Probably not needed since the duration is so small + val bootWorkload = SimWorkloads.runtime(Duration.ofMillis(1), 0.8, 0L, 0L) return SimWorkloads.chain(bootWorkload, workload) } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt index e268c506..cbdda0cb 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt @@ -186,7 +186,11 @@ internal class Guest( private fun doStop(target: ServerState) { assert(ctx != null) { "Invalid job state" } val ctx = ctx ?: return - ctx.shutdown() + if (target == ServerState.ERROR) { + ctx.shutdown(Exception("Stopped because of ERROR")) + } else { + ctx.shutdown() + } state = target } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt deleted file mode 100644 index c75ce528..00000000 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/HostFaultInjectorImpl.kt +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.internal - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import org.apache.commons.math3.distribution.RealDistribution -import org.opendc.compute.simulator.SimHost -import org.opendc.compute.simulator.failure.HostFault -import org.opendc.compute.simulator.failure.HostFaultInjector -import org.opendc.compute.simulator.failure.VictimSelector -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext -import kotlin.math.roundToLong - -/** - * Internal implementation of the [HostFaultInjector] interface. - * - * @param context The scope to run the fault injector in. - * @param clock The [InstantSource] to keep track of simulation time. - * @param hosts The set of hosts to inject faults into. - * @param iat The inter-arrival time distribution of the failures (in hours). - * @param selector The [VictimSelector] to select the host victims. - * @param fault The type of [HostFault] to inject. - */ -internal class HostFaultInjectorImpl( - private val context: CoroutineContext, - private val clock: InstantSource, - private val hosts: Set<SimHost>, - private val iat: RealDistribution, - private val selector: VictimSelector, - private val fault: HostFault, -) : HostFaultInjector { - /** - * The scope in which the injector runs. - */ - private val scope = CoroutineScope(context + Job()) - - /** - * The [Job] that awaits the nearest fault in the system. - */ - private var job: Job? = null - - /** - * Start the fault injection into the system. - */ - override fun start() { - if (job != null) { - return - } - - job = - scope.launch { - runInjector() - job = null - } - } - - /** - * Converge the injection process. - */ - private suspend fun runInjector() { - while (true) { - // Make sure to convert delay from hours to milliseconds - val d = (iat.sample() * 3.6e6).roundToLong() - - // Handle long overflow - if (clock.millis() + d <= 0) { - return - } - - delay(d) - - val victims = selector.select(hosts) - fault.apply(clock, victims) - } - } - - /** - * Stop the fault injector. - */ - public override fun close() { - scope.cancel() - } -} diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt deleted file mode 100644 index 690bf472..00000000 --- a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/failure/HostFaultInjectorTest.kt +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.compute.simulator.failure - -import io.mockk.coVerify -import io.mockk.mockk -import kotlinx.coroutines.delay -import org.apache.commons.math3.distribution.LogNormalDistribution -import org.apache.commons.math3.random.Well19937c -import org.junit.jupiter.api.Test -import org.opendc.compute.simulator.SimHost -import org.opendc.simulator.kotlin.runSimulation -import java.time.Duration -import java.time.InstantSource -import kotlin.coroutines.CoroutineContext -import kotlin.math.ln - -/** - * Test suite for [HostFaultInjector] class. - */ -class HostFaultInjectorTest { - /** - * Simple test case to test that nothing happens when the injector is not started. - */ - @Test - fun testInjectorNotStarted() = - runSimulation { - val host = mockk<SimHost>(relaxUnitFun = true) - - val injector = createSimpleInjector(coroutineContext, timeSource, setOf(host)) - - coVerify(exactly = 0) { host.fail() } - coVerify(exactly = 0) { host.recover() } - - injector.close() - } - - /** - * Simple test case to test a start stop fault where the machine is stopped and started after some time. - */ - @Test - fun testInjectorStopsMachine() = - runSimulation { - val host = mockk<SimHost>(relaxUnitFun = true) - - val injector = createSimpleInjector(coroutineContext, timeSource, setOf(host)) - - injector.start() - - delay(Duration.ofDays(55).toMillis()) - - injector.close() - - coVerify(exactly = 1) { host.fail() } - coVerify(exactly = 1) { host.recover() } - } - - /** - * Simple test case to test a start stop fault where multiple machines are stopped. - */ - @Test - fun testInjectorStopsMultipleMachines() = - runSimulation { - val hosts = - listOf<SimHost>( - mockk(relaxUnitFun = true), - mockk(relaxUnitFun = true), - ) - - val injector = createSimpleInjector(coroutineContext, timeSource, hosts.toSet()) - - injector.start() - - delay(Duration.ofDays(55).toMillis()) - - injector.close() - - coVerify(exactly = 1) { hosts[0].fail() } - coVerify(exactly = 1) { hosts[1].fail() } - coVerify(exactly = 1) { hosts[0].recover() } - coVerify(exactly = 1) { hosts[1].recover() } - } - - /** - * Create a simple start stop fault injector. - */ - private fun createSimpleInjector( - context: CoroutineContext, - clock: InstantSource, - hosts: Set<SimHost>, - ): HostFaultInjector { - val rng = Well19937c(0) - val iat = LogNormalDistribution(rng, ln(24 * 7.0), 1.03) - val selector = StochasticVictimSelector(LogNormalDistribution(rng, 1.88, 1.25)) - val fault = StartStopHostFault(LogNormalDistribution(rng, 8.89, 2.71)) - - return HostFaultInjector(context, clock, hosts, iat, selector, fault) - } -} |
