From 6b10881f123f5e6a8e7bce1045d02eba5e48c3a2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 17 Mar 2020 17:37:41 +0100 Subject: [ci skip] feat: Add support for correlated failures --- .../opendc/core/failure/CorrelatedFaultInjector.kt | 111 +++++++++++++++++++++ .../core/failure/UncorrelatedFaultInjector.kt | 2 +- .../opendc/core/services/ServiceRegistry.kt | 2 +- .../opendc/core/services/ServiceRegistryImpl.kt | 2 + 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt (limited to 'opendc/opendc-core') diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt new file mode 100644 index 00000000..41412195 --- /dev/null +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt @@ -0,0 +1,111 @@ +/* + * MIT License + * + * Copyright (c) 2020 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.core.failure + +import com.atlarge.odcsim.Domain +import com.atlarge.odcsim.simulationContext +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlin.math.exp +import kotlin.random.Random +import kotlin.random.asJavaRandom + +/** + * A [FaultInjector] that injects fault in the system which are correlated to each other. Failures do not occur in + * isolation, but will trigger other faults. + */ +public class CorrelatedFaultInjector( + private val domain: Domain, + private val iatScale: Double, + private val iatShape: Double, + private val sizeScale: Double, + private val sizeShape: Double, + random: Random = Random +) : FaultInjector { + /** + * The active failure domains that have been registered. + */ + private val active = mutableSetOf() + + /** + * The [Job] that awaits the nearest fault in the system. + */ + private var job: Job? = null + + /** + * The [Random] instance to use. + */ + private val random: java.util.Random = random.asJavaRandom() + + /** + * Enqueue the specified [FailureDomain] to fail some time in the future. + */ + override fun enqueue(domain: FailureDomain) { + active += domain + + // Clean up the domain if it finishes + domain.scope.coroutineContext[Job]!!.invokeOnCompletion { + this@CorrelatedFaultInjector.domain.launch { + println("CANCELLED") + active -= domain + + if (active.isEmpty()) { + job?.cancel() + job = null + } + } + } + + if (job != null) { + return + } + + job = this.domain.launch { + while (true) { + ensureActive() + + val d = lognvariate(iatScale, iatShape) * 1e3 // Make sure to convert delay to milliseconds + + // Handle long overflow + if (simulationContext.clock.millis() + d <= 0) { + return@launch + } + + delay(d.toLong()) + + val n = lognvariate(sizeScale, sizeShape).toInt() + + for (failureDomain in active.shuffled(random).take(n)) { + failureDomain.fail() + } + } + } + } + + // XXX We should extract this in some common package later on. + private fun lognvariate(scale: Double, shape: Double) = exp(scale + shape * random.nextGaussian()) +} diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt index 56706824..3883eb11 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/UncorrelatedFaultInjector.kt @@ -42,7 +42,7 @@ public class UncorrelatedFaultInjector(private val alpha: Double, private val be override fun enqueue(domain: FailureDomain) { domain.scope.launch { val d = random.weibull(alpha, beta) * 1e3 // Make sure to convert delay to milliseconds - + // Handle long overflow if (simulationContext.clock.millis() + d <= 0) { return@launch diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistry.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistry.kt index d9a85231..a036a705 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistry.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistry.kt @@ -48,5 +48,5 @@ public interface ServiceRegistry { /** * Register the specified [ServiceKey] in this registry. */ - public operator fun set(key: ServiceKey, service: T) + public operator fun set(key: ServiceKey, service: T): ServiceRegistry } diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistryImpl.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistryImpl.kt index 91147839..e3fa171d 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistryImpl.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/services/ServiceRegistryImpl.kt @@ -43,4 +43,6 @@ public class ServiceRegistryImpl : ServiceRegistry { @Suppress("UNCHECKED_CAST") return services[key] as T } + + override fun toString(): String = services.toString() } -- cgit v1.2.3