diff options
| author | Georgios Andreadis <g.andreadis@student.tudelft.nl> | 2020-04-03 22:13:35 +0200 |
|---|---|---|
| committer | Georgios Andreadis <g.andreadis@student.tudelft.nl> | 2020-04-03 22:13:35 +0200 |
| commit | 6fc21e6df6345c6ec029cc50674352949af83510 (patch) | |
| tree | ce62c5f919fb45e7bf00800bab86defb3521a18c /opendc/opendc-core/src | |
| parent | a625066b997cfeeb31c88dddeb17fc67ea75d6e6 (diff) | |
| parent | 1f67deb18d1430931aec955e7c129cb0d714718c (diff) | |
Merge branch 'feat/failure-recovery' into '2.x'
Add initial prototype for failure recovery
See merge request opendc/opendc-simulator!51
Diffstat (limited to 'opendc/opendc-core/src')
2 files changed, 31 insertions, 3 deletions
diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt index c5189764..2904fbec 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt @@ -44,6 +44,8 @@ public class CorrelatedFaultInjector( private val iatShape: Double, private val sizeScale: Double, private val sizeShape: Double, + private val dScale: Double, + private val dShape: Double, random: Random = Random(0) ) : FaultInjector { /** @@ -84,11 +86,11 @@ public class CorrelatedFaultInjector( } job = this.domain.launch { - while (true) { + while (active.isNotEmpty()) { ensureActive() // Make sure to convert delay from hours to milliseconds - val d = lognvariate(iatScale, iatShape) * 3600 * 1e6 + val d = lognvariate(iatScale, iatShape) * 3.6e6 // Handle long overflow if (simulationContext.clock.millis() + d <= 0) { @@ -98,10 +100,31 @@ public class CorrelatedFaultInjector( delay(d.toLong()) val n = lognvariate(sizeScale, sizeShape).toInt() - for (failureDomain in active.shuffled(random).take(n)) { + val targets = active.shuffled(random).take(n) + + for (failureDomain in targets) { + active -= failureDomain failureDomain.fail() } + + val df = lognvariate(dScale, dShape) * 6e4 + + // Handle long overflow + if (simulationContext.clock.millis() + df <= 0) { + return@launch + } + + delay(df.toLong()) + + for (failureDomain in targets) { + failureDomain.recover() + + // Re-enqueue machine to be failed + enqueue(failureDomain) + } } + + job = null } } diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt index 91ca9b83..d56df3c9 100644 --- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt +++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt @@ -39,4 +39,9 @@ public interface FailureDomain { * Fail the domain externally. */ public suspend fun fail() + + /** + * Resume the failure domain. + */ + public suspend fun recover() } |
