summaryrefslogtreecommitdiff
path: root/opendc/opendc-core/src
diff options
context:
space:
mode:
authorGeorgios Andreadis <g.andreadis@student.tudelft.nl>2020-04-03 22:13:35 +0200
committerGeorgios Andreadis <g.andreadis@student.tudelft.nl>2020-04-03 22:13:35 +0200
commit6fc21e6df6345c6ec029cc50674352949af83510 (patch)
treece62c5f919fb45e7bf00800bab86defb3521a18c /opendc/opendc-core/src
parenta625066b997cfeeb31c88dddeb17fc67ea75d6e6 (diff)
parent1f67deb18d1430931aec955e7c129cb0d714718c (diff)
Merge branch 'feat/failure-recovery' into '2.x'
Add initial prototype for failure recovery See merge request opendc/opendc-simulator!51
Diffstat (limited to 'opendc/opendc-core/src')
-rw-r--r--opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt29
-rw-r--r--opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt5
2 files changed, 31 insertions, 3 deletions
diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
index c5189764..2904fbec 100644
--- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
+++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
@@ -44,6 +44,8 @@ public class CorrelatedFaultInjector(
private val iatShape: Double,
private val sizeScale: Double,
private val sizeShape: Double,
+ private val dScale: Double,
+ private val dShape: Double,
random: Random = Random(0)
) : FaultInjector {
/**
@@ -84,11 +86,11 @@ public class CorrelatedFaultInjector(
}
job = this.domain.launch {
- while (true) {
+ while (active.isNotEmpty()) {
ensureActive()
// Make sure to convert delay from hours to milliseconds
- val d = lognvariate(iatScale, iatShape) * 3600 * 1e6
+ val d = lognvariate(iatScale, iatShape) * 3.6e6
// Handle long overflow
if (simulationContext.clock.millis() + d <= 0) {
@@ -98,10 +100,31 @@ public class CorrelatedFaultInjector(
delay(d.toLong())
val n = lognvariate(sizeScale, sizeShape).toInt()
- for (failureDomain in active.shuffled(random).take(n)) {
+ val targets = active.shuffled(random).take(n)
+
+ for (failureDomain in targets) {
+ active -= failureDomain
failureDomain.fail()
}
+
+ val df = lognvariate(dScale, dShape) * 6e4
+
+ // Handle long overflow
+ if (simulationContext.clock.millis() + df <= 0) {
+ return@launch
+ }
+
+ delay(df.toLong())
+
+ for (failureDomain in targets) {
+ failureDomain.recover()
+
+ // Re-enqueue machine to be failed
+ enqueue(failureDomain)
+ }
}
+
+ job = null
}
}
diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt
index 91ca9b83..d56df3c9 100644
--- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt
+++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/FailureDomain.kt
@@ -39,4 +39,9 @@ public interface FailureDomain {
* Fail the domain externally.
*/
public suspend fun fail()
+
+ /**
+ * Resume the failure domain.
+ */
+ public suspend fun recover()
}