Add mixed recovery mode

JetBrains · olgalupuleac · Dec 1, 2020 · Dec 1, 2020 · Dec 7, 2020 · Dec 9, 2020
commit db7d1a608cda4f5d7ddf3b2824bdb93028d3134d
diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/DistributedCTestConfiguration.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/DistributedCTestConfiguration.kt
@@ -40,7 +40,7 @@ class DistributedCTestConfiguration<Message, Log>(testClass: Class<*>, iteration
                                                   val isNetworkReliable: Boolean,
                                                   val messageOrder: MessageOrder,
                                                   val maxNumberOfFailedNodes: (Int) -> Int,
-                                                  val supportRecovery: Boolean,
+                                                  val supportRecovery: RecoveryMode,
                                                   val messageDuplication: Boolean,
                                                   val networkPartitions : Boolean,
                                                   val asyncRun : Boolean,

diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/DistributedOptions.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/DistributedOptions.kt
@@ -10,12 +10,19 @@ enum class MessageOrder {
     ASYNCHRONOUS
 }
 
+
+enum class RecoveryMode {
+    NO_RECOVERIES,
+    ALL_NODES_RECOVER,
+    MIXED
+}
+
 class DistributedOptions<Message, Log> : Options<DistributedOptions<Message, Log>,
         DistributedCTestConfiguration<Message, Log>>() {
     private var isNetworkReliable: Boolean = true
     private var messageOrder: MessageOrder = MessageOrder.SYNCHRONOUS
     private var maxNumberOfFailedNodes: (Int) -> Int = { 0 }
-    private var supportRecovery: Boolean = true
+    private var supportRecovery: RecoveryMode = RecoveryMode.ALL_NODES_RECOVER
     private var invocationsPerIteration: Int = DistributedCTestConfiguration.DEFAULT_INVOCATIONS
     private var messageDuplication: Boolean = false
     private var networkPartitions: Boolean = false
@@ -43,7 +50,7 @@ class DistributedOptions<Message, Log> : Options<DistributedOptions<Message, Log
         return this
     }
 
-    fun supportRecovery(supportRecovery: Boolean): DistributedOptions<Message, Log> {
+    fun supportRecovery(supportRecovery: RecoveryMode): DistributedOptions<Message, Log> {
         this.supportRecovery = supportRecovery
         return this
     }

diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/NodeFailureInfo.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/NodeFailureInfo.kt
@@ -63,16 +63,6 @@ class NodeFailureInfo(
             return
         }
         failedNodes[iNode].lazySet(false)
-        //semaphore?.release()
-    }
-
-    /**
-     * Resets to initial state.
-     */
-    fun reset() {
-        repeat(numberOfNodes) {
-            failedNodes[it].lazySet(false)
-        }
-        semaphore = if (maxNumberOfFailedNodes != 0) Semaphore(maxNumberOfFailedNodes, 0) else null
+        semaphore?.release()
     }
 }
diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/DistributedRunner.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/DistributedRunner.kt
@@ -421,7 +421,10 @@ open class DistributedRunner<Message, Log>(
         if (iNode < context.addressResolver.nodesWithScenario && context.actorIds[iNode] < scenarioSize) {
             context.testNodeExecutions[iNode].results[context.actorIds[iNode]++] = CrashResult
         }
-        if (testCfg.supportRecovery) {
+        if (testCfg.supportRecovery == RecoveryMode.ALL_NODES_RECOVER ||
+            testCfg.supportRecovery == RecoveryMode.MIXED
+            && context.probabilities[iNode].nodeRecovered()
+        ) {
             val delta = initTasksForNode(iNode)
             taskCounter.add(delta)
             logMessage(LogLevel.ALL_EVENTS) {

diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/DistributedRunnerContext.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/DistributedRunnerContext.kt
@@ -80,4 +80,8 @@ class DistributedRunnerContext<Message, Log>(
     val logs = Array(addressResolver.totalNumberOfNodes) {
         emptyList<Log>()
     }
+
+    val probabilities = Array(addressResolver.totalNumberOfNodes) {
+        Probability(tesCfg, addressResolver.totalNumberOfNodes)
+    }
 }
diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/EnvironmentImpl.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/EnvironmentImpl.kt
@@ -37,7 +37,7 @@ internal class EnvironmentImpl<Message, Log>(
 
     override val numberOfNodes = context.addressResolver.totalNumberOfNodes
 
-    private val probability = Probability(context.tesCfg, numberOfNodes)
+    private val probability = context.probabilities[nodeId]
 
     override fun getAddressesForClass(cls: Class<out Node<Message>>) = context.addressResolver[cls]
 

diff --git a/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/Probability.kt b/src/jvm/main/org/jetbrains/kotlinx/lincheck/distributed/stress/Probability.kt
@@ -28,6 +28,7 @@ class Probability(private val testCfg: DistributedCTestConfiguration<*, *>, val
         const val MESSAGE_SENT_PROBABILITY = 0.95
         const val MESSAGE_DUPLICATION_PROBABILITY = 0.9
         const val NODE_FAIL_PROBABILITY = 0.05
+        const val NODE_RECOVERY_PROBABILITY = 0.7
     }
 
     private val rand = ThreadLocalRandom.current()
@@ -50,4 +51,6 @@ class Probability(private val testCfg: DistributedCTestConfiguration<*, *>, val
     }
 
     fun nodeFailed(): Boolean = rand.nextDouble(1.0) < NODE_FAIL_PROBABILITY
+
+    fun nodeRecovered(): Boolean = rand.nextDouble(1.0) < NODE_RECOVERY_PROBABILITY
 }