feat: complete system redesign: KafkaComputeMonitor stays, SmartScheduler is working and only HTTPClient remains, the rest will be moved to Python scriptsHEADmaster

3 files changed, 146 insertions, 43 deletions

diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
index fc40fac0..240d21d5 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
@@ -39,6 +39,7 @@ import java.util.SplittableRandom
 import java.util.random.RandomGenerator
 
 public enum class ComputeSchedulerEnum {
+    Smart,
     Mem,
     MemInv,
     CoreMem,
@@ -81,6 +82,9 @@ public fun createPrefabComputeScheduler(
 
 /**
  * Create a [ComputeScheduler] for the experiment.
+ * I have changed this to use the new SmartScheduler.
+ *
+ * @author Mateusz Kwiatkowski
  */
 public fun createPrefabComputeScheduler(
     name: ComputeSchedulerEnum,
@@ -92,6 +96,8 @@ public fun createPrefabComputeScheduler(
     val ramAllocationRatio = 1.0
     val gpuAllocationRatio = 1.0
     return when (name) {
+        ComputeSchedulerEnum.Smart ->
+            SmartScheduler()
         ComputeSchedulerEnum.Mem ->
             FilterScheduler(
                 filters = listOf(VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)),
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/SmartScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/SmartScheduler.kt
index 0edd97cf..7a540e86 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/SmartScheduler.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/SmartScheduler.kt
@@ -1,69 +1,167 @@
-/*
- * Copyright (c) 2021 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
 package org.opendc.compute.simulator.scheduler
 
-import org.opendc.common.utils.HTTPClient
+
+import org.opendc.compute.simulator.scheduler.filters.HostFilter
 import org.opendc.compute.simulator.service.HostView
 import org.opendc.compute.simulator.service.ServiceTask
+import java.util.SplittableRandom
+import java.util.random.RandomGenerator
 
+/**
+ * A scheduler that is able to cooperate with a digital twin.
+ * Currently, this is hardcoded.
+ * Important: the random seed can aid you in finding good standard deviation measure.
+ * The scheduling significantly affects downtime.
+ *
+ * @author Mateusz Kwiatkowski
+ */
 public class SmartScheduler : ComputeScheduler {
-    private val client = HTTPClient.getInstance()
-    // the point is that a smart scheduler listens for suggestions from the
-    // digital twin
-    // and here is where you change your actions based on the result from the DT
-    // predictive analytics is going to be much easier to do.
-    // you will completely overcome the overhead of having to tap-into
-    // the digital twin mid-through the simulation/in between two hosts being scheduled
-    // i.e., the normal simulation will NOT have to wait.
-    // predictive analytics will overcome the problem of the scheduling time overhead
+    private val maxTimesSkipped: Int = 7
+    private val random: RandomGenerator = SplittableRandom(128)
+    // We will use a priority queue.
+    private val hostsQueue = List(100, { mutableListOf<HostView>() })
+    private var minAvailableHost = 0
+    private var numHosts = 0
+
+
+    // Unlucky Hosts for WhatsApp
+    //private var  unluckyHosts = listOf("H01","H02-109","H02-229","H02-107","H02-227","H02-105","H02-226","H02-103","H02-101","H02-222","H02-221","H02-220","H02-213","H02-212","H02-210","H02-208","H02-207","H02-206","H02-204","H02-203","H02-202","H01-181","H01-184","H01-185","H01-186","H01-189","H01-170","H01-174","H01-175","H01-178","H01-179","H01-169","H01-164","H01-166","H01-167","H01-168","H02-197","H02-196","H02-195","H02-194","H01-270","H02-193","H01-271","H02-192","H01-152","H02-190","H01-275","H01-155","H01-157"," H01-91"," H01-94"," H01-84"," H02-52"," H02-54"," H01-87"," H02-55"," H01-88"," H01-89"," H02-57"," H02-58"," H02-59"," H02-70"," H01-95"," H02-63"," H01-96"," H02-64"," H02-65"," H01-98"," H02-66"," H01-99"," H02-67"," H02-69"," H02-83"," H02-74"," H02-76"," H02-78"," H02-90"," H02-91","H01-192","H01-193","H01-195"," H02-87"," H02-89","H01-103","H01-224","H01-105","H01-226","H01-106","H01-227","H01-228"," H01-50","H02-148","H02-147"," H01-48"," H02-16"," H02-17","H02-144","H02-265"," H02-18","H02-264","H02-141","H02-262","H02-140","H02-261","H02-260"," H01-42"," H02-10"," H01-43"," H02-12"," H01-45","H01-221"," H02-13"," H01-46","H01-101","H01-222"," H01-47","H01-102","H01-223"," H02-15","H01-213","H01-214","H01-215","H01-218","H01-219","H02-139","H02-138","H02-135","H02-256"," H01-59"," H02-27","H02-134","H02-133","H02-254","H02-253","H02-130","H02-250"," H01-53"," H02-21"," H01-54"," H02-22"," H01-55"," H01-56","H01-210","H01-211"," H02-25","H01-202","H01-203","H01-204","H01-206"," H01-70","H01-207"," H01-72","H01-209","H02-129","H02-248","H02-126","H02-246","H02-245"," H02-38","H02-122","H02-121","H02-242","H02-241"," H01-62"," H02-30"," H01-64"," H01-65"," H01-66"," H02-35","H01-200"," H02-36","H01-201","  H02-0","  H02-1"," H01-80"," H02-50"," H01-83"," H02-51","H02-118","H02-239","H02-117","H02-238","H02-237","H02-115","H02-113"," H02-49","H02-112","H02-233","H02-111","H02-231"," H02-41"," H01-75"," H02-43","  H02-9","  H02-4"," H02-45"," H02-46"," H01-79","H01-147","H01-268","H01-149","H02-189","H02-186","H02-185","H02-184","H02-183","H02-182","H01-260","H01-140","H02-180","H01-141","H01-262","H01-263","H01-264","H01-144","H01-145","H01-146","H01-267","H01-136","H01-257","  H01-3","  H01-2"," H01-17","H02-176","H02-175","H02-173","H02-172","  H01-9","H02-170","  H01-8","H01-250","H01-130","H01-251","H01-252","  H01-5"," H01-11"," H01-12"," H01-13","  H01-6","H01-135","H01-246","H01-126","H01-127","H01-129","H02-169"," H01-27","H02-165","H02-162","H02-161","H02-160","H01-240","H01-122"," H01-24","H01-124","H01-114","H01-236","H01-117","H01-238","H01-239","H01-119","H02-158","H02-157"," H01-39","H02-275","H02-274","H02-151","H02-150","H02-270"," H01-31"," H01-33","H01-110"," H01-34","H01-111","H01-232","H01-233"," H01-36","H01-113")
+
+    // Unlucky Hosts for Skype
+    //private var unluckyHosts = listOf("H01-44","H01-66","H01-73","H01-74","H01-85","H01-86","H01-89","H01-10","H01-98","H01-13","H01-39")
+
+    // Unlucky Hosts for Twitter
+    //private var unluckyHosts = listOf("H01-180","H01-181","H01-182","H01-184","H01-185","H01-186","H01-188","H01-171","H01-173","H01-176","H01-177","H01-178","H01-179","H01-169","    H01","H01-162","H01-163","H01-164","H01-165","H01-166","H01-168","H01-158","H01-270","H01-150","H01-271","H01-272","H01-273","H01-274","H01-154","H01-155","H01-156","H01-157"," H01-90"," H01-91"," H01-92"," H01-93"," H01-88"," H01-89"," H01-95"," H01-96"," H01-99","H01-190","H01-191","H01-192","H01-193","H01-196","H01-197","H01-198","H01-103","H01-224","H01-225","H01-106","H01-228","H01-108"," H01-50"," H01-40"," H01-42"," H01-43"," H01-44"," H01-45","H01-100","H01-222","H01-102","H01-223","H01-214","H01-215","H01-216","H01-217","H01-218"," H01-60"," H01-59"," H01-51"," H01-52"," H01-53"," H01-56","H01-210"," H01-57","H01-211","H01-212","H01-202","H01-204","H01-205","H01-206"," H01-70"," H01-71"," H01-72","H01-209"," H01-63"," H01-64"," H01-80"," H01-82"," H01-73"," H01-76"," H01-77","H01-268","H01-269","H01-149","H01-260","H01-140","H01-262","H01-263","H01-143","H01-144","H01-265","H01-145","H01-146","H01-267","  H01-1","H01-136","H01-257","H01-137","H01-258","  H01-3","H01-139"," H01-17"," H01-18","  H01-9","  H01-8","H01-130","H01-251","H01-252","  H01-4"," H01-12","H01-133","H01-254","  H01-7"," H01-14","H01-135","H01-256","H01-126"," H01-26"," H01-27"," H01-28"," H01-29"," H01-20","H01-120","H01-241"," H01-22","H01-242","H01-243"," H01-24","H01-123"," H01-25","H01-124","H01-115","H01-116","H01-237","H01-117","H01-118","H01-239"," H01-38"," H01-31","H01-230"," H01-33","H01-110"," H01-35","H01-233","H01-113")
 
     override fun addHost(host: HostView) {
-        TODO("Not yet implemented")
+        // We do not want a host available if it is in the list of unluckyHosts
+        //if(host.host.getName() in unluckyHosts) {return}
+        val zeroQueue = hostsQueue[0]
+        zeroQueue.add(host)
+        host.priorityIndex = 0;
+        host.listIndex = zeroQueue.size - 1
+        numHosts++
+        minAvailableHost = 0
     }
 
     override fun removeHost(host: HostView) {
-        TODO("Not yet implemented")
+        val priorityIdx = host.priorityIndex
+        val listIdx = host.listIndex
+        val chosenList = hostsQueue[priorityIdx]
+
+        if (chosenList.size == 1) {
+            chosenList.removeLast()
+            if (listIdx == minAvailableHost) {
+                for (i in minAvailableHost + 1..hostsQueue.lastIndex) {
+                    if (hostsQueue[i].size > 0) {
+                        minAvailableHost = i
+                        break
+                    }
+                }
+            }
+        } else {
+            val lastItem = chosenList.removeLast()
+            chosenList[listIdx] = lastItem
+            lastItem.listIndex = listIdx
+        }
+        numHosts--
     }
 
     override fun updateHost(host: HostView) {
-        TODO("Not yet implemented")
+        // No-op
+
     }
 
     override fun setHostEmpty(hostView: HostView) {
-        TODO("Not yet implemented")
+        // No-op
     }
 
     override fun select(iter: MutableIterator<SchedulingRequest>): SchedulingResult {
-        TODO("Not yet implemented")
-        // Here be the API calls using HTTP between the other OpenDC
-        // You need to specify how much time do you have to make the prediction between receiving a time and putting onto a host
-        return SchedulingResult(SchedulingResultType.EMPTY)
+        if (numHosts == 0) {
+            return SchedulingResult(SchedulingResultType.FAILURE)
+        }
+
+        val maxIters = 10000
+        var numIters = 0
+
+        var chosenList: MutableList<HostView>? = null
+        var chosenHost: HostView? = null
+
+        var result: SchedulingResult? = null
+        taskloop@ for (req in iter) {
+            if (req.isCancelled) {
+                iter.remove()
+                continue
+            }
+
+            numIters++
+            if (numIters > maxIters) {
+                return SchedulingResult(SchedulingResultType.EMPTY)
+            }
+
+            for (chosenListIndex in minAvailableHost until hostsQueue.size) {
+                chosenList = hostsQueue[chosenListIndex]
+                for (host in chosenList) {
+                    // Here be filtering, but for now we remove it completely.
+                        iter.remove()
+                        chosenHost = host
+                        result = SchedulingResult(SchedulingResultType.SUCCESS, host, req)
+                        break@taskloop
+                }
+            }
+            req.timesSkipped++
+        }
+
+        if (result == null) return SchedulingResult(SchedulingResultType.EMPTY) // No tasks to schedule that fit
+
+        // Bookkeeping to maintain the calendar priority queue
+        if (chosenList!!.size == 1) {
+            chosenList.removeLast()
+            minAvailableHost++
+        } else {
+            val listIdx = chosenHost!!.listIndex
+            // Not using removeLast here as it would cause problems during swapping
+            // if chosenHost is lastItem
+            val lastItem = chosenList.last()
+            chosenList[listIdx] = lastItem
+            lastItem.listIndex = listIdx
+            chosenList.removeLast()
+        }
+
+        val nextList = hostsQueue[chosenHost!!.priorityIndex + 1]
+        nextList.add(chosenHost)
+        chosenHost.priorityIndex++
+        chosenHost.listIndex = nextList.size - 1
+
+        return result
     }
 
     override fun removeTask(
         task: ServiceTask,
-        host: HostView?,
+        host: HostView?
     ) {
-        TODO("Not yet implemented")
+        if (host == null) return
+
+        val priorityIdx = host.priorityIndex
+        val listIdx = host.listIndex
+        val chosenList = hostsQueue[priorityIdx]
+        val nextList = hostsQueue[priorityIdx - 1]
+
+        if (chosenList.size == 1) {
+            chosenList.removeLast()
+        } else {
+            val lastItem = chosenList.last()
+            chosenList[listIdx] = lastItem
+            lastItem.listIndex = listIdx
+            chosenList.removeLast()
+        }
+
+        nextList.add(host)
+        host.priorityIndex--
+        host.listIndex = nextList.size - 1
+        if (priorityIdx == minAvailableHost) {
+            minAvailableHost--
+        }
     }
+
 }
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/KafkaComputeMonitor.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/KafkaComputeMonitor.kt
index ddd4a28a..d158f782 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/KafkaComputeMonitor.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/telemetry/KafkaComputeMonitor.kt
@@ -51,7 +51,6 @@ public class KafkaComputeMonitor : ComputeMonitor {
                     .setDowntime(reader.downtime.toDouble())
                     .build()
             this.send(packet)
-
         } catch (e: Exception) {
             println("${e.message}")
         }