From ab7d7c84652ec4b0ac1081b76add56824de851f1 Mon Sep 17 00:00:00 2001
From: mjkwiatkowski <mati.rewa@gmail.com>
Date: Sun, 26 Jul 2026 22:33:21 +0200
Subject: feat: removed the generated output and the python scripts to plot the
 results

---
 python_scripts/src/python_scripts/__main__.py      | 141 ++++++++++++++-------
 .../__pycache__/__main__.cpython-314.pyc           | Bin 4058 -> 6500 bytes
 .../resources/Skype06Exp/battery.parquet           | Bin 0 -> 364 bytes
 .../resources/Skype06Exp/host.parquet              | Bin 0 -> 1954533 bytes
 .../resources/Skype06Exp/powerSource.parquet       | Bin 0 -> 11535 bytes
 .../resources/Skype06Exp/service.parquet           | Bin 0 -> 4776 bytes
 .../resources/Skype06Exp/task.parquet              | Bin 0 -> 1532643 bytes
 .../resources/Skype06Gam/battery.parquet           | Bin 0 -> 364 bytes
 .../resources/Skype06Gam/host.parquet              | Bin 0 -> 1957452 bytes
 .../resources/Skype06Gam/powerSource.parquet       | Bin 0 -> 11535 bytes
 .../resources/Skype06Gam/service.parquet           | Bin 0 -> 4733 bytes
 .../resources/Skype06Gam/task.parquet              | Bin 0 -> 1523976 bytes
 .../resources/Skype06LogN/battery.parquet          | Bin 0 -> 364 bytes
 .../resources/Skype06LogN/host.parquet             | Bin 0 -> 1953541 bytes
 .../resources/Skype06LogN/powerSource.parquet      | Bin 0 -> 11533 bytes
 .../resources/Skype06LogN/service.parquet          | Bin 0 -> 4756 bytes
 .../resources/Skype06LogN/task.parquet             | Bin 0 -> 1526316 bytes
 .../resources/Skype06Wbl/battery.parquet           | Bin 0 -> 364 bytes
 .../resources/Skype06Wbl/host.parquet              | Bin 0 -> 1949687 bytes
 .../resources/Skype06Wbl/powerSource.parquet       | Bin 0 -> 11535 bytes
 .../resources/Skype06Wbl/service.parquet           | Bin 0 -> 4756 bytes
 .../resources/Skype06Wbl/task.parquet              | Bin 0 -> 1523076 bytes
 .../src/python_scripts/resources/battery.parquet   | Bin 364 -> 0 bytes
 .../src/python_scripts/resources/host.parquet      | Bin 1976402 -> 0 bytes
 .../src/python_scripts/resources/main.py           |   6 +
 .../python_scripts/resources/powerSource.parquet   | Bin 11559 -> 0 bytes
 .../resources/prefabs_old/battery.parquet          | Bin 0 -> 364 bytes
 .../resources/prefabs_old/host.parquet             | Bin 0 -> 1976402 bytes
 .../resources/prefabs_old/powerSource.parquet      | Bin 0 -> 11559 bytes
 .../resources/prefabs_old/readme.txt               |   7 +
 .../resources/prefabs_old/service.parquet          | Bin 0 -> 4853 bytes
 .../resources/prefabs_old/task.parquet             | Bin 0 -> 1568225 bytes
 .../src/python_scripts/resources/readme.txt        |   7 -
 .../src/python_scripts/resources/service.parquet   | Bin 4853 -> 0 bytes
 .../src/python_scripts/resources/task.parquet      | Bin 1568225 -> 0 bytes
 35 files changed, 110 insertions(+), 51 deletions(-)
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet
 delete mode 100644 python_scripts/src/python_scripts/resources/battery.parquet
 delete mode 100644 python_scripts/src/python_scripts/resources/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/main.py
 delete mode 100644 python_scripts/src/python_scripts/resources/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/host.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/readme.txt
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/service.parquet
 create mode 100644 python_scripts/src/python_scripts/resources/prefabs_old/task.parquet
 delete mode 100644 python_scripts/src/python_scripts/resources/readme.txt
 delete mode 100644 python_scripts/src/python_scripts/resources/service.parquet
 delete mode 100644 python_scripts/src/python_scripts/resources/task.parquet

(limited to 'python_scripts/src')

diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py
index 896c7c3c..76b985f9 100644
--- a/python_scripts/src/python_scripts/__main__.py
+++ b/python_scripts/src/python_scripts/__main__.py
@@ -1,14 +1,8 @@
+from inspect import currentframe
+
 import redis
 import pandas as pd
 import threshold_based
-import requests
-from redis.exceptions import (
-    ConnectionError,
-    DataError,
-    NoScriptError,
-    RedisError,
-    ResponseError,
-)
 
 api = "http://localhost:1234"
 redis_host = "localhost"
@@ -16,38 +10,28 @@ stream_key = "postgres_topic"
 group = "python_consumer"
 r = redis.Redis(redis_host)
 PATH = "/home/matt/src/sunfish/python_scripts/src/python_scripts/resources"
-threshold = 0
-dictionary = {}
-
-
-def iterate(frame):
-    dictionary = {}
-    for i in range(len(frame)):
-        if frame["downtime"].iloc[i] > 0.0:
-            ts = frame["timestamp"].iloc[i]
-            dictionary[(ts)] = dictionary.get((ts), 0) + 1
-    return dictionary
-
-
-def caluclate_threshold(frame):
-    dict = iterate(frame)
-    global threshold
-    res = 0
-    for val in dict.values():
-        res += val
-    res = res / len(dict)
-    threshold = float(res)
-    return res
+similarity_scores = {
+    "Skype06Exp": 0,
+    "Skype06LogN": 0,
+    "Skype06Gam": 0,
+    "Skype06Wbl": 0,
+}
 
 
 def create_dataframes():
-    hosts: str = "%s/host.parquet" % PATH
-    tasks: str = "%s/task.parquet" % PATH
+    Skype06Exp: str = "%s/Skype06Exp/host.parquet" % PATH
+    Skype06LogN: str = "%s/Skype06LogN/host.parquet" % PATH
+    Skype06Gam: str = "%s/Skype06Gam/host.parquet" % PATH
+    Skype06Wbl: str = "%s/Skype06Wbl/host.parquet" % PATH
+    # Keep this order whenever operating
+    listOfHosts = [Skype06Exp, Skype06LogN, Skype06Gam, Skype06Wbl]
+    listOfDataFrames = []
 
     try:
-        df_hosts = pd.read_parquet(hosts)
-        df_tasks = pd.read_parquet(tasks)
-        return (df_hosts, df_tasks)
+        for item in listOfHosts:
+            df_hosts = pd.read_parquet(item)
+            listOfDataFrames.append(df_hosts)
+        return listOfDataFrames
 
     except Exception as e:
         print(f"exception caught: {e}")
@@ -60,14 +44,82 @@ As suggested in https://redis.readthedocs.io/en/stable/examples/redis-stream-exa
 toack = lambda k, g, e: r.xack(k, g, e)
 
 
-def print_xreadgroup_reply(reply, f, output_file):
+def update_score(
+    stream,
+    frames,
+):
+    global similarity_scores
+    host_stream = str(stream[b"host_id"].decode())
+    timestamp_stream = int(stream[b"timestamp"].decode())
+    current_frame = frames[0]
+
+    for i in range(len(frames)):
+        current_frame = frames[i]
+        filtered_df = current_frame[
+            (current_frame["timestamp"] == timestamp_stream)
+            & (current_frame["host_name"] == host_stream)
+        ]
+
+        y_t = float(stream[b"downtime"].decode())
+        y_t_hat = float(filtered_df["downtime"].iloc[0])
+        # Abs is mandatory here! See https://en.wikipedia.org/wiki/Euclidean_distance
+        e_t_downtime = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"cpuutilization"].decode())
+        y_t_hat = float(filtered_df["cpu_utilization"].iloc[0])
+        e_t_cpuUtil = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"energyusage"].decode())
+        y_t_hat = float(filtered_df["energy_usage"].iloc[0])
+        e_t_energyUsage = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"tasksactive"].decode())
+        y_t_hat = float(filtered_df["tasks_running"].iloc[0])
+        e_t_tasksActive = abs(y_t - y_t_hat)
+
+        S = (
+            e_t_downtime * 0.9
+            + e_t_cpuUtil * 0.7
+            + e_t_tasksActive * 0.6
+            + e_t_energyUsage * 0.7
+        )
+        S = S / (0.9 + 0.7 + 0.6 + 0.7)
+
+        res = list(similarity_scores)[i]
+        print(res)
+        oldS = similarity_scores[res]
+        print(oldS)
+        new_S = (oldS + S) / 2
+        similarity_scores[res] = new_S
+
+    total = 0
+    for value in similarity_scores.values():
+        total += value
+
+    for key, value in similarity_scores.items():
+        similarity_scores[key] = value / total
+
+    return
+
+
+def pretty_print(output_file, stream):
+    timestamp = int(stream[b"timestamp"].decode())
+    output_str = f"{timestamp}, {similarity_scores['Skype06Exp']}, {similarity_scores['Skype06LogN']}, {similarity_scores['Skype06Gam']}, {similarity_scores['Skype06Wbl']}\n"
+    output_file.write(output_str)
+
+
+def send_insights(stream, frames):
+    return
+
+
+def print_xreadgroup_reply(reply, frames, output_file):
     global threshold
     for d_stream in reply:
         for element in d_stream[1]:
-            threshold_based.check_metric(
-                element[1], f, threshold, dictionary, output_file
-            )
+            update_score(element[1], frames)
+            send_insights(element[1], frames)
             toack(d_stream[0], group, element[0])
+            pretty_print(output_file, element[1])
 
 
 def read_entries(r, f, output_file):
@@ -83,17 +135,18 @@ def read_entries(r, f, output_file):
 
 
 def main():
+    f = open("output.txt", "w")
     try:
         frames = create_dataframes()
-        caluclate_threshold(frames[0])
-
-        with open("output.txt", "w", encoding="utf-8") as f:
-            f.write(f"Threshold: {threshold}\n")
-            read_entries(r, frames[0], output_file=f)
+        read_entries(r, frames, output_file=f)
 
+        f.close()
     except Exception as e:
         print(f"exception caught: {e}")
 
+    finally:
+        f.close()
+
 
 if __name__ == "__main__":
     main()
diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc
index 705a695c..b353e8de 100644
Binary files a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc and b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet
new file mode 100644
index 00000000..93b3f21c
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet
new file mode 100644
index 00000000..e11dc8b0
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet
new file mode 100644
index 00000000..bf7ab98d
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet
new file mode 100644
index 00000000..3d494013
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet
new file mode 100644
index 00000000..2b79e0ad
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet
new file mode 100644
index 00000000..93b3f21c
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet
new file mode 100644
index 00000000..dc98c778
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet
new file mode 100644
index 00000000..7247d165
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet
new file mode 100644
index 00000000..951429cf
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet
new file mode 100644
index 00000000..643939d1
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet
new file mode 100644
index 00000000..93b3f21c
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet
new file mode 100644
index 00000000..b145bb65
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet
new file mode 100644
index 00000000..466d3215
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet
new file mode 100644
index 00000000..33795ed3
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet
new file mode 100644
index 00000000..e356ef75
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet
new file mode 100644
index 00000000..93b3f21c
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet
new file mode 100644
index 00000000..9fdf3210
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet
new file mode 100644
index 00000000..3c79b19b
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet
new file mode 100644
index 00000000..4d40319d
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet
new file mode 100644
index 00000000..1453b91e
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/battery.parquet b/python_scripts/src/python_scripts/resources/battery.parquet
deleted file mode 100644
index 93b3f21c..00000000
Binary files a/python_scripts/src/python_scripts/resources/battery.parquet and /dev/null differ
diff --git a/python_scripts/src/python_scripts/resources/host.parquet b/python_scripts/src/python_scripts/resources/host.parquet
deleted file mode 100644
index e9add02b..00000000
Binary files a/python_scripts/src/python_scripts/resources/host.parquet and /dev/null differ
diff --git a/python_scripts/src/python_scripts/resources/main.py b/python_scripts/src/python_scripts/resources/main.py
new file mode 100644
index 00000000..accd0314
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/main.py
@@ -0,0 +1,6 @@
+import pandas as pd
+import numpy as np
+
+df = pd.read_parquet("Skype06Gam/task.parquet")
+
+print(df.loc[:, "num_failures"].mean())
diff --git a/python_scripts/src/python_scripts/resources/powerSource.parquet b/python_scripts/src/python_scripts/resources/powerSource.parquet
deleted file mode 100644
index 304b09cf..00000000
Binary files a/python_scripts/src/python_scripts/resources/powerSource.parquet and /dev/null differ
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet
new file mode 100644
index 00000000..93b3f21c
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet
new file mode 100644
index 00000000..e9add02b
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet
new file mode 100644
index 00000000..304b09cf
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt
new file mode 100644
index 00000000..b00086b8
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt
@@ -0,0 +1,7 @@
+This is the result of a raw simulation with the failureModels set to some statistical distribution in OpenDC.
+Based on the downtime in the `host.parquet` file, we calculate the threshold.
+These results are based on this failureModel:
+{
+    "type": "prefab",
+    "prefabName" : "Pl05Exp"
+}
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet
new file mode 100644
index 00000000..371a66a7
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet
new file mode 100644
index 00000000..1e62dae2
Binary files /dev/null and b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet differ
diff --git a/python_scripts/src/python_scripts/resources/readme.txt b/python_scripts/src/python_scripts/resources/readme.txt
deleted file mode 100644
index b00086b8..00000000
--- a/python_scripts/src/python_scripts/resources/readme.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-This is the result of a raw simulation with the failureModels set to some statistical distribution in OpenDC.
-Based on the downtime in the `host.parquet` file, we calculate the threshold.
-These results are based on this failureModel:
-{
-    "type": "prefab",
-    "prefabName" : "Pl05Exp"
-}
diff --git a/python_scripts/src/python_scripts/resources/service.parquet b/python_scripts/src/python_scripts/resources/service.parquet
deleted file mode 100644
index 371a66a7..00000000
Binary files a/python_scripts/src/python_scripts/resources/service.parquet and /dev/null differ
diff --git a/python_scripts/src/python_scripts/resources/task.parquet b/python_scripts/src/python_scripts/resources/task.parquet
deleted file mode 100644
index 1e62dae2..00000000
Binary files a/python_scripts/src/python_scripts/resources/task.parquet and /dev/null differ
-- 
cgit v1.2.3