feat: removed the generated output and the python scripts to plot the results

29 files changed, 103 insertions, 44 deletions

diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py
index 896c7c3c..76b985f9 100644
--- a/python_scripts/src/python_scripts/__main__.py
+++ b/python_scripts/src/python_scripts/__main__.py
@@ -1,14 +1,8 @@
+from inspect import currentframe
+
 import redis
 import pandas as pd
 import threshold_based
-import requests
-from redis.exceptions import (
-    ConnectionError,
-    DataError,
-    NoScriptError,
-    RedisError,
-    ResponseError,
-)
 
 api = "http://localhost:1234"
 redis_host = "localhost"
@@ -16,38 +10,28 @@ stream_key = "postgres_topic"
 group = "python_consumer"
 r = redis.Redis(redis_host)
 PATH = "/home/matt/src/sunfish/python_scripts/src/python_scripts/resources"
-threshold = 0
-dictionary = {}
-
-
-def iterate(frame):
-    dictionary = {}
-    for i in range(len(frame)):
-        if frame["downtime"].iloc[i] > 0.0:
-            ts = frame["timestamp"].iloc[i]
-            dictionary[(ts)] = dictionary.get((ts), 0) + 1
-    return dictionary
-
-
-def caluclate_threshold(frame):
-    dict = iterate(frame)
-    global threshold
-    res = 0
-    for val in dict.values():
-        res += val
-    res = res / len(dict)
-    threshold = float(res)
-    return res
+similarity_scores = {
+    "Skype06Exp": 0,
+    "Skype06LogN": 0,
+    "Skype06Gam": 0,
+    "Skype06Wbl": 0,
+}
 
 
 def create_dataframes():
-    hosts: str = "%s/host.parquet" % PATH
-    tasks: str = "%s/task.parquet" % PATH
+    Skype06Exp: str = "%s/Skype06Exp/host.parquet" % PATH
+    Skype06LogN: str = "%s/Skype06LogN/host.parquet" % PATH
+    Skype06Gam: str = "%s/Skype06Gam/host.parquet" % PATH
+    Skype06Wbl: str = "%s/Skype06Wbl/host.parquet" % PATH
+    # Keep this order whenever operating
+    listOfHosts = [Skype06Exp, Skype06LogN, Skype06Gam, Skype06Wbl]
+    listOfDataFrames = []
 
     try:
-        df_hosts = pd.read_parquet(hosts)
-        df_tasks = pd.read_parquet(tasks)
-        return (df_hosts, df_tasks)
+        for item in listOfHosts:
+            df_hosts = pd.read_parquet(item)
+            listOfDataFrames.append(df_hosts)
+        return listOfDataFrames
 
     except Exception as e:
         print(f"exception caught: {e}")
@@ -60,14 +44,82 @@ As suggested in https://redis.readthedocs.io/en/stable/examples/redis-stream-exa
 toack = lambda k, g, e: r.xack(k, g, e)
 
 
-def print_xreadgroup_reply(reply, f, output_file):
+def update_score(
+    stream,
+    frames,
+):
+    global similarity_scores
+    host_stream = str(stream[b"host_id"].decode())
+    timestamp_stream = int(stream[b"timestamp"].decode())
+    current_frame = frames[0]
+
+    for i in range(len(frames)):
+        current_frame = frames[i]
+        filtered_df = current_frame[
+            (current_frame["timestamp"] == timestamp_stream)
+            & (current_frame["host_name"] == host_stream)
+        ]
+
+        y_t = float(stream[b"downtime"].decode())
+        y_t_hat = float(filtered_df["downtime"].iloc[0])
+        # Abs is mandatory here! See https://en.wikipedia.org/wiki/Euclidean_distance
+        e_t_downtime = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"cpuutilization"].decode())
+        y_t_hat = float(filtered_df["cpu_utilization"].iloc[0])
+        e_t_cpuUtil = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"energyusage"].decode())
+        y_t_hat = float(filtered_df["energy_usage"].iloc[0])
+        e_t_energyUsage = abs(y_t - y_t_hat)
+
+        y_t = float(stream[b"tasksactive"].decode())
+        y_t_hat = float(filtered_df["tasks_running"].iloc[0])
+        e_t_tasksActive = abs(y_t - y_t_hat)
+
+        S = (
+            e_t_downtime * 0.9
+            + e_t_cpuUtil * 0.7
+            + e_t_tasksActive * 0.6
+            + e_t_energyUsage * 0.7
+        )
+        S = S / (0.9 + 0.7 + 0.6 + 0.7)
+
+        res = list(similarity_scores)[i]
+        print(res)
+        oldS = similarity_scores[res]
+        print(oldS)
+        new_S = (oldS + S) / 2
+        similarity_scores[res] = new_S
+
+    total = 0
+    for value in similarity_scores.values():
+        total += value
+
+    for key, value in similarity_scores.items():
+        similarity_scores[key] = value / total
+
+    return
+
+
+def pretty_print(output_file, stream):
+    timestamp = int(stream[b"timestamp"].decode())
+    output_str = f"{timestamp}, {similarity_scores['Skype06Exp']}, {similarity_scores['Skype06LogN']}, {similarity_scores['Skype06Gam']}, {similarity_scores['Skype06Wbl']}\n"
+    output_file.write(output_str)
+
+
+def send_insights(stream, frames):
+    return
+
+
+def print_xreadgroup_reply(reply, frames, output_file):
     global threshold
     for d_stream in reply:
         for element in d_stream[1]:
-            threshold_based.check_metric(
-                element[1], f, threshold, dictionary, output_file
-            )
+            update_score(element[1], frames)
+            send_insights(element[1], frames)
             toack(d_stream[0], group, element[0])
+            pretty_print(output_file, element[1])
 
 
 def read_entries(r, f, output_file):
@@ -83,17 +135,18 @@ def read_entries(r, f, output_file):
 
 
 def main():
+    f = open("output.txt", "w")
     try:
         frames = create_dataframes()
-        caluclate_threshold(frames[0])
-
-        with open("output.txt", "w", encoding="utf-8") as f:
-            f.write(f"Threshold: {threshold}\n")
-            read_entries(r, frames[0], output_file=f)
+        read_entries(r, frames, output_file=f)
 
+        f.close()
     except Exception as e:
         print(f"exception caught: {e}")
 
+    finally:
+        f.close()
+
 
 if __name__ == "__main__":
     main()
diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc
index 705a695c..b353e8de 100644
--- a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc
+++ b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc
diff --git a/python_scripts/src/python_scripts/resources/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet
index 93b3f21c..93b3f21c 100644
--- a/python_scripts/src/python_scripts/resources/battery.parquet
+++ b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet
new file mode 100644
index 00000000..e11dc8b0
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet
new file mode 100644
index 00000000..bf7ab98d
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet
new file mode 100644
index 00000000..3d494013
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet
new file mode 100644
index 00000000..2b79e0ad
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet
new file mode 100644
index 00000000..93b3f21c
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet
new file mode 100644
index 00000000..dc98c778
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet
new file mode 100644
index 00000000..7247d165
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet
new file mode 100644
index 00000000..951429cf
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet
new file mode 100644
index 00000000..643939d1
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet
new file mode 100644
index 00000000..93b3f21c
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet
new file mode 100644
index 00000000..b145bb65
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet
new file mode 100644
index 00000000..466d3215
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet
new file mode 100644
index 00000000..33795ed3
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet
new file mode 100644
index 00000000..e356ef75
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet
new file mode 100644
index 00000000..93b3f21c
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet
new file mode 100644
index 00000000..9fdf3210
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet
new file mode 100644
index 00000000..3c79b19b
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet
new file mode 100644
index 00000000..4d40319d
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet
diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet
new file mode 100644
index 00000000..1453b91e
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet
diff --git a/python_scripts/src/python_scripts/resources/main.py b/python_scripts/src/python_scripts/resources/main.py
new file mode 100644
index 00000000..accd0314
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/main.py
@@ -0,0 +1,6 @@
+import pandas as pd
+import numpy as np
+
+df = pd.read_parquet("Skype06Gam/task.parquet")
+
+print(df.loc[:, "num_failures"].mean())
diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet
new file mode 100644
index 00000000..93b3f21c
--- /dev/null
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet
diff --git a/python_scripts/src/python_scripts/resources/host.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet
index e9add02b..e9add02b 100644
--- a/python_scripts/src/python_scripts/resources/host.parquet
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet
diff --git a/python_scripts/src/python_scripts/resources/powerSource.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet
index 304b09cf..304b09cf 100644
--- a/python_scripts/src/python_scripts/resources/powerSource.parquet
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet
diff --git a/python_scripts/src/python_scripts/resources/readme.txt b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt
index b00086b8..b00086b8 100644
--- a/python_scripts/src/python_scripts/resources/readme.txt
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt
diff --git a/python_scripts/src/python_scripts/resources/service.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet
index 371a66a7..371a66a7 100644
--- a/python_scripts/src/python_scripts/resources/service.parquet
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet
diff --git a/python_scripts/src/python_scripts/resources/task.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet
index 1e62dae2..1e62dae2 100644
--- a/python_scripts/src/python_scripts/resources/task.parquet
+++ b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet