diff options
Diffstat (limited to 'python_scripts/src')
29 files changed, 103 insertions, 44 deletions
diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py index 896c7c3c..76b985f9 100644 --- a/python_scripts/src/python_scripts/__main__.py +++ b/python_scripts/src/python_scripts/__main__.py @@ -1,14 +1,8 @@ +from inspect import currentframe + import redis import pandas as pd import threshold_based -import requests -from redis.exceptions import ( - ConnectionError, - DataError, - NoScriptError, - RedisError, - ResponseError, -) api = "http://localhost:1234" redis_host = "localhost" @@ -16,38 +10,28 @@ stream_key = "postgres_topic" group = "python_consumer" r = redis.Redis(redis_host) PATH = "/home/matt/src/sunfish/python_scripts/src/python_scripts/resources" -threshold = 0 -dictionary = {} - - -def iterate(frame): - dictionary = {} - for i in range(len(frame)): - if frame["downtime"].iloc[i] > 0.0: - ts = frame["timestamp"].iloc[i] - dictionary[(ts)] = dictionary.get((ts), 0) + 1 - return dictionary - - -def caluclate_threshold(frame): - dict = iterate(frame) - global threshold - res = 0 - for val in dict.values(): - res += val - res = res / len(dict) - threshold = float(res) - return res +similarity_scores = { + "Skype06Exp": 0, + "Skype06LogN": 0, + "Skype06Gam": 0, + "Skype06Wbl": 0, +} def create_dataframes(): - hosts: str = "%s/host.parquet" % PATH - tasks: str = "%s/task.parquet" % PATH + Skype06Exp: str = "%s/Skype06Exp/host.parquet" % PATH + Skype06LogN: str = "%s/Skype06LogN/host.parquet" % PATH + Skype06Gam: str = "%s/Skype06Gam/host.parquet" % PATH + Skype06Wbl: str = "%s/Skype06Wbl/host.parquet" % PATH + # Keep this order whenever operating + listOfHosts = [Skype06Exp, Skype06LogN, Skype06Gam, Skype06Wbl] + listOfDataFrames = [] try: - df_hosts = pd.read_parquet(hosts) - df_tasks = pd.read_parquet(tasks) - return (df_hosts, df_tasks) + for item in listOfHosts: + df_hosts = pd.read_parquet(item) + listOfDataFrames.append(df_hosts) + return listOfDataFrames except Exception as e: print(f"exception caught: {e}") @@ -60,14 +44,82 @@ As suggested in https://redis.readthedocs.io/en/stable/examples/redis-stream-exa toack = lambda k, g, e: r.xack(k, g, e) -def print_xreadgroup_reply(reply, f, output_file): +def update_score( + stream, + frames, +): + global similarity_scores + host_stream = str(stream[b"host_id"].decode()) + timestamp_stream = int(stream[b"timestamp"].decode()) + current_frame = frames[0] + + for i in range(len(frames)): + current_frame = frames[i] + filtered_df = current_frame[ + (current_frame["timestamp"] == timestamp_stream) + & (current_frame["host_name"] == host_stream) + ] + + y_t = float(stream[b"downtime"].decode()) + y_t_hat = float(filtered_df["downtime"].iloc[0]) + # Abs is mandatory here! See https://en.wikipedia.org/wiki/Euclidean_distance + e_t_downtime = abs(y_t - y_t_hat) + + y_t = float(stream[b"cpuutilization"].decode()) + y_t_hat = float(filtered_df["cpu_utilization"].iloc[0]) + e_t_cpuUtil = abs(y_t - y_t_hat) + + y_t = float(stream[b"energyusage"].decode()) + y_t_hat = float(filtered_df["energy_usage"].iloc[0]) + e_t_energyUsage = abs(y_t - y_t_hat) + + y_t = float(stream[b"tasksactive"].decode()) + y_t_hat = float(filtered_df["tasks_running"].iloc[0]) + e_t_tasksActive = abs(y_t - y_t_hat) + + S = ( + e_t_downtime * 0.9 + + e_t_cpuUtil * 0.7 + + e_t_tasksActive * 0.6 + + e_t_energyUsage * 0.7 + ) + S = S / (0.9 + 0.7 + 0.6 + 0.7) + + res = list(similarity_scores)[i] + print(res) + oldS = similarity_scores[res] + print(oldS) + new_S = (oldS + S) / 2 + similarity_scores[res] = new_S + + total = 0 + for value in similarity_scores.values(): + total += value + + for key, value in similarity_scores.items(): + similarity_scores[key] = value / total + + return + + +def pretty_print(output_file, stream): + timestamp = int(stream[b"timestamp"].decode()) + output_str = f"{timestamp}, {similarity_scores['Skype06Exp']}, {similarity_scores['Skype06LogN']}, {similarity_scores['Skype06Gam']}, {similarity_scores['Skype06Wbl']}\n" + output_file.write(output_str) + + +def send_insights(stream, frames): + return + + +def print_xreadgroup_reply(reply, frames, output_file): global threshold for d_stream in reply: for element in d_stream[1]: - threshold_based.check_metric( - element[1], f, threshold, dictionary, output_file - ) + update_score(element[1], frames) + send_insights(element[1], frames) toack(d_stream[0], group, element[0]) + pretty_print(output_file, element[1]) def read_entries(r, f, output_file): @@ -83,17 +135,18 @@ def read_entries(r, f, output_file): def main(): + f = open("output.txt", "w") try: frames = create_dataframes() - caluclate_threshold(frames[0]) - - with open("output.txt", "w", encoding="utf-8") as f: - f.write(f"Threshold: {threshold}\n") - read_entries(r, frames[0], output_file=f) + read_entries(r, frames, output_file=f) + f.close() except Exception as e: print(f"exception caught: {e}") + finally: + f.close() + if __name__ == "__main__": main() diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc Binary files differindex 705a695c..b353e8de 100644 --- a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc +++ b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/resources/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet Binary files differindex 93b3f21c..93b3f21c 100644 --- a/python_scripts/src/python_scripts/resources/battery.parquet +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet Binary files differnew file mode 100644 index 00000000..e11dc8b0 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet Binary files differnew file mode 100644 index 00000000..bf7ab98d --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet Binary files differnew file mode 100644 index 00000000..3d494013 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet Binary files differnew file mode 100644 index 00000000..2b79e0ad --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet Binary files differnew file mode 100644 index 00000000..dc98c778 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet Binary files differnew file mode 100644 index 00000000..7247d165 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet Binary files differnew file mode 100644 index 00000000..951429cf --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet Binary files differnew file mode 100644 index 00000000..643939d1 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet Binary files differnew file mode 100644 index 00000000..b145bb65 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet Binary files differnew file mode 100644 index 00000000..466d3215 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet Binary files differnew file mode 100644 index 00000000..33795ed3 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet Binary files differnew file mode 100644 index 00000000..e356ef75 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet Binary files differnew file mode 100644 index 00000000..9fdf3210 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet Binary files differnew file mode 100644 index 00000000..3c79b19b --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet Binary files differnew file mode 100644 index 00000000..4d40319d --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet Binary files differnew file mode 100644 index 00000000..1453b91e --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet diff --git a/python_scripts/src/python_scripts/resources/main.py b/python_scripts/src/python_scripts/resources/main.py new file mode 100644 index 00000000..accd0314 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/main.py @@ -0,0 +1,6 @@ +import pandas as pd +import numpy as np + +df = pd.read_parquet("Skype06Gam/task.parquet") + +print(df.loc[:, "num_failures"].mean()) diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/host.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet Binary files differindex e9add02b..e9add02b 100644 --- a/python_scripts/src/python_scripts/resources/host.parquet +++ b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet diff --git a/python_scripts/src/python_scripts/resources/powerSource.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet Binary files differindex 304b09cf..304b09cf 100644 --- a/python_scripts/src/python_scripts/resources/powerSource.parquet +++ b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/readme.txt b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt index b00086b8..b00086b8 100644 --- a/python_scripts/src/python_scripts/resources/readme.txt +++ b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt diff --git a/python_scripts/src/python_scripts/resources/service.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet Binary files differindex 371a66a7..371a66a7 100644 --- a/python_scripts/src/python_scripts/resources/service.parquet +++ b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet diff --git a/python_scripts/src/python_scripts/resources/task.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet Binary files differindex 1e62dae2..1e62dae2 100644 --- a/python_scripts/src/python_scripts/resources/task.parquet +++ b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet |