diff options
Diffstat (limited to 'python_scripts/src')
34 files changed, 243 insertions, 0 deletions
diff --git a/python_scripts/src/python_scripts/__init__.py b/python_scripts/src/python_scripts/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/python_scripts/src/python_scripts/__init__.py diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py new file mode 100644 index 00000000..76b985f9 --- /dev/null +++ b/python_scripts/src/python_scripts/__main__.py @@ -0,0 +1,152 @@ +from inspect import currentframe + +import redis +import pandas as pd +import threshold_based + +api = "http://localhost:1234" +redis_host = "localhost" +stream_key = "postgres_topic" +group = "python_consumer" +r = redis.Redis(redis_host) +PATH = "/home/matt/src/sunfish/python_scripts/src/python_scripts/resources" +similarity_scores = { + "Skype06Exp": 0, + "Skype06LogN": 0, + "Skype06Gam": 0, + "Skype06Wbl": 0, +} + + +def create_dataframes(): + Skype06Exp: str = "%s/Skype06Exp/host.parquet" % PATH + Skype06LogN: str = "%s/Skype06LogN/host.parquet" % PATH + Skype06Gam: str = "%s/Skype06Gam/host.parquet" % PATH + Skype06Wbl: str = "%s/Skype06Wbl/host.parquet" % PATH + # Keep this order whenever operating + listOfHosts = [Skype06Exp, Skype06LogN, Skype06Gam, Skype06Wbl] + listOfDataFrames = [] + + try: + for item in listOfHosts: + df_hosts = pd.read_parquet(item) + listOfDataFrames.append(df_hosts) + return listOfDataFrames + + except Exception as e: + print(f"exception caught: {e}") + exit(1) + + +""" +As suggested in https://redis.readthedocs.io/en/stable/examples/redis-stream-example.html +""" +toack = lambda k, g, e: r.xack(k, g, e) + + +def update_score( + stream, + frames, +): + global similarity_scores + host_stream = str(stream[b"host_id"].decode()) + timestamp_stream = int(stream[b"timestamp"].decode()) + current_frame = frames[0] + + for i in range(len(frames)): + current_frame = frames[i] + filtered_df = current_frame[ + (current_frame["timestamp"] == timestamp_stream) + & (current_frame["host_name"] == host_stream) + ] + + y_t = float(stream[b"downtime"].decode()) + y_t_hat = float(filtered_df["downtime"].iloc[0]) + # Abs is mandatory here! See https://en.wikipedia.org/wiki/Euclidean_distance + e_t_downtime = abs(y_t - y_t_hat) + + y_t = float(stream[b"cpuutilization"].decode()) + y_t_hat = float(filtered_df["cpu_utilization"].iloc[0]) + e_t_cpuUtil = abs(y_t - y_t_hat) + + y_t = float(stream[b"energyusage"].decode()) + y_t_hat = float(filtered_df["energy_usage"].iloc[0]) + e_t_energyUsage = abs(y_t - y_t_hat) + + y_t = float(stream[b"tasksactive"].decode()) + y_t_hat = float(filtered_df["tasks_running"].iloc[0]) + e_t_tasksActive = abs(y_t - y_t_hat) + + S = ( + e_t_downtime * 0.9 + + e_t_cpuUtil * 0.7 + + e_t_tasksActive * 0.6 + + e_t_energyUsage * 0.7 + ) + S = S / (0.9 + 0.7 + 0.6 + 0.7) + + res = list(similarity_scores)[i] + print(res) + oldS = similarity_scores[res] + print(oldS) + new_S = (oldS + S) / 2 + similarity_scores[res] = new_S + + total = 0 + for value in similarity_scores.values(): + total += value + + for key, value in similarity_scores.items(): + similarity_scores[key] = value / total + + return + + +def pretty_print(output_file, stream): + timestamp = int(stream[b"timestamp"].decode()) + output_str = f"{timestamp}, {similarity_scores['Skype06Exp']}, {similarity_scores['Skype06LogN']}, {similarity_scores['Skype06Gam']}, {similarity_scores['Skype06Wbl']}\n" + output_file.write(output_str) + + +def send_insights(stream, frames): + return + + +def print_xreadgroup_reply(reply, frames, output_file): + global threshold + for d_stream in reply: + for element in d_stream[1]: + update_score(element[1], frames) + send_insights(element[1], frames) + toack(d_stream[0], group, element[0]) + pretty_print(output_file, element[1]) + + +def read_entries(r, f, output_file): + while True: + d = r.xreadgroup( + groupname=group, + consumername="c", + block=5, + count=10, + streams={stream_key: ">"}, + ) + print_xreadgroup_reply(d, f, output_file) + + +def main(): + f = open("output.txt", "w") + try: + frames = create_dataframes() + read_entries(r, frames, output_file=f) + + f.close() + except Exception as e: + print(f"exception caught: {e}") + + finally: + f.close() + + +if __name__ == "__main__": + main() diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc Binary files differnew file mode 100644 index 00000000..b353e8de --- /dev/null +++ b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc Binary files differnew file mode 100644 index 00000000..a88d583d --- /dev/null +++ b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc Binary files differnew file mode 100644 index 00000000..8817f209 --- /dev/null +++ b/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/module2.old b/python_scripts/src/python_scripts/module2.old new file mode 100644 index 00000000..3df7a768 --- /dev/null +++ b/python_scripts/src/python_scripts/module2.old @@ -0,0 +1,44 @@ +import pandas as pd +import requests +import numpy as np + +""" +The threshold should be set based on a OpenDC failure model with statistical distribution. +""" +confidence_red = 0.95 +confidence_yellow = 0.80 +sigma = 0 + + +def check_metric(stream, frame, threshold, dictionary): + timestamp = int(stream[b"timestamp"].decode()) + host = str(stream[b"host_id"].decode()) + new_threshold = threshold + filtered_df = frame[ + (frame["timestamp"] == timestamp) & (frame["host_name"] == host) + ] + y_t = float(stream[b"downtime"].decode()) + y_t_hat = float(filtered_df["downtime"].iloc[0]) + e_t = y_t - y_t_hat + + if e_t > threshold * confidence_red: + print(f"Red,{timestamp}") + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + + alert = {"Name": "Alert", "Type": "Red", "Timestamp": str(timestamp)} + alert_dt(alert) + return new_threshold + + if e_t > threshold * confidence_yellow: + print(new_threshold) + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + print(f"Yellow,{timestamp}") + alert = {"Name": "Alert", "Type": "Yellow", "Timestamp": str(timestamp)} + alert_dt(alert) + return new_threshold + + return new_threshold + + +def alert_dt(alert): + requests.post("http://localhost:1234/insight", json=alert) diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet Binary files differnew file mode 100644 index 00000000..e11dc8b0 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet Binary files differnew file mode 100644 index 00000000..bf7ab98d --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet Binary files differnew file mode 100644 index 00000000..3d494013 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet Binary files differnew file mode 100644 index 00000000..2b79e0ad --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Exp/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet Binary files differnew file mode 100644 index 00000000..dc98c778 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet Binary files differnew file mode 100644 index 00000000..7247d165 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet Binary files differnew file mode 100644 index 00000000..951429cf --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet Binary files differnew file mode 100644 index 00000000..643939d1 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Gam/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet Binary files differnew file mode 100644 index 00000000..b145bb65 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet Binary files differnew file mode 100644 index 00000000..466d3215 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet Binary files differnew file mode 100644 index 00000000..33795ed3 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet Binary files differnew file mode 100644 index 00000000..e356ef75 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06LogN/task.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet Binary files differnew file mode 100644 index 00000000..9fdf3210 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/host.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet Binary files differnew file mode 100644 index 00000000..3c79b19b --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet Binary files differnew file mode 100644 index 00000000..4d40319d --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/service.parquet diff --git a/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet Binary files differnew file mode 100644 index 00000000..1453b91e --- /dev/null +++ b/python_scripts/src/python_scripts/resources/Skype06Wbl/task.parquet diff --git a/python_scripts/src/python_scripts/resources/main.py b/python_scripts/src/python_scripts/resources/main.py new file mode 100644 index 00000000..accd0314 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/main.py @@ -0,0 +1,6 @@ +import pandas as pd +import numpy as np + +df = pd.read_parquet("Skype06Gam/task.parquet") + +print(df.loc[:, "num_failures"].mean()) diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet Binary files differnew file mode 100644 index 00000000..93b3f21c --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/battery.parquet diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet Binary files differnew file mode 100644 index 00000000..e9add02b --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/host.parquet diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet Binary files differnew file mode 100644 index 00000000..304b09cf --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt new file mode 100644 index 00000000..b00086b8 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/readme.txt @@ -0,0 +1,7 @@ +This is the result of a raw simulation with the failureModels set to some statistical distribution in OpenDC. +Based on the downtime in the `host.parquet` file, we calculate the threshold. +These results are based on this failureModel: +{ + "type": "prefab", + "prefabName" : "Pl05Exp" +} diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet Binary files differnew file mode 100644 index 00000000..371a66a7 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/service.parquet diff --git a/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet Binary files differnew file mode 100644 index 00000000..1e62dae2 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/prefabs_old/task.parquet diff --git a/python_scripts/src/python_scripts/threshold_based.py b/python_scripts/src/python_scripts/threshold_based.py new file mode 100644 index 00000000..13ce5e42 --- /dev/null +++ b/python_scripts/src/python_scripts/threshold_based.py @@ -0,0 +1,34 @@ +import pandas as pd +import requests +import numpy as np + +""" +The threshold should be set based on a OpenDC failure model with statistical distribution. +""" +confidence_red = 0.90 +confidence_yellow = 0.80 +sigma = 0 + + +def check_metric(stream, frame, threshold, dictionary, output_file): + timestamp = int(stream[b"timestamp"].decode()) + host = str(stream[b"host_id"].decode()) + + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + + e_t = dictionary.get(str(timestamp)) + # Heuristic based recognition + if e_t > threshold * confidence_red: + output_file.write(f"Timestamp: {timestamp}, Alarm: red\n") + alert = {"Name": "Alert", "Type": "Red", "Timestamp": str(timestamp)} + alert_dt(alert) + return + + if e_t > threshold * confidence_yellow: + output_file.write(f"Timestamp: {timestamp}, Alarm: yellow\n") + alert = {"Name": "Alert", "Type": "Yellow", "Timestamp": str(timestamp)} + alert_dt(alert) + + +def alert_dt(alert): + requests.post("http://localhost:1234/insight", json=alert) |