From 15a3d5c6f55bec4a39f81f127dd1feec9660ecdc Mon Sep 17 00:00:00 2001 From: mjkwiatkowski Date: Tue, 23 Jun 2026 16:47:05 +0200 Subject: feat: added experiment2 --- python_scripts/src/python_scripts/__main__.py | 31 ++++++++++++--- .../__pycache__/__main__.cpython-314.pyc | Bin 3179 -> 3850 bytes .../__pycache__/module2.cpython-314.pyc | Bin 2134 -> 1961 bytes python_scripts/src/python_scripts/module2.old | 44 +++++++++++++++++++++ python_scripts/src/python_scripts/module2.py | 28 ++++++------- .../src/python_scripts/resources/host.parquet | Bin 1954382 -> 1976402 bytes .../python_scripts/resources/powerSource.parquet | Bin 11537 -> 11559 bytes .../src/python_scripts/resources/readme.txt | 2 + .../src/python_scripts/resources/service.parquet | Bin 4759 -> 4853 bytes .../src/python_scripts/resources/task.parquet | Bin 1526451 -> 1568225 bytes 10 files changed, 84 insertions(+), 21 deletions(-) create mode 100644 python_scripts/src/python_scripts/module2.old create mode 100644 python_scripts/src/python_scripts/resources/readme.txt (limited to 'python_scripts/src') diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py index ecddd38f..4760e17c 100644 --- a/python_scripts/src/python_scripts/__main__.py +++ b/python_scripts/src/python_scripts/__main__.py @@ -18,13 +18,28 @@ r = redis.Redis(redis_host) PATH = "/home/matt/src/sunfish/python_scripts/src/python_scripts/resources" threshold = 0 dictionary = {} +alert_dictionary_red = {} +alert_dictionary_yellow = {} + + +def iterate(frame): + dictionary = {} + for i in range(len(frame)): + if frame["downtime"].iloc[i] > 0.0: + ts = frame["timestamp"].iloc[i] + dictionary[(ts)] = dictionary.get((ts), 0) + 1 + return dictionary def caluclate_threshold(frame): + dict = iterate(frame) global threshold - df = frame[(frame["downtime"] > 0)] - threshold = df["downtime"].mean() - return threshold + res = 0 + for val in dict.values(): + res += val + res = res / len(dict) + threshold = float(res) + return res def create_dataframes(): @@ -51,7 +66,14 @@ def print_xreadgroup_reply(reply, f): global threshold for d_stream in reply: for element in d_stream[1]: - threshold = module2.check_metric(element[1], f, threshold, dictionary) + module2.check_metric( + element[1], + f, + threshold, + dictionary, + alert_dictionary_red, + alert_dictionary_yellow, + ) toack(d_stream[0], group, element[0]) @@ -75,7 +97,6 @@ def main(): read_entries(r, frames[0]) except Exception as e: - print(dictionary) print(f"exception caught: {e}") diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc index 82758955..f906b7d3 100644 Binary files a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc and b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc differ diff --git a/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc index 95473381..4dc64294 100644 Binary files a/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc and b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc differ diff --git a/python_scripts/src/python_scripts/module2.old b/python_scripts/src/python_scripts/module2.old new file mode 100644 index 00000000..3df7a768 --- /dev/null +++ b/python_scripts/src/python_scripts/module2.old @@ -0,0 +1,44 @@ +import pandas as pd +import requests +import numpy as np + +""" +The threshold should be set based on a OpenDC failure model with statistical distribution. +""" +confidence_red = 0.95 +confidence_yellow = 0.80 +sigma = 0 + + +def check_metric(stream, frame, threshold, dictionary): + timestamp = int(stream[b"timestamp"].decode()) + host = str(stream[b"host_id"].decode()) + new_threshold = threshold + filtered_df = frame[ + (frame["timestamp"] == timestamp) & (frame["host_name"] == host) + ] + y_t = float(stream[b"downtime"].decode()) + y_t_hat = float(filtered_df["downtime"].iloc[0]) + e_t = y_t - y_t_hat + + if e_t > threshold * confidence_red: + print(f"Red,{timestamp}") + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + + alert = {"Name": "Alert", "Type": "Red", "Timestamp": str(timestamp)} + alert_dt(alert) + return new_threshold + + if e_t > threshold * confidence_yellow: + print(new_threshold) + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + print(f"Yellow,{timestamp}") + alert = {"Name": "Alert", "Type": "Yellow", "Timestamp": str(timestamp)} + alert_dt(alert) + return new_threshold + + return new_threshold + + +def alert_dt(alert): + requests.post("http://localhost:1234/insight", json=alert) diff --git a/python_scripts/src/python_scripts/module2.py b/python_scripts/src/python_scripts/module2.py index 8c09d965..1d54df75 100644 --- a/python_scripts/src/python_scripts/module2.py +++ b/python_scripts/src/python_scripts/module2.py @@ -10,34 +10,30 @@ confidence_yellow = 0.80 sigma = 0 -def check_metric(stream, frame, threshold, dictionary): +def check_metric( + stream, frame, threshold, dictionary, alert_dictionary_red, alert_dictionary_yellow +): timestamp = int(stream[b"timestamp"].decode()) host = str(stream[b"host_id"].decode()) - new_threshold = threshold - filtered_df = frame[ - (frame["timestamp"] == timestamp) & (frame["host_name"] == host) - ] - y_t = float(stream[b"downtime"].decode()) - y_t_hat = float(filtered_df["downtime"].iloc[0]) - e_t = abs(y_t - y_t_hat) + + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + + e_t = dictionary.get(str(timestamp)) if e_t > threshold * confidence_red: print(f"Red,{timestamp}") - dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 - + alert_dictionary_red[str(timestamp)] = alert_dictionary_red.get(str(timestamp), 0) + 1 + print(alert_dictionary_red) alert = {"Name": "Alert", "Type": "Red", "Timestamp": str(timestamp)} alert_dt(alert) - return new_threshold + return if e_t > threshold * confidence_yellow: - print(new_threshold) - dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + alert_dictionary_yellow[str(timestamp)] = alert_dictionary_yellow.get(str(timestamp), 0) + 1 + print(alert_dictionary_yellow) print(f"Yellow,{timestamp}") alert = {"Name": "Alert", "Type": "Yellow", "Timestamp": str(timestamp)} alert_dt(alert) - return new_threshold - - return new_threshold def alert_dt(alert): diff --git a/python_scripts/src/python_scripts/resources/host.parquet b/python_scripts/src/python_scripts/resources/host.parquet index a99f3967..e9add02b 100644 Binary files a/python_scripts/src/python_scripts/resources/host.parquet and b/python_scripts/src/python_scripts/resources/host.parquet differ diff --git a/python_scripts/src/python_scripts/resources/powerSource.parquet b/python_scripts/src/python_scripts/resources/powerSource.parquet index aa5e56c0..304b09cf 100644 Binary files a/python_scripts/src/python_scripts/resources/powerSource.parquet and b/python_scripts/src/python_scripts/resources/powerSource.parquet differ diff --git a/python_scripts/src/python_scripts/resources/readme.txt b/python_scripts/src/python_scripts/resources/readme.txt new file mode 100644 index 00000000..f921b523 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/readme.txt @@ -0,0 +1,2 @@ +This is the result of a raw simulation with the failureModels set to some statistical distribution in OpenDC. +Based on the downtime in the `host.parquet` file, we calculate the threshold. diff --git a/python_scripts/src/python_scripts/resources/service.parquet b/python_scripts/src/python_scripts/resources/service.parquet index 694f1271..371a66a7 100644 Binary files a/python_scripts/src/python_scripts/resources/service.parquet and b/python_scripts/src/python_scripts/resources/service.parquet differ diff --git a/python_scripts/src/python_scripts/resources/task.parquet b/python_scripts/src/python_scripts/resources/task.parquet index fda77c58..1e62dae2 100644 Binary files a/python_scripts/src/python_scripts/resources/task.parquet and b/python_scripts/src/python_scripts/resources/task.parquet differ -- cgit v1.2.3