diff options
Diffstat (limited to 'python_scripts/src')
| -rw-r--r-- | python_scripts/src/python_scripts/__main__.py | 38 | ||||
| -rw-r--r-- | python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc | bin | 3179 -> 4058 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc | bin | 2134 -> 1964 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc | bin | 0 -> 1683 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/module2.old (renamed from python_scripts/src/python_scripts/module2.py) | 2 | ||||
| -rw-r--r-- | python_scripts/src/python_scripts/resources/host.parquet | bin | 1954382 -> 1976402 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/resources/powerSource.parquet | bin | 11537 -> 11559 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/resources/readme.txt | 7 | ||||
| -rw-r--r-- | python_scripts/src/python_scripts/resources/service.parquet | bin | 4759 -> 4853 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/resources/task.parquet | bin | 1526451 -> 1568225 bytes | |||
| -rw-r--r-- | python_scripts/src/python_scripts/threshold_based.py | 34 |
11 files changed, 69 insertions, 12 deletions
diff --git a/python_scripts/src/python_scripts/__main__.py b/python_scripts/src/python_scripts/__main__.py index ecddd38f..896c7c3c 100644 --- a/python_scripts/src/python_scripts/__main__.py +++ b/python_scripts/src/python_scripts/__main__.py @@ -1,6 +1,6 @@ import redis import pandas as pd -import module2 +import threshold_based import requests from redis.exceptions import ( ConnectionError, @@ -20,11 +20,24 @@ threshold = 0 dictionary = {} +def iterate(frame): + dictionary = {} + for i in range(len(frame)): + if frame["downtime"].iloc[i] > 0.0: + ts = frame["timestamp"].iloc[i] + dictionary[(ts)] = dictionary.get((ts), 0) + 1 + return dictionary + + def caluclate_threshold(frame): + dict = iterate(frame) global threshold - df = frame[(frame["downtime"] > 0)] - threshold = df["downtime"].mean() - return threshold + res = 0 + for val in dict.values(): + res += val + res = res / len(dict) + threshold = float(res) + return res def create_dataframes(): @@ -47,15 +60,17 @@ As suggested in https://redis.readthedocs.io/en/stable/examples/redis-stream-exa toack = lambda k, g, e: r.xack(k, g, e) -def print_xreadgroup_reply(reply, f): +def print_xreadgroup_reply(reply, f, output_file): global threshold for d_stream in reply: for element in d_stream[1]: - threshold = module2.check_metric(element[1], f, threshold, dictionary) + threshold_based.check_metric( + element[1], f, threshold, dictionary, output_file + ) toack(d_stream[0], group, element[0]) -def read_entries(r, f): +def read_entries(r, f, output_file): while True: d = r.xreadgroup( groupname=group, @@ -64,18 +79,19 @@ def read_entries(r, f): count=10, streams={stream_key: ">"}, ) - print_xreadgroup_reply(d, f) + print_xreadgroup_reply(d, f, output_file) def main(): try: frames = create_dataframes() caluclate_threshold(frames[0]) - print(threshold) - read_entries(r, frames[0]) + + with open("output.txt", "w", encoding="utf-8") as f: + f.write(f"Threshold: {threshold}\n") + read_entries(r, frames[0], output_file=f) except Exception as e: - print(dictionary) print(f"exception caught: {e}") diff --git a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc Binary files differindex 82758955..705a695c 100644 --- a/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc +++ b/python_scripts/src/python_scripts/__pycache__/__main__.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc Binary files differindex 95473381..a88d583d 100644 --- a/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc +++ b/python_scripts/src/python_scripts/__pycache__/module2.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc b/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc Binary files differnew file mode 100644 index 00000000..8817f209 --- /dev/null +++ b/python_scripts/src/python_scripts/__pycache__/threshold_based.cpython-314.pyc diff --git a/python_scripts/src/python_scripts/module2.py b/python_scripts/src/python_scripts/module2.old index 8c09d965..3df7a768 100644 --- a/python_scripts/src/python_scripts/module2.py +++ b/python_scripts/src/python_scripts/module2.old @@ -19,7 +19,7 @@ def check_metric(stream, frame, threshold, dictionary): ] y_t = float(stream[b"downtime"].decode()) y_t_hat = float(filtered_df["downtime"].iloc[0]) - e_t = abs(y_t - y_t_hat) + e_t = y_t - y_t_hat if e_t > threshold * confidence_red: print(f"Red,{timestamp}") diff --git a/python_scripts/src/python_scripts/resources/host.parquet b/python_scripts/src/python_scripts/resources/host.parquet Binary files differindex a99f3967..e9add02b 100644 --- a/python_scripts/src/python_scripts/resources/host.parquet +++ b/python_scripts/src/python_scripts/resources/host.parquet diff --git a/python_scripts/src/python_scripts/resources/powerSource.parquet b/python_scripts/src/python_scripts/resources/powerSource.parquet Binary files differindex aa5e56c0..304b09cf 100644 --- a/python_scripts/src/python_scripts/resources/powerSource.parquet +++ b/python_scripts/src/python_scripts/resources/powerSource.parquet diff --git a/python_scripts/src/python_scripts/resources/readme.txt b/python_scripts/src/python_scripts/resources/readme.txt new file mode 100644 index 00000000..b00086b8 --- /dev/null +++ b/python_scripts/src/python_scripts/resources/readme.txt @@ -0,0 +1,7 @@ +This is the result of a raw simulation with the failureModels set to some statistical distribution in OpenDC. +Based on the downtime in the `host.parquet` file, we calculate the threshold. +These results are based on this failureModel: +{ + "type": "prefab", + "prefabName" : "Pl05Exp" +} diff --git a/python_scripts/src/python_scripts/resources/service.parquet b/python_scripts/src/python_scripts/resources/service.parquet Binary files differindex 694f1271..371a66a7 100644 --- a/python_scripts/src/python_scripts/resources/service.parquet +++ b/python_scripts/src/python_scripts/resources/service.parquet diff --git a/python_scripts/src/python_scripts/resources/task.parquet b/python_scripts/src/python_scripts/resources/task.parquet Binary files differindex fda77c58..1e62dae2 100644 --- a/python_scripts/src/python_scripts/resources/task.parquet +++ b/python_scripts/src/python_scripts/resources/task.parquet diff --git a/python_scripts/src/python_scripts/threshold_based.py b/python_scripts/src/python_scripts/threshold_based.py new file mode 100644 index 00000000..13ce5e42 --- /dev/null +++ b/python_scripts/src/python_scripts/threshold_based.py @@ -0,0 +1,34 @@ +import pandas as pd +import requests +import numpy as np + +""" +The threshold should be set based on a OpenDC failure model with statistical distribution. +""" +confidence_red = 0.90 +confidence_yellow = 0.80 +sigma = 0 + + +def check_metric(stream, frame, threshold, dictionary, output_file): + timestamp = int(stream[b"timestamp"].decode()) + host = str(stream[b"host_id"].decode()) + + dictionary[str(timestamp)] = dictionary.get(str(timestamp), 0) + 1 + + e_t = dictionary.get(str(timestamp)) + # Heuristic based recognition + if e_t > threshold * confidence_red: + output_file.write(f"Timestamp: {timestamp}, Alarm: red\n") + alert = {"Name": "Alert", "Type": "Red", "Timestamp": str(timestamp)} + alert_dt(alert) + return + + if e_t > threshold * confidence_yellow: + output_file.write(f"Timestamp: {timestamp}, Alarm: yellow\n") + alert = {"Name": "Alert", "Type": "Yellow", "Timestamp": str(timestamp)} + alert_dt(alert) + + +def alert_dt(alert): + requests.post("http://localhost:1234/insight", json=alert) |