diff options
| -rw-r--r-- | appendix/appendix.tex | 72 | ||||
| -rw-r--r-- | citations/MastenbroekRADICE22.pdf | bin | 0 -> 2916697 bytes | |||
| -rw-r--r-- | content/implementation.tex | 35 | ||||
| -rw-r--r-- | images/grafana_panel.pdf | bin | 0 -> 170437 bytes | |||
| -rw-r--r-- | main.tex | 2 | ||||
| -rw-r--r-- | sources/config_files.tex | 48 | ||||
| -rw-r--r-- | style/style.tex | 2 |
7 files changed, 146 insertions, 13 deletions
diff --git a/appendix/appendix.tex b/appendix/appendix.tex index 2530a19..c86f476 100644 --- a/appendix/appendix.tex +++ b/appendix/appendix.tex @@ -1,3 +1,73 @@ \begin{appendices} - \chapter{Reproducibility} + \chapter{Reproducibility}\label{s:reproducibility} + \section{Artifact Checklist}\label{ss:artifact_checklist} + \begin{enumerate}[label=\textcolor{Green}{\textbf{\ding{51}}}] + \item Program(s): \code{OpenDC}, \code{Confluent Kafka}, \code{PostgreSQL}, \code{Redis}, \code{Grafana}. + \item Data set(s): Failure traces from Talluri \etal and failure models from Javadi \etal (see also \url{https://opendc.org/learn/documentation/Input/FailureModel}). + \item Experiments: in \Cref{s:evaluation}. + \item Time is needed to reproduce the experiments: around 4 hours. + \item Public Experiment Archive: \url{https://git.denounce.ai}. + \end{enumerate} + \section{Experimental Setup}\label{ss:experimental_setup_appendix} + Confluent Kafka, (see \url{https://www.confluent.io/}) must be present on the system. + This includes the two different connectors from \Cref{s:implementation}. + The scripts to start the Confluent Kafka (assuming installation in \code{/opt/confluent} instance are listed below. First, start Kafka Connect: + \begin{verbatim} +/opt/confluent/bin/connect-standalone \ +/opt/confluent/etc/kafka/connect-standalone.properties \ +/opt/confluent/share/confluent-common/connectors/sink-jdbc.properties \ +/opt/confluent/share/confluent-common/connectors/sink-redis.properties +\end{verbatim} + Then, format Kafka Storage: + \begin{verbatim} +/opt/confluent/bin/kafka-storage format -t \ +2vi2WtHxQAOPyXb1Bj1Jvw -c /opt/confluent/etc/kafka/server.properties \ +--standalone > /dev/null 2>&1; echo 0 + \end{verbatim} + Afterwards, start Kafka Broker: + \begin{verbatim} +/opt/confluent/bin/kafka-server-start /opt/confluent/etc/kafka/server.properties + \end{verbatim} + Then create a new Kafka Topic (\eg \code{postgres\_topic}): + \begin{verbatim} +kafka-topics --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1 + \end{verbatim} + Lastly, run the Schema Registry: + \begin{verbatim} +/opt/confluent/bin/schema-registry-start \ +/opt/confluent/etc/schema-registry/schema-registry.properties + \end{verbatim} + At this point, ensure \code{PostgreSQL}, \code{Redi} and \code{Grafana} are up and running on the system, on their default port configurations. + To see metrics flow seamlessly into \code{PostgreSQL}, once can run the following command (\ie assuming the database name is \code{opendc}): + \begin{verbatim} +sudo su postgres; psql -d opendc + \end{verbatim} + Additionally, to interact with the \code{Redis} cache, we suggest to use the excellent \code{redis-cli} tool: + \begin{verbatim} +redis-cli -h localhost -p 6379 + \end{verbatim} + The following commands can be used to list the contents of the cache (\ie assuming stream name \code{postgres\_topic}), and to clear the cache respectively: + \begin{verbatim} +XRANGE postgres_topic - + +XTRIM postgres_topic MAXLEN 0 + \end{verbatim} + For the predictive analytics in our implementation it is useful to create a separate consumer group: + \begin{verbatim} +XGROUP CREATE mystream mygroup 0 + \end{verbatim} + Lastly, each time you change the database schema, you must run (\ie assuming \code{schema.proto} lives in \code{resources/experiments}): + \begin{verbatim} +cd resources/experiments/; +protoc --java_out=/home/matt/git/opendc/opendc-common/src/main/java/ schema.proto + \end{verbatim} + Lastly, one has to checkout the \url{https://git.denounce.ai/opendc.git} and \url{https://git.denounce.ai/sunfish.git} repositories on their local machine. + \code{sunfish.git} acts as the digital twin and \code{opendc.git} acts as the physical datacenter. + Both contain modifications. + Change the directory to \code{sunfish.git}. + Create a \code{.venv} in \code{python\_scripts} and \code{http\_server} and install all the dependencies. + Then, run respectively the HTTP Server and the Python Scripts. + Afterwards, start \mysystem using IntellijIDEA. + \section{Configuration files}\label{ss:configuration_files} + \input{sources/config_files.tex} \end{appendices} + diff --git a/citations/MastenbroekRADICE22.pdf b/citations/MastenbroekRADICE22.pdf Binary files differnew file mode 100644 index 0000000..40d5a14 --- /dev/null +++ b/citations/MastenbroekRADICE22.pdf diff --git a/content/implementation.tex b/content/implementation.tex index 8db19cf..b289aa8 100644 --- a/content/implementation.tex +++ b/content/implementation.tex @@ -1,16 +1,25 @@ \chapter{Implementation of \mysystem}\label{s:implementation} -In this chapter we describe the implementation of \gls{my_system}. -The main contribution of this chapter towards answering \emph{RQ3} is the prototype of \gls{my_system}. + +\section{Overview}\label{ss:overview_implementation} + +\begin{mynote} + The contribution of this chapter is two-fold: + \vspace{-0.2cm} + \begin{enumerate}[label=\emph{C\textsubscript{\arabic*}}] + \item We implement the real-world prototype of \mysystem (see \Cref{ss:detailed_implementation_overview}) realizing key features of the design. + \item We engineer \mysystem to enable predictive analytics engines (see \Cref{ss:programming}) + \end{enumerate} +\end{mynote} + +\section{Detailed Implementation of \mysystem}\label{ss:detailed_implementation_overview} +In this section we describe the detailed implementation of \gls{my_system}. After reading one should understand the technical decisions, choice of tools and modifications to existing software necessary for evaluation of \gls{my_system} in \Cref{s:evaluation}. Any complex system is more than the sum of its parts~\cite{Wikipedia:article/Systems_Thinking}. -To understand why \gls{my_system} it is crucial to provide a holistic view on the prototype. -Therefore, the rest of the chapter is structured in a top-down approach: \Cref{ss:implementation_overview} +To understand \gls{my_system} it is crucial to provide a holistic view on the prototype. +Therefore, the rest of this chapter is structured in a top-down approach: \Cref{ss:detailed_implementation_overview} presents the rationale for using the specific software packages, \Cref{ss:data_flow} shows the flow of data within the system, and \Cref{ss:extensions} details the different modifications and new software extensions to \code{OpenDC}. Lastly, \Cref{ss:programming} carefully explains the design decisions behind the major Python modules. - -\section{Overview}\label{ss:implementation_overview} - \begin{figure}[t] \centering \includegraphics[width=0.85\linewidth]{images/implementation.pdf} @@ -27,8 +36,15 @@ To store the in-band data we use a \code{Redis} (\myCircled{3b})~\cite{Wikipedia To enable predictive analytics, we chose a discrete-event simulator, \code{OpenDC}(\myCircled{4a})~\cite{GitHub:software/OpenDC}. The \code{Analytics Engine} (\myCircled{4b}), \code{Monitoring Service} (\myCircled{4c}), and \code{HTTP Server} (\myCircled{3c}) are described in detail in \Cref{ss:programming}. -\code{Grafana} (\myCircled{2a})is a state-of-the-art industry tool to visualize dashboards. -We posit it is crucial to include a user-friendly \gls{ui}. +\begin{figure}[t] + \centering + {\includegraphics[width=0.9\linewidth]{images/grafana_panel.pdf}} + \caption[Example Grafana dashboard.]{Example Grafana Dashboard with downtime plotted against time.} + \label{fig:example_grafana_dashboard} +\end{figure} + +\code{Grafana} (\myCircled{2a}) is a state-of-the-art industry tool to visualize dashboards. +We posit it is crucial to include a user-friendly performance dashboard. A number of previous publications on \gls{dt}s find dashboards important~\cite{DBLP:conf/sc/TaheriBPRHDEWPM24, DBLP:conf/wosp/SumanCNTMI24, DBLP:conf/wosp/NicolaeTKLI26}. We chose \code{Grafana} (\myCircled{2a}) instead of other software packages because of its seamless integration with \code{PostgreSQL} (\myCircled{3a}). \code{Grafana} (\myCircled{2a}) provides good separation of concerns and compartmentalization as it does not store the displayed metrics itself. @@ -95,7 +111,6 @@ Should anything unusual occur, the \code{Monitoring Service} notifies the \code{ Then, the \code{Analytics Engine}, ingests the data from the \code{Redis} stream (\myCircled{5}) and analyzes it for further insights. All insights generated in this way, are sent to the \code{HTTP Server} (\myCircled{4}) to communicate to the system knobs within the datacenter and to the datacenter operators (\grayCircled{1}). - \begin{figure}[t] \input{sources/listing_schema.tex} \end{figure} diff --git a/images/grafana_panel.pdf b/images/grafana_panel.pdf Binary files differnew file mode 100644 index 0000000..b9549a1 --- /dev/null +++ b/images/grafana_panel.pdf @@ -16,7 +16,7 @@ \input{content/conclusion} \printglossary[type=\acronymtype] +\input{appendix/appendix} \bibliographystyle{is-unsrt} \bibliography{main} -\input{appendix/appendix} \end{document} diff --git a/sources/config_files.tex b/sources/config_files.tex new file mode 100644 index 0000000..e959942 --- /dev/null +++ b/sources/config_files.tex @@ -0,0 +1,48 @@ +\begin{lstlisting}[caption=The default experiment configuration. Here the trace is set to WhatsApp's.] +{ + "name": "greenifier-demo-scaling", + "topologies": [ + { + "pathToFile": "resources/topologies/surf.json" + } + ], + "workloads": [ + { + "pathToFile": "resources/workloads/surf_month", + "type": "ComputeWorkload" + } + ], + "failureModels" : [ + { + "type": "trace-based", + "pathToFile" : "resources/failures/Whatsapp_user_reported.parquet" + } + ], + "exportModels": [ + { + "exportInterval": 3600 + } + ] +}\end{lstlisting} +\begin{lstlisting}[caption=The default toplogy used in all experiments.] +{"clusters": [ + { + "name": "C01", + "hosts" : [ + { + "name": "H01", + "cpu": { + "coreCount": 16, + "coreSpeed": 2100 + }, + "memory": { + "memorySize": 128000000 + }, + "count": 277 + }], + "powerSource": { + "carbonTracePath": "resources/carbon_traces/NL_2021-2024.parquet" + } +}]}\end{lstlisting} + + diff --git a/style/style.tex b/style/style.tex index 6a328cc..7675240 100644 --- a/style/style.tex +++ b/style/style.tex @@ -29,7 +29,7 @@ \renewcommand\cftchapfont{\bfseries} \renewcommand\cftchappagefont{\bfseries} % Makes the chapter spacing in the TOC less. -\setlength{\cftbeforechapskip}{5pt} +\setlength{\cftbeforechapskip}{4pt} % Specifies the spacing between the lines. \renewcommand\baselinestretch{1.1} |