From 0f835d57b0e989e25aa0b71fe374a0fb1a94e86f Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Tue, 5 Nov 2024 14:17:08 +0100 Subject: Documentation update (#261) * Updated a lot of documentation, added a new get-started tutorial. * Applied Spotless * Applied Spotless Java * Added bitbrains workload to site --- site/docs/getting-started/0-installation.md | 40 +---- site/docs/getting-started/1-design.mdx | 154 ---------------- site/docs/getting-started/1-first-experiment.md | 200 +++++++++++++++++++++ site/docs/getting-started/2-experiment.mdx | 74 -------- .../documents/experiments/simple_experiment.json | 13 ++ .../getting-started/documents/topologies/big.json | 59 ++++++ .../documents/topologies/small.json | 22 +++ .../documents/workloads/bitbrains-small.zip | Bin 0 -> 573038 bytes 8 files changed, 302 insertions(+), 260 deletions(-) delete mode 100644 site/docs/getting-started/1-design.mdx create mode 100644 site/docs/getting-started/1-first-experiment.md delete mode 100644 site/docs/getting-started/2-experiment.mdx create mode 100644 site/docs/getting-started/documents/experiments/simple_experiment.json create mode 100644 site/docs/getting-started/documents/topologies/big.json create mode 100644 site/docs/getting-started/documents/topologies/small.json create mode 100644 site/docs/getting-started/documents/workloads/bitbrains-small.zip (limited to 'site/docs/getting-started') diff --git a/site/docs/getting-started/0-installation.md b/site/docs/getting-started/0-installation.md index e9f539a8..281e811f 100644 --- a/site/docs/getting-started/0-installation.md +++ b/site/docs/getting-started/0-installation.md @@ -6,7 +6,7 @@ description: How to install OpenDC locally, and start experimenting in no time. This page describes how to set up and configure a local single-user OpenDC installation so that you can quickly get your experiments running. You can also use the [hosted version of OpenDC](https://app.opendc.org) to get started even -quicker. +quicker (The web server is however missing some more complex features). ## Prerequisites @@ -14,42 +14,18 @@ quicker. 1. **Supported Platforms** OpenDC is actively tested on Windows, macOS and GNU/Linux. 2. **Required Software** - A Java installation of version 17 or higher is required for OpenDC. You may download the + A Java installation of version 19 or higher is required for OpenDC. You may download the [Java distribution from Oracle](https://www.oracle.com/java/technologies/downloads/) or use the distribution provided by your package manager. ## Download -To get an OpenDC distribution, download a recent version from our [Releases](https://github.com/atlarge-research/opendc/releases) -page on GitHub. +To get an OpenDC distribution, download a recent version from our [Releases](https://github.com/atlarge-research/opendc/releases) page on GitHub. +For basic usage, the OpenDCExperimentRunner is all that is needed. ## Setup -Unpack the downloaded OpenDC distribution and try the following command: - -```bash -$ bin/opendc-server -__ ____ __ _____ ___ __ ____ ______ - --/ __ \/ / / / _ | / _ \/ //_/ / / / __/ - -/ /_/ / /_/ / __ |/ , _/ ,< / /_/ /\ \ ---\___\_\____/_/ |_/_/|_/_/|_|\____/___/ -2022-09-12 10:30:22,064 INFO [org.fly.cor.int.dat.bas.BaseDatabaseType] (main) Database: jdbc:h2:file:./data/opendc.db (H2 2.1) -2022-09-12 10:30:22,089 WARN [org.fly.cor.int.dat.bas.Database] (main) Flyway upgrade recommended: H2 2.1.214 is newer than this version of Flyway and support has not been tested. The latest supported version of H2 is 2.1.210. -2022-09-12 10:30:22,098 INFO [org.fly.cor.int.com.DbMigrate] (main) Current version of schema "PUBLIC": 1.0.0 -2022-09-12 10:30:22,099 INFO [org.fly.cor.int.com.DbMigrate] (main) Schema "PUBLIC" is up to date. No migration necessary. -2022-09-12 10:30:22,282 INFO [org.ope.web.run.run.OpenDCRunnerRecorder] (main) Starting OpenDC Runner in background (polling every PT30S) -2022-09-12 10:30:22,347 INFO [io.quarkus] (main) opendc-web-server 2.1-rc1 on JVM (powered by Quarkus 2.11.1.Final) started in 1.366s. Listening on: http://0.0.0.0:8080 -2022-09-12 10:30:22,348 INFO [io.quarkus] (main) Profile prod activated. -2022-09-12 10:30:22,348 INFO [io.quarkus] (main) Installed features: [agroal, cdi, flyway, hibernate-orm, hibernate-validator, jdbc-h2, jdbc-postgresql, kotlin, narayana-jta, opendc-runner, opendc-ui, resteasy, resteasy-jackson, security, smallrye-simHyperVisorContext-propagation, smallrye-openapi, swagger-ui, vertx] -``` -This will launch the built-in single-user OpenDC server on port 8080. Visit -[http://localhost:8080](http://localhost:8080) to access the bundled web UI. - -## Configuration - -OpenDC can be configured using the configuration files located in the `conf` directory. By default, all user data is -stored in the `data` directory using the H2 database engine. - -## Multi-tenant deployment - -For more information on setting up multi-tenant, non-trivial deployments, see the [Deployment Guide](docs/advanced-guides/deploy.md). +Unpack the downloaded OpenDC distribution. Opening OpenDCExperimentRunner results in two folders, `bin` and `lib`. +`lib` contains all `.jar` files needed to run OpenDC. `bin` two executable versions of the OpenDCExperimentRunner. +In the following pages, we discuss how to run an experiment using the executables. + diff --git a/site/docs/getting-started/1-design.mdx b/site/docs/getting-started/1-design.mdx deleted file mode 100644 index e8ab2c58..00000000 --- a/site/docs/getting-started/1-design.mdx +++ /dev/null @@ -1,154 +0,0 @@ ---- -description: How to design a virtual datacenter in OpenDC from scratch. ---- - -# Design a Datacenter - -Now that you have installed OpenDC (or are using the hosted version), we will start designing a (virtual) datacenter -in OpenDC. - -## Before we start - -There are a couple of steps we need to perform before we can start designing a datacenter in OpenDC. First, we need to -enter the OpenDC web application. This done as follows: - -
-
-
-
-
-

Hosted Deployment

- - To enter the hosted version of OpenDC, you need a user account. User management is provided - by Auth0, which allows you to login with social accounts or via - email. - -
- -
-
-
-
-
-

Local Deployment

- - The local distribution of OpenDC runs in single-user mode by default, which does not require - authentication. This allows you to quickly start designing and experimenting with new - datacenters. - -
- -
-
-
-
- -### Create a Project - -Next, we need to create a new project. Projects allow you to organize your designs and experiments together. -Click on ‘+ New Project’ in the right corner to open the project creation dialog. -Give your project a name and save it. You can now open it by clicking on it in the project table. If all went well, -you’re redirected to your new project, and are presented with an empty project overview. - -### Create a Topology - -In OpenDC, the datacenter design is also called a **topology**. This topology represents the physical layout of a -datacenter and specifies everything from the architectural layout of the datacenter’s rooms to which CPUs are in a -particular machine. - -To create a design (topology), click on ‘+ New Topology’ in the top right corner of the topology table. -Once you have created the topology, it will appear the topology table. By clicking on the topology, you will be -redirected to a (still empty) overview of the topology. From here, we'll start designing a datacenter. - -### Terminology - -Here’s an overview of some of the language you’ll find when designing a datacenter in OpenDC: - -- **Topology**: the physical layout of your datacenter -- **Room**: a room in the datacenter -- **Tile**: one of the tiles that forms a room -- **Rack**: a rack of servers that stands on top of a tile -- **Machine**: a machine that takes up a single slot in a server rack, containing several components such as CPUs, GPUs, - network interfaces and storage drives. - -## Build the datacenter - -Open the project and topology that you have created and click on the 'Floor Plan' tab (see [Figure 1](#floor-plan)). -We’re now in datacenter construction mode. Notice the grid on the canvas? That’s where you’ll place tiles, in order to -build rooms. Let’s take a moment to familiarize ourselves with the interface. - -If you dismiss the sidebar on your left, you have controls for zooming in and out. Next to the zooming buttons, you also -have a ‘Screenshot’ button, in case you want to record the state of the canvas and export it to an image file. On the -right side of the screen, you have the simHyperVisorContext menu. This menu changes depending on your zoom level. - -As there are currently no rooms, we are in ‘Building’ mode, and our only option is to ‘Construct a new room’. Click on -that button to build a first datacenter room - once you’ve clicked on it, every tile of the canvas that you click on -becomes a tile of that room. There is one restriction though: Each tile that you add must be adjacent to any previous -tiles that you have added. You can see for yourself which tile positions are clickable through the highlight color that -is shown on hovering over them. - -
- Analysis of results reported by OpenDC -
The floor plan of a (virtual) datacenter in OpenDC.
-
- -### Create a Room - -:::note Action - -Create at least a single room, with help of the above instructions. - -::: - -Once you’ve placed the tiles, you can give the room a name, if you want to. To do this, click on the room you want to -edit. You’ll notice the application going into ‘Room’ mode, allowing you to manipulate the topology of the datacenter at -a more fine-grained level. In the simHyperVisorContext menu, change the room name, and click on the ‘Save’ button. You can exit -‘Room’ mode by clicking on any of the darkened areas outside of the selected room. This will bring you back to -‘Building’ mode. - -### Place Server Racks - -:::note Action - -Add at least a single rack in the room. - -::: - -Empty rooms are of no use to the stakeholders of a datacenter. They want machines! Let’s place some racks in the room -to fulfill this demand. Click on the room and add some racks. To stop adding racks, click on the blue element in the -sidebar, again. - -### Fill the Racks with Servers - -:::note Action - -Add a couple of servers to the rack. - -::: - -To add actual servers to the empty racks, we’ll need to go one level deeper in the topological hierarchy of the -datacenter. Clicking on a rack lets you do just that. Once you’ve clicked on it, you’ll notice the simHyperVisorContext menu now -displaying slots. In each slot fits exactly one server unit. To add such a server unit, click on the ‘Add machine’ -button of that slot. -Just like in ‘Room’ mode, you can exit ‘Rack’ mode by clicking on any of the darkened tiles around the currently -selected rack. - -### Add Resources to the Servers - -We’re almost done creating our datacenter! The only problem we have is that the machines / servers we just added lack -any real resources (such as CPUs, GPUs, memory cards, and disk storage). - -:::note Action - -Populate the machines with CPU and memory resources. - -::: - -To do this, click on any machine you want to edit. Notice the simHyperVisorContext menu changing, with tabs to add different kinds of -units to your machine. Have a look around as to what can be added. - -Once you are satisfied with the datacenter design, we will experiment with the design in the next chapter. diff --git a/site/docs/getting-started/1-first-experiment.md b/site/docs/getting-started/1-first-experiment.md new file mode 100644 index 00000000..313d757b --- /dev/null +++ b/site/docs/getting-started/1-first-experiment.md @@ -0,0 +1,200 @@ +--- +description: Designing a simple experiment +--- + +# First Experiment +Now that you have downloaded OpenDC, we will start creating a simple experiment. +In this experiment we will compare the performance of a small, and a big data center on the same workload. + +:::info Learning goal +During this tutorial, we will learn how to create and execute a simple experiment in OpenDC. +::: + +## Designing a Data Center + +The first requirement to run an experiment in OpenDC is a **topology**. +A **topology** defines the hardware on which a **workload** is executed. +Larger topologies will be capable of running more workloads, and will often quicker. + +A **topology** is defined using a JSON file. A **topology** contains one or more _clusters_. +_clusters_ are groups of _hosts_ on a specific location. Each cluster consists of one or more _hosts_. +A _host_ is a machine on which one or more tasks can be executed. _hosts_ are composed of a _cpu_ and a _memory_ unit. + +### Simple Data Center +in this experiment, we are comparing two data centers. Below is an example of the small **topology** file: + +```json +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 12, + "coreSpeed": 3300 + }, + "memory": { + "memorySize": 140457600000 + } + } + ] + } + ] +} +``` + +This **topology** consist of a single _cluster_, with a single _host_. + +:::tip +To use this **topology** in experiment copy the content to a new JSON file, or download it [here](documents/topologies/small.json "download") +::: + +### Simple Data Center +in this experiment, we are comparing two data centers. Below is an example of the bigger **topology** file: + +```json +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 32, + "coreSpeed": 3200 + }, + "memory": { + "memorySize": 256000 + } + } + ] + }, + { + "name": "C02", + "hosts" : + [ + { + "name": "H02", + "count": 6, + "cpu": + { + "coreCount": 8, + "coreSpeed": 2930 + }, + "memory": { + "memorySize": 64000 + } + } + ] + }, + { + "name": "C03", + "hosts" : + [ + { + "name": "H03", + "count": 2, + "cpu": + { + "coreCount": 16, + "coreSpeed": 3200 + }, + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} +``` + +Compared to the small topology, the big topology consist of three clusters, all consisting of a single host. + +:::tip +To use this **topology** in experiment copy the content to a new JSON file, or download it [here](documents/topologies/big.json "download") +::: + +:::info +For more in depth information about Topologies, see [Topology](../documentation/Input/Topology) +::: + +## Workloads + +Next to the topology, we need a workload to simulate on the data center. +In OpenDC, workloads are defined as a bag of tasks. Each task is accompanied by one or more fragments. +These fragments define the computational requirements of the task over time. +For this experiment, we will use the bitbrains-small workload. This is a small workload of 50 tasks, +spanning over a bit more than a month time. You can download the workload [here](documents/workloads/bitbrains-small.zip "download") + +:::info +For more in depth information about Workloads, see [Workload](../documentation/Input/Workload) +::: + +## Executing an experiment + +To run an experiment, we need to create an **experiment** file. This is a JSON file, that defines what should be executed +by OpenDC, and how. Below is an example of a simple **experiment** file: + +```json +{ + "name": "simple", + "topologies": [{ + "pathToFile": "topologies/small.json" + }, + { + "pathToFile": "topologies/big.json" + }], + "workloads": [{ + "pathToFile": "traces/bitbrains-small", + "type": "ComputeWorkload" + }] +} +``` + +In this **experiment**, three things are defined. First, is the `name`. This defines how the experiment is called +in the output folder. Second, is the `topologies`. This defines where OpenDC can find the topology files. +Finally, the `workloads`. This defines which workload OpenDC should run. You can download the experiment file [here](documents/experiments/simple_experiment.json "download") + +As you can see, `topologies` defines two topologies. In this case OpenDC will run two simulations, one with the small +topology, and one with the big topology. + +:::info +For more in depth information about Experiments, see [Experiment](../documentation/Input/Experiment) +::: + +## Running OpenDC +At this point, we should have all components to run an experiment. To make sure every file can be used by OpenDC, +please create an experiment folder such as the one shown below: +``` +── {simulation-folder-name} 📁 🔧 + ├── topologies 📁 🔒 + │ └── small.json 📄 🔧 + │ └── big.json 📄 🔧 + ├── experiments 📁 🔒 + │ └── simple_experiment.json 📄 🔧 + ├── workloads 📁 🔒 + │ └── bitbrains-small 📁 🔒 + │ └── fragments.parquet 📄 🔧 + │ └── tasks.parquet 📄 🔧 + ├── OpenDCExperimentRunner 📁 🔒 + │ └── lib 📁 🔒 + │ └── bin 📁 🔒 + ├── output 📁 🔒 +``` + +Executing the experiment can be done directly from the terminal. +Execute the following code from the terminal in simulation-folder-name + +``` +$ ./OpenDCExperimentRunner/bin/OpenDCExperimentRunner.sh --experiment-path "experiments/simple_experiment.json" +``` diff --git a/site/docs/getting-started/2-experiment.mdx b/site/docs/getting-started/2-experiment.mdx deleted file mode 100644 index 14970ea6..00000000 --- a/site/docs/getting-started/2-experiment.mdx +++ /dev/null @@ -1,74 +0,0 @@ ---- -description: How to experiment with your datacenter designs. ---- - -# Create an Experiment - -After designing a datacenter in OpenDC, the next step is to experiment with the design using OpenDC's built-in -simulator, in order to analyze its performance and compare it against other designs. - -In OpenDC, we use the concept of portfolios of scenarios to experiment with datacenter designs. In the next sections, we -will explain how you can use these powerful concepts for the experimental analysis of your designs. - -## Create a Portfolio -OpenDC organizes multiple scenarios (experiments) into a **portfolio**. Each portfolio is composed of a base scenario, -a set of candidate scenarios given by the user and a set of targets (e.g., metrics) used to compare scenarios. - -To create a new portfolio, open a project in the OpenDC web interface and click on ‘+ New Portfolio’ in the top right -corner of the portfolio table. This opens a modal with the following options: - -1. **Name**: the name of your portfolio. -2. **Metrics**: the metrics that you want to use to compare the scenarios in the portfolio. -3. **Repeats per Scenario**: the number of times each scenario should be simulated (to account for variance). - -## Create Scenarios - -A **scenario** represents a point in the datacenter design space that should be explored. It consists of a combination -of workload, topology, and a set of operational phenomena. Phenomena can include correlated failures, performance -variability, security breaches, etc., allowing the scenarios to more accurately capture the real-world operations. - -The baseline for comparison in a portfolio is the **base scenario**. It represents the status quo of the infrastructure -or, when planning infrastructure from scratch, it consists of very simple base workloads and topologies. -The other scenarios in a portfolio, called the **candidate scenarios**, represent changes to the configuration -that might be of interest to the datacenter designer. Dividing scenarios into these two categories ensures that any -comparative insights provided by OpenDC are meaningful within the simHyperVisorContext of the current architecture. - -To create a new scenario, open a portfolio in the OpenDC web interface and click on ‘+ New Scenario’ in the top right -corner of the scenario table. This opens a modal with the following options (as shown in [Figure 1](#explore)): - -1. **Name**: the name of your scenario. The first scenario of a portfolio is always called the _Base scenario_. -2. **Workload**: the applications (e.g., virtual machines, workflows, functions) that consume resources in your - datacenter. - 1. **Workload Trace**: A dataset that characterizes the historical runtime behavior of virtual machines in the - workload over time. - 2. **Load Sampling Fraction**: The percentage of the workload that should be simulated (e.g., 10% of virtual - machines in the workload trace). -3. **Environment**: - 1. **Topology**: one of the topologies of the project to use for the scenario. - 2. **Scheduler**: the algorithm that decides on which hosts the virtual machines should be placed. -4. **Operational Phenomena**: - 1. **Failures**: a flag to enable stochastic host failures during simulation. - 2. **Performance interference**: a flag to enable performance interference between virtual machines (only available - for a subset of traces). - -Once you have created the scenario, it will be enqueued for simulation. Usually the results of the simulation should be -available within one minute after creation. However, if there are lots of queued simulation jobs, it might take a bit -longer. - -
- Creating a new scenario in OpenDC -
Creating a new scenario in OpenDC. The user can select the topology, workload, and operational phenomena.
-
- -## Analyze Results - -After creating scenarios, the scenario table will show whether the simulation is still pending, completed successfully, -or failed for some reason. If the scenario was simulated successfully, its results will become visible on the ‘Results’ -tab as shown in [Figure 2](#analysis). - -
- Analysis of results reported by OpenDC -
Plots and visual summaries generated by OpenDC comparing different scenarios.
-
- -This tab will show the selected metrics for the portfolio and allow you to compare their values for different scenarios. diff --git a/site/docs/getting-started/documents/experiments/simple_experiment.json b/site/docs/getting-started/documents/experiments/simple_experiment.json new file mode 100644 index 00000000..74429fdb --- /dev/null +++ b/site/docs/getting-started/documents/experiments/simple_experiment.json @@ -0,0 +1,13 @@ +{ + "name": "simple", + "topologies": [{ + "pathToFile": "topologies/small.json" + }, + { + "pathToFile": "topologies/big.json" + }], + "workloads": [{ + "pathToFile": "traces/bitbrains-small", + "type": "ComputeWorkload" + }] +} diff --git a/site/docs/getting-started/documents/topologies/big.json b/site/docs/getting-started/documents/topologies/big.json new file mode 100644 index 00000000..c3a060cc --- /dev/null +++ b/site/docs/getting-started/documents/topologies/big.json @@ -0,0 +1,59 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 32, + "coreSpeed": 3200 + }, + "memory": { + "memorySize": 256000 + } + } + ] + }, + { + "name": "C02", + "hosts" : + [ + { + "name": "H02", + "count": 6, + "cpu": + { + "coreCount": 8, + "coreSpeed": 2930 + }, + "memory": { + "memorySize": 64000 + } + } + ] + }, + { + "name": "C03", + "hosts" : + [ + { + "name": "H03", + "count": 2, + "cpu": + { + "coreCount": 16, + "coreSpeed": 3200 + }, + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + diff --git a/site/docs/getting-started/documents/topologies/small.json b/site/docs/getting-started/documents/topologies/small.json new file mode 100644 index 00000000..54e3c6fc --- /dev/null +++ b/site/docs/getting-started/documents/topologies/small.json @@ -0,0 +1,22 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpu": + { + "coreCount": 12, + "coreSpeed": 3300 + }, + "memory": { + "memorySize": 140457600000 + } + } + ] + } + ] +} diff --git a/site/docs/getting-started/documents/workloads/bitbrains-small.zip b/site/docs/getting-started/documents/workloads/bitbrains-small.zip new file mode 100644 index 00000000..f128e636 Binary files /dev/null and b/site/docs/getting-started/documents/workloads/bitbrains-small.zip differ -- cgit v1.2.3