diff options
Diffstat (limited to 'opendc-experiments')
21 files changed, 752 insertions, 112 deletions
diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt index 6b538240..9a00c80e 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt @@ -40,8 +40,8 @@ import org.opendc.compute.simulator.provisioner.setupHosts import org.opendc.compute.telemetry.ComputeMonitor import org.opendc.compute.telemetry.table.HostTableReader import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.topology.HostSpec import org.opendc.compute.topology.clusterTopology +import org.opendc.compute.topology.specs.HostSpec import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.VirtualMachine import org.opendc.compute.workload.sampleByLoad @@ -126,7 +126,7 @@ class CapelinIntegrationTest { { assertEquals(66977091124, monitor.activeTime) { "Incorrect active time" } }, { assertEquals(3160267873, monitor.stealTime) { "Incorrect steal time" } }, { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.8407E9, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, + { assertEquals(7.767237E9, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, ) } @@ -138,7 +138,7 @@ class CapelinIntegrationTest { runSimulation { val seed = 1L val workload = createTestWorkload(0.25, seed) - val topology = createTopology("single") + val topology = createTopology("single.json") val monitor = monitor Provisioner(dispatcher, seed).use { provisioner -> @@ -167,7 +167,7 @@ class CapelinIntegrationTest { { assertEquals(9741285381, monitor.activeTime) { "Active time incorrect" } }, { assertEquals(152, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, - { assertEquals(7.0109E8, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, + { assertEquals(7.933686E8, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, ) } @@ -179,7 +179,7 @@ class CapelinIntegrationTest { runSimulation { val seed = 0L val workload = createTestWorkload(1.0, seed) - val topology = createTopology("single") + val topology = createTopology("single.json") Provisioner(dispatcher, seed).use { provisioner -> provisioner.runSteps( @@ -217,7 +217,7 @@ class CapelinIntegrationTest { fun testFailures() = runSimulation { val seed = 0L - val topology = createTopology("single") + val topology = createTopology("single.json") val workload = createTestWorkload(0.25, seed) val monitor = monitor @@ -256,8 +256,8 @@ class CapelinIntegrationTest { /** * Obtain the topology factory for the test. */ - private fun createTopology(name: String = "topology"): List<HostSpec> { - val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name.txt")) + private fun createTopology(name: String = "topology.json"): List<HostSpec> { + val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name")) return stream.use { clusterTopology(stream) } } diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinRunnerTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinRunnerTest.kt index 32d53aee..4587f6dc 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinRunnerTest.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinRunnerTest.kt @@ -46,7 +46,8 @@ class CapelinRunnerTest { private val tracePath = File("src/test/resources/trace") /** - * Smoke test with output. fixme: Fix failures and enable Test + * Smoke test with output. + * fixme: Fix failures and enable */ fun testSmoke() { val outputPath = Files.createTempDirectory("output").toFile() @@ -68,7 +69,8 @@ class CapelinRunnerTest { } /** - * Smoke test without output. fixme: Fix failures and enable Test + * Smoke test without output. + * fixme: Fix failures and enable */ fun testSmokeNoOutput() { val runner = CapelinRunner(envPath, tracePath, null) diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.json b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.json new file mode 100644 index 00000000..a1c8d95a --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.json @@ -0,0 +1,26 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt deleted file mode 100644 index 5642003d..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/single.txt +++ /dev/null @@ -1,3 +0,0 @@ -ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;8;3.2;128;1;128;8 - diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.json b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.json new file mode 100644 index 00000000..721005b0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.json @@ -0,0 +1,66 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 32, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 256000 + } + } + ] + }, + { + "name": "C02", + "hosts" : + [ + { + "name": "H02", + "count": 6, + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 2930 + } + ], + "memory": { + "memorySize": 64000 + } + } + ] + }, + { + "name": "C03", + "hosts" : + [ + { + "name": "H03", + "count": 2, + "cpus": + [ + { + "coreCount": 16, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.txt b/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.txt deleted file mode 100644 index 6b347bff..00000000 --- a/opendc-experiments/opendc-experiments-capelin/src/test/resources/env/topology.txt +++ /dev/null @@ -1,5 +0,0 @@ -ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;32;3.2;2048;1;256;32 -B01;B01;48;2.93;1256;6;64;8 -C01;C01;32;3.2;2048;2;128;16 - diff --git a/opendc-experiments/opendc-experiments-greenifier/src/jmh/kotlin/org/opendc/experiments/greenifier/GreenifierBenchmarks.kt b/opendc-experiments/opendc-experiments-greenifier/src/jmh/kotlin/org/opendc/experiments/greenifier/GreenifierBenchmarks.kt index 6cc6df36..85ee6b82 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/jmh/kotlin/org/opendc/experiments/greenifier/GreenifierBenchmarks.kt +++ b/opendc-experiments/opendc-experiments-greenifier/src/jmh/kotlin/org/opendc/experiments/greenifier/GreenifierBenchmarks.kt @@ -31,8 +31,8 @@ import org.opendc.compute.service.scheduler.weights.CoreRamWeigher import org.opendc.compute.simulator.provisioner.Provisioner import org.opendc.compute.simulator.provisioner.setupComputeService import org.opendc.compute.simulator.provisioner.setupHosts -import org.opendc.compute.topology.HostSpec import org.opendc.compute.topology.clusterTopology +import org.opendc.compute.topology.specs.HostSpec import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.VirtualMachine import org.opendc.compute.workload.trace diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/Python_scripts/OpenDCdemo.ipynb b/opendc-experiments/opendc-experiments-greenifier/src/main/Python_scripts/OpenDCdemo.ipynb index 09ff26d6..9bee908e 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/main/Python_scripts/OpenDCdemo.ipynb +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/Python_scripts/OpenDCdemo.ipynb @@ -128,10 +128,10 @@ " <td>A01</td>\n", " <td>A01</td>\n", " <td>8</td>\n", - " <td>3.2</td>\n", - " <td>128</td>\n", " <td>1</td>\n", - " <td>128</td>\n", + " <td>100</td>\n", + " <td>1</td>\n", + " <td>100</td>\n", " <td>8</td>\n", " </tr>\n", " </tbody>\n", @@ -200,55 +200,55 @@ " <tbody>\n", " <tr>\n", " <th>0</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 13:40:46+00:00</td>\n", - " <td>300000</td>\n", + " <td>0</td>\n", + " <td>2024-02-02</td>\n", + " <td>86400000</td>\n", " <td>1</td>\n", - " <td>0.000000</td>\n", + " <td>1000.0</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 13:45:46+00:00</td>\n", - " <td>300000</td>\n", " <td>1</td>\n", - " <td>11.703998</td>\n", + " <td>2024-02-02</td>\n", + " <td>86400000</td>\n", + " <td>1</td>\n", + " <td>1000.0</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 13:55:46+00:00</td>\n", - " <td>600000</td>\n", + " <td>2</td>\n", + " <td>2024-02-02</td>\n", + " <td>86400000</td>\n", " <td>1</td>\n", - " <td>0.000000</td>\n", + " <td>1000.0</td>\n", " </tr>\n", " <tr>\n", " <th>3</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 14:00:46+00:00</td>\n", - " <td>300000</td>\n", + " <td>3</td>\n", + " <td>2024-02-02</td>\n", + " <td>86400000</td>\n", " <td>1</td>\n", - " <td>11.703998</td>\n", + " <td>1000.0</td>\n", " </tr>\n", " <tr>\n", " <th>4</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 14:15:46+00:00</td>\n", - " <td>900000</td>\n", + " <td>4</td>\n", + " <td>2024-02-02</td>\n", + " <td>86400000</td>\n", " <td>1</td>\n", - " <td>0.000000</td>\n", + " <td>1000.0</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "</div>" ], "text/plain": [ - " id timestamp duration cpu_count cpu_usage\n", - "0 1019 2013-08-12 13:40:46+00:00 300000 1 0.000000\n", - "1 1019 2013-08-12 13:45:46+00:00 300000 1 11.703998\n", - "2 1019 2013-08-12 13:55:46+00:00 600000 1 0.000000\n", - "3 1019 2013-08-12 14:00:46+00:00 300000 1 11.703998\n", - "4 1019 2013-08-12 14:15:46+00:00 900000 1 0.000000" + " id timestamp duration cpu_count cpu_usage\n", + "0 0 2024-02-02 86400000 1 1000.0\n", + "1 1 2024-02-02 86400000 1 1000.0\n", + "2 2 2024-02-02 86400000 1 1000.0\n", + "3 3 2024-02-02 86400000 1 1000.0\n", + "4 4 2024-02-02 86400000 1 1000.0" ] }, "execution_count": 3, @@ -257,7 +257,7 @@ } ], "source": [ - "df_trace = pd.read_parquet(f\"{base_folder}/resources/bitbrains-small/trace/trace.parquet\")\n", + "df_trace = pd.read_parquet(f\"{base_folder}/resources/benchmark/trace/trace.parquet\")\n", "df_trace.head()" ] }, @@ -301,67 +301,60 @@ " <tbody>\n", " <tr>\n", " <th>0</th>\n", - " <td>1019</td>\n", - " <td>2013-08-12 13:35:46+00:00</td>\n", - " <td>2013-09-11 13:39:58+00:00</td>\n", + " <td>0</td>\n", + " <td>2024-02-01</td>\n", + " <td>2024-02-02</td>\n", " <td>1</td>\n", - " <td>2926.000135</td>\n", - " <td>181352</td>\n", + " <td>1000.0</td>\n", + " <td>100000</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", - " <td>1023</td>\n", - " <td>2013-08-12 13:35:46+00:00</td>\n", - " <td>2013-09-11 13:39:58+00:00</td>\n", " <td>1</td>\n", - " <td>2925.999560</td>\n", - " <td>260096</td>\n", + " <td>2024-02-01</td>\n", + " <td>2024-02-02</td>\n", + " <td>1</td>\n", + " <td>1000.0</td>\n", + " <td>100000</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", - " <td>1026</td>\n", - " <td>2013-08-12 13:35:46+00:00</td>\n", - " <td>2013-09-11 13:39:58+00:00</td>\n", + " <td>2</td>\n", + " <td>2024-02-01</td>\n", + " <td>2024-02-02</td>\n", " <td>1</td>\n", - " <td>2925.999717</td>\n", - " <td>249972</td>\n", + " <td>1000.0</td>\n", + " <td>100000</td>\n", " </tr>\n", " <tr>\n", " <th>3</th>\n", - " <td>1052</td>\n", - " <td>2013-08-29 14:38:12+00:00</td>\n", - " <td>2013-09-05 07:09:07+00:00</td>\n", + " <td>3</td>\n", + " <td>2024-02-01</td>\n", + " <td>2024-02-02</td>\n", " <td>1</td>\n", - " <td>2926.000107</td>\n", - " <td>131245</td>\n", + " <td>1000.0</td>\n", + " <td>100000</td>\n", " </tr>\n", " <tr>\n", " <th>4</th>\n", - " <td>1073</td>\n", - " <td>2013-08-21 11:07:12+00:00</td>\n", - " <td>2013-09-11 13:39:58+00:00</td>\n", + " <td>4</td>\n", + " <td>2024-02-01</td>\n", + " <td>2024-02-02</td>\n", " <td>1</td>\n", - " <td>2599.999649</td>\n", - " <td>179306</td>\n", + " <td>1000.0</td>\n", + " <td>100000</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "</div>" ], "text/plain": [ - " id start_time stop_time cpu_count \\\n", - "0 1019 2013-08-12 13:35:46+00:00 2013-09-11 13:39:58+00:00 1 \n", - "1 1023 2013-08-12 13:35:46+00:00 2013-09-11 13:39:58+00:00 1 \n", - "2 1026 2013-08-12 13:35:46+00:00 2013-09-11 13:39:58+00:00 1 \n", - "3 1052 2013-08-29 14:38:12+00:00 2013-09-05 07:09:07+00:00 1 \n", - "4 1073 2013-08-21 11:07:12+00:00 2013-09-11 13:39:58+00:00 1 \n", - "\n", - " cpu_capacity mem_capacity \n", - "0 2926.000135 181352 \n", - "1 2925.999560 260096 \n", - "2 2925.999717 249972 \n", - "3 2926.000107 131245 \n", - "4 2599.999649 179306 " + " id start_time stop_time cpu_count cpu_capacity mem_capacity\n", + "0 0 2024-02-01 2024-02-02 1 1000.0 100000\n", + "1 1 2024-02-01 2024-02-02 1 1000.0 100000\n", + "2 2 2024-02-01 2024-02-02 1 1000.0 100000\n", + "3 3 2024-02-01 2024-02-02 1 1000.0 100000\n", + "4 4 2024-02-01 2024-02-02 1 1000.0 100000" ] }, "execution_count": 4, @@ -370,7 +363,7 @@ } ], "source": [ - "df_meta = pd.read_parquet(f\"{base_folder}/resources/bitbrains-small/trace/meta.parquet\")\n", + "df_meta = pd.read_parquet(f\"{base_folder}/resources/benchmark/trace/meta.parquet\")\n", "df_meta.head()" ] }, @@ -1095,6 +1088,390 @@ "output_file_path = Path(output_file)\n", "df_trace_new.to_parquet(output_file_path, index=False)" ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d80ee1db", + "metadata": {}, + "outputs": [], + "source": [ + "df_server = pd.read_parquet(f\"../output/topology=topology.json/workload=benchmark/seed=0/server.parquet\")\n", + "df_host = pd.read_parquet(f\"../output/topology=topology.json/workload=benchmark/seed=0/host.parquet\")\n", + "df_service = pd.read_parquet(f\"../output/topology=topology.json/workload=benchmark/seed=0/service.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "4ec05a5b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>timestamp</th>\n", + " <th>hosts_up</th>\n", + " <th>hosts_down</th>\n", + " <th>servers_pending</th>\n", + " <th>servers_active</th>\n", + " <th>attempts_success</th>\n", + " <th>attempts_failure</th>\n", + " <th>attempts_error</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>300000</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>10</td>\n", + " <td>10</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>600000</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>10</td>\n", + " <td>10</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>900000</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>10</td>\n", + " <td>10</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>1200000</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>10</td>\n", + " <td>10</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>1500000</td>\n", + " <td>2</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>10</td>\n", + " <td>10</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "</div>" + ], + "text/plain": [ + " timestamp hosts_up hosts_down servers_pending servers_active \\\n", + "0 300000 2 0 0 10 \n", + "1 600000 2 0 0 10 \n", + "2 900000 2 0 0 10 \n", + "3 1200000 2 0 0 10 \n", + "4 1500000 2 0 0 10 \n", + "\n", + " attempts_success attempts_failure attempts_error \n", + "0 10 0 0 \n", + "1 10 0 0 \n", + "2 10 0 0 \n", + "3 10 0 0 \n", + "4 10 0 0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_service.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "7f147582", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>timestamp</th>\n", + " <th>host_id</th>\n", + " <th>cpu_count</th>\n", + " <th>mem_capacity</th>\n", + " <th>guests_terminated</th>\n", + " <th>guests_running</th>\n", + " <th>guests_error</th>\n", + " <th>guests_invalid</th>\n", + " <th>cpu_limit</th>\n", + " <th>cpu_usage</th>\n", + " <th>...</th>\n", + " <th>cpu_utilization</th>\n", + " <th>cpu_time_active</th>\n", + " <th>cpu_time_idle</th>\n", + " <th>cpu_time_steal</th>\n", + " <th>cpu_time_lost</th>\n", + " <th>power_draw</th>\n", + " <th>energy_usage</th>\n", + " <th>uptime</th>\n", + " <th>downtime</th>\n", + " <th>boot_time</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>300000</td>\n", + " <td>e220a839-7b1d-cdaf-0000-000000000000</td>\n", + " <td>6</td>\n", + " <td>100000</td>\n", + " <td>0</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>8000.0</td>\n", + " <td>5000.0</td>\n", + " <td>...</td>\n", + " <td>0.625</td>\n", + " <td>1125002</td>\n", + " <td>674998</td>\n", + " <td>18</td>\n", + " <td>0</td>\n", + " <td>325.0</td>\n", + " <td>97500.075</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>300000</td>\n", + " <td>6e789e6a-a1b9-65f4-0000-000000000001</td>\n", + " <td>4</td>\n", + " <td>100000</td>\n", + " <td>0</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>4000.0</td>\n", + " <td>4000.0</td>\n", + " <td>...</td>\n", + " <td>1.000</td>\n", + " <td>1200000</td>\n", + " <td>0</td>\n", + " <td>300011</td>\n", + " <td>0</td>\n", + " <td>400.0</td>\n", + " <td>120000.000</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>600000</td>\n", + " <td>e220a839-7b1d-cdaf-0000-000000000000</td>\n", + " <td>6</td>\n", + " <td>100000</td>\n", + " <td>0</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>8000.0</td>\n", + " <td>5000.0</td>\n", + " <td>...</td>\n", + " <td>0.625</td>\n", + " <td>1125000</td>\n", + " <td>675000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>325.0</td>\n", + " <td>97500.000</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>600000</td>\n", + " <td>6e789e6a-a1b9-65f4-0000-000000000001</td>\n", + " <td>4</td>\n", + " <td>100000</td>\n", + " <td>0</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>4000.0</td>\n", + " <td>4000.0</td>\n", + " <td>...</td>\n", + " <td>1.000</td>\n", + " <td>1200000</td>\n", + " <td>0</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>400.0</td>\n", + " <td>120000.000</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>900000</td>\n", + " <td>e220a839-7b1d-cdaf-0000-000000000000</td>\n", + " <td>6</td>\n", + " <td>100000</td>\n", + " <td>0</td>\n", + " <td>5</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>8000.0</td>\n", + " <td>5000.0</td>\n", + " <td>...</td>\n", + " <td>0.625</td>\n", + " <td>1125000</td>\n", + " <td>675000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " <td>325.0</td>\n", + " <td>97500.000</td>\n", + " <td>300000</td>\n", + " <td>0</td>\n", + " <td>0</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 21 columns</p>\n", + "</div>" + ], + "text/plain": [ + " timestamp host_id cpu_count mem_capacity \\\n", + "0 300000 e220a839-7b1d-cdaf-0000-000000000000 6 100000 \n", + "1 300000 6e789e6a-a1b9-65f4-0000-000000000001 4 100000 \n", + "2 600000 e220a839-7b1d-cdaf-0000-000000000000 6 100000 \n", + "3 600000 6e789e6a-a1b9-65f4-0000-000000000001 4 100000 \n", + "4 900000 e220a839-7b1d-cdaf-0000-000000000000 6 100000 \n", + "\n", + " guests_terminated guests_running guests_error guests_invalid cpu_limit \\\n", + "0 0 5 0 0 8000.0 \n", + "1 0 5 0 0 4000.0 \n", + "2 0 5 0 0 8000.0 \n", + "3 0 5 0 0 4000.0 \n", + "4 0 5 0 0 8000.0 \n", + "\n", + " cpu_usage ... cpu_utilization cpu_time_active cpu_time_idle \\\n", + "0 5000.0 ... 0.625 1125002 674998 \n", + "1 4000.0 ... 1.000 1200000 0 \n", + "2 5000.0 ... 0.625 1125000 675000 \n", + "3 4000.0 ... 1.000 1200000 0 \n", + "4 5000.0 ... 0.625 1125000 675000 \n", + "\n", + " cpu_time_steal cpu_time_lost power_draw energy_usage uptime downtime \\\n", + "0 18 0 325.0 97500.075 300000 0 \n", + "1 300011 0 400.0 120000.000 300000 0 \n", + "2 0 0 325.0 97500.000 300000 0 \n", + "3 300000 0 400.0 120000.000 300000 0 \n", + "4 0 0 325.0 97500.000 300000 0 \n", + "\n", + " boot_time \n", + "0 0 \n", + "1 0 \n", + "2 0 \n", + "3 0 \n", + "4 0 \n", + "\n", + "[5 rows x 21 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_host.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "678ede60", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Index(['timestamp', 'host_id', 'cpu_count', 'mem_capacity',\n", + " 'guests_terminated', 'guests_running', 'guests_error', 'guests_invalid',\n", + " 'cpu_limit', 'cpu_usage', 'cpu_demand', 'cpu_utilization',\n", + " 'cpu_time_active', 'cpu_time_idle', 'cpu_time_steal', 'cpu_time_lost',\n", + " 'power_draw', 'energy_usage', 'uptime', 'downtime', 'boot_time'],\n", + " dtype='object')" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_host.columns" + ] } ], "metadata": { diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierPortfolio.kt b/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierPortfolio.kt index f7fd204f..f7452be2 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierPortfolio.kt +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierPortfolio.kt @@ -36,8 +36,8 @@ import org.opendc.experiments.base.portfolio.model.Workload public class GreenifierPortfolio : Portfolio { private val topologies = listOf( - Topology("single"), - Topology("multi"), + Topology("single.json"), + Topology("multi.json"), ) private val workloads = diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierRunner.kt b/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierRunner.kt index 6da35cd1..bd855aac 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierRunner.kt +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/kotlin/org/opendc/experiments/greenifier/GreenifierRunner.kt @@ -65,7 +65,7 @@ public class GreenifierRunner( seed: Long, ) = runSimulation { val serviceDomain = "compute.opendc.org" - val topology = clusterTopology(File(envPath, "${scenario.topology.name}.txt")) + val topology = clusterTopology(File(envPath, scenario.topology.name)) Provisioner(dispatcher, seed).use { provisioner -> provisioner.runSteps( diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/meta.parquet b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/meta.parquet Binary files differnew file mode 100644 index 00000000..2ca31107 --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/meta.parquet diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/trace.parquet b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/trace.parquet Binary files differnew file mode 100644 index 00000000..34fa1c0c --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/benchmark/trace/trace.parquet diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/multi.json b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/multi.json new file mode 100644 index 00000000..721005b0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/multi.json @@ -0,0 +1,66 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 32, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 256000 + } + } + ] + }, + { + "name": "C02", + "hosts" : + [ + { + "name": "H02", + "count": 6, + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 2930 + } + ], + "memory": { + "memorySize": 64000 + } + } + ] + }, + { + "name": "C03", + "hosts" : + [ + { + "name": "H03", + "count": 2, + "cpus": + [ + { + "coreCount": 16, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.json b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.json new file mode 100644 index 00000000..a1c8d95a --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.json @@ -0,0 +1,26 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.txt b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.txt index 5642003d..7c641f4b 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.txt +++ b/opendc-experiments/opendc-experiments-greenifier/src/main/resources/env/single.txt @@ -1,3 +1,2 @@ ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;8;3.2;128;1;128;8 - +A01;A01;8;1;100;1;100;8 diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierIntegrationTest.kt b/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierIntegrationTest.kt index 36b15ee0..15f6cdf6 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierIntegrationTest.kt +++ b/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierIntegrationTest.kt @@ -40,8 +40,8 @@ import org.opendc.compute.simulator.provisioner.setupHosts import org.opendc.compute.telemetry.ComputeMonitor import org.opendc.compute.telemetry.table.HostTableReader import org.opendc.compute.telemetry.table.ServiceTableReader -import org.opendc.compute.topology.HostSpec import org.opendc.compute.topology.clusterTopology +import org.opendc.compute.topology.specs.HostSpec import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.VirtualMachine import org.opendc.compute.workload.sampleByLoad @@ -126,7 +126,7 @@ class GreenifierIntegrationTest { { assertEquals(66977091124, monitor.activeTime) { "Incorrect active time" } }, { assertEquals(3160267873, monitor.stealTime) { "Incorrect steal time" } }, { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } }, - { assertEquals(5.8407E9, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, + { assertEquals(7.767237E9, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, ) } @@ -138,7 +138,7 @@ class GreenifierIntegrationTest { runSimulation { val seed = 1L val workload = createTestWorkload(0.25, seed) - val topology = createTopology("single") + val topology = createTopology("single.json") val monitor = monitor Provisioner(dispatcher, seed).use { provisioner -> @@ -167,7 +167,7 @@ class GreenifierIntegrationTest { { assertEquals(9741285381, monitor.activeTime) { "Active time incorrect" } }, { assertEquals(152, monitor.stealTime) { "Steal time incorrect" } }, { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } }, - { assertEquals(7.0109E8, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, + { assertEquals(7.933686E8, monitor.energyUsage, 1E4) { "Incorrect power draw" } }, ) } @@ -179,7 +179,7 @@ class GreenifierIntegrationTest { runSimulation { val seed = 0L val workload = createTestWorkload(1.0, seed) - val topology = createTopology("single") + val topology = createTopology("single.json") Provisioner(dispatcher, seed).use { provisioner -> provisioner.runSteps( @@ -217,7 +217,7 @@ class GreenifierIntegrationTest { fun testFailures() = runSimulation { val seed = 0L - val topology = createTopology("single") + val topology = createTopology("single.json") val workload = createTestWorkload(0.25, seed) val monitor = monitor @@ -256,8 +256,8 @@ class GreenifierIntegrationTest { /** * Obtain the topology factory for the test. */ - private fun createTopology(name: String = "topology"): List<HostSpec> { - val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name.txt")) + private fun createTopology(name: String = "topology.json"): List<HostSpec> { + val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name")) return stream.use { clusterTopology(stream) } } diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierRunnerTest.kt b/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierRunnerTest.kt index b6d6a6e9..b73317be 100644 --- a/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierRunnerTest.kt +++ b/opendc-experiments/opendc-experiments-greenifier/src/test/kotlin/org/opendc/experiments/greenifier/GreenifierRunnerTest.kt @@ -46,7 +46,8 @@ class GreenifierRunnerTest { private val tracePath = File("src/test/resources/trace") /** - * Smoke test with output. fixme: Fix failures and enable Test + * Smoke test with output. + * fixme: Fix failures and enable */ fun testSmoke() { val outputPath = Files.createTempDirectory("output").toFile() @@ -55,7 +56,7 @@ class GreenifierRunnerTest { val runner = GreenifierRunner(envPath, tracePath, outputPath) val scenario = Scenario( - Topology("topology"), + Topology("topology.json"), Workload("bitbrains-small", trace("bitbrains-small")), OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true), "active-servers", @@ -68,7 +69,8 @@ class GreenifierRunnerTest { } /** - * Smoke test without output. fixme: Fix failures and enable Test + * Smoke test without output. + * fixme: Fix failures and enable */ fun testSmokeNoOutput() { val runner = GreenifierRunner(envPath, tracePath, null) diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.json b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.json new file mode 100644 index 00000000..a1c8d95a --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.json @@ -0,0 +1,26 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.txt b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.txt deleted file mode 100644 index 5642003d..00000000 --- a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/single.txt +++ /dev/null @@ -1,3 +0,0 @@ -ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;8;3.2;128;1;128;8 - diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.json b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.json new file mode 100644 index 00000000..721005b0 --- /dev/null +++ b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.json @@ -0,0 +1,66 @@ +{ + "clusters": + [ + { + "name": "C01", + "hosts" : + [ + { + "name": "H01", + "cpus": + [ + { + "coreCount": 32, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 256000 + } + } + ] + }, + { + "name": "C02", + "hosts" : + [ + { + "name": "H02", + "count": 6, + "cpus": + [ + { + "coreCount": 8, + "coreSpeed": 2930 + } + ], + "memory": { + "memorySize": 64000 + } + } + ] + }, + { + "name": "C03", + "hosts" : + [ + { + "name": "H03", + "count": 2, + "cpus": + [ + { + "coreCount": 16, + "coreSpeed": 3200 + } + ], + "memory": { + "memorySize": 128000 + } + } + ] + } + ] +} + + diff --git a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.txt b/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.txt deleted file mode 100644 index 6b347bff..00000000 --- a/opendc-experiments/opendc-experiments-greenifier/src/test/resources/env/topology.txt +++ /dev/null @@ -1,5 +0,0 @@ -ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost -A01;A01;32;3.2;2048;1;256;32 -B01;B01;48;2.93;1256;6;64;8 -C01;C01;32;3.2;2048;2;128;16 - |